diff options
626 files changed, 65668 insertions, 6526 deletions
diff --git a/.gitignore b/.gitignore index 4f8766a43..18653c238 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ /doc /build /tags +log/ # vim swap files *.swp @@ -19,5 +20,85 @@ cscope.out cscope.in.out cscope.po.out +external/miniupnpc/Makefile +miniupnpcstrings.h +version/ +# Created by https://www.gitignore.io +### C++ ### +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + + +### CMake ### +CMakeCache.txt +CMakeFiles +cmake_install.cmake +install_manifest.txt +*.cmake + +### Linux ### +*~ + +# KDE directory preferences +.directory + + +### Eclipse ### +*.pydevproject +.metadata +.gradle +bin/ +tmp/ +*.tmp +*.bak +*.swp +*~.nib +local.properties +.settings/ +.loadpath + +# External tool builders +.externalToolBuilders/ + +# Locally stored "Eclipse launch configurations" +*.launch + +# CDT-specific +.cproject + +# PDT-specific +.buildpath + +# sbteclipse plugin +.target + +# TeXlipse plugin +.texlipse diff --git a/CMakeLists.txt b/CMakeLists.txt index a3d61daf3..c50e13065 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # @@ -51,7 +51,7 @@ list(INSERT CMAKE_MODULE_PATH 0 if (NOT DEFINED ENV{DEVELOPER_LOCAL_TOOLS}) message(STATUS "Could not find DEVELOPER_LOCAL_TOOLS in env (not required)") set(BOOST_IGNORE_SYSTEM_PATHS_DEFAULT OFF) -elseif (ENV{DEVELOPER_LOCAL_TOOLS} EQUAL 1) +elseif ("$ENV{DEVELOPER_LOCAL_TOOLS}" EQUAL 1) message(STATUS "Found: env DEVELOPER_LOCAL_TOOLS = 1") set(BOOST_IGNORE_SYSTEM_PATHS_DEFAULT ON) else() @@ -62,9 +62,38 @@ endif() message(STATUS "BOOST_IGNORE_SYSTEM_PATHS defaults to ${BOOST_IGNORE_SYSTEM_PATHS_DEFAULT}") option(BOOST_IGNORE_SYSTEM_PATHS "Ignore boost system paths for local boost installation" ${BOOST_IGNORE_SYSTEM_PATHS_DEFAULT}) + +if (NOT DEFINED ENV{DEVELOPER_LIBUNBOUND_OLD}) + message(STATUS "Could not find DEVELOPER_LIBUNBOUND_OLD in env (not required)") +elseif ("$ENV{DEVELOPER_LIBUNBOUND_OLD}" EQUAL 1) + message(STATUS "Found: env DEVELOPER_LIBUNBOUND_OLD = 1, will use the work around") + add_definitions(-DDEVELOPER_LIBUNBOUND_OLD) +elseif ("$ENV{DEVELOPER_LIBUNBOUND_OLD}" EQUAL 0) + message(STATUS "Found: env DEVELOPER_LIBUNBOUND_OLD = 0") +else() + message(STATUS "Found: env DEVELOPER_LIBUNBOUND_OLD with bad value. Will NOT use the work around") +endif() + set_property(GLOBAL PROPERTY USE_FOLDERS ON) enable_testing() +option(BUILD_DOCUMENTATION "Build the Doxygen documentation." ON) + +# Check whether we're on a 32-bit or 64-bit system +if(CMAKE_SIZEOF_VOID_P EQUAL "8") + set(DEFAULT_BUILD_64 ON) +else() + set(DEFAULT_BUILD_64 OFF) +endif() +option(BUILD_64 "Build for 64-bit? 'OFF' builds for 32-bit." ${DEFAULT_BUILD_64}) + +if(BUILD_64) + set(ARCH_WIDTH "64") +else() + set(ARCH_WIDTH "32") +endif() +message(STATUS "Building for a ${ARCH_WIDTH}-bit system") + # Check if we're on FreeBSD so we can exclude the local miniupnpc (it should be installed from ports instead) # CMAKE_SYSTEM_NAME checks are commonly known, but specifically taken from libsdl's CMakeLists if(CMAKE_SYSTEM_NAME MATCHES "kFreeBSD.*") @@ -96,16 +125,16 @@ endif() option(STATIC "Link libraries statically" ${DEFAULT_STATIC}) if(MINGW) - get_filename_component(msys2_install_path "[HKEY_CURRENT_USER\\Software\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\MSYS2 64bit;InstallLocation]" ABSOLUTE) - set(CMAKE_INCLUDE_PATH "${msys2_install_path}/mingw64/include") + get_filename_component(msys2_install_path "[HKEY_CURRENT_USER\\Software\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\MSYS2 ${ARCH_WIDTH}bit;InstallLocation]" ABSOLUTE) + set(CMAKE_INCLUDE_PATH "${msys2_install_path}/mingw${ARCH_WIDTH}/include") # This is necessary because otherwise CMake will make Boost libraries -lfoo # rather than a full path. Unfortunately, this makes the shared libraries get # linked due to a bug in CMake which misses putting -static flags around the # -lfoo arguments. list(REMOVE_ITEM CMAKE_C_IMPLICIT_LINK_DIRECTORIES - "${msys2_install_path}/mingw64/lib") + "${msys2_install_path}/mingw${ARCH_WIDTH}/lib") list(REMOVE_ITEM CMAKE_CXX_IMPLICIT_LINK_DIRECTORIES - "${msys2_install_path}/mingw64/lib") + "${msys2_install_path}/mingw${ARCH_WIDTH}/lib") endif() if(STATIC) @@ -116,6 +145,27 @@ if(STATIC) endif() endif() +# default database: +# should be lmdb for testing, memory for production still +# set(DATABASE memory) +set(DATABASE lmdb) + +if (DEFINED ENV{DATABASE}) + set(DATABASE $ENV{DATABASE}) + message(STATUS "DATABASE set: ${DATABASE}") +else() + message(STATUS "Could not find DATABASE in env (not required unless you want to change database type from default: ${DATABASE})") +endif() +if (DATABASE STREQUAL "lmdb") + set(BLOCKCHAIN_DB DB_LMDB) +elseif (DATABASE STREQUAL "memory") + set(BLOCKCHAIN_DB DB_MEMORY) +else() + die("Invalid database type: ${DATABASE}") +endif() + +add_definitions("-DBLOCKCHAIN_DB=${BLOCKCHAIN_DB}") + if (UNIX AND NOT APPLE) # Note that at the time of this writing the -Wstrict-prototypes flag added below will make this fail find_package(Threads) @@ -138,6 +188,14 @@ link_directories(${UNBOUND_LIBRARY_DIRS}) # Final setup for rapidjson include_directories(external/rapidjson) +# Final setup for liblmdb +include_directories(${LMDB_INCLUDE}) + +# Final setup for Berkeley DB +if (NOT STATIC) + include_directories(${BDB_INCLUDE}) +endif() + if(MSVC) add_definitions("/bigobj /MP /W3 /GS- /D_CRT_SECURE_NO_WARNINGS /wd4996 /wd4345 /D_WIN32_WINNT=0x0600 /DWIN32_LEAN_AND_MEAN /DGTEST_HAS_TR1_TUPLE=0 /FIinline_c.h /D__SSE4_1__") # set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /Dinline=__inline") @@ -157,7 +215,7 @@ else() endif() set(WARNINGS "-Wall -Wextra -Wpointer-arith -Wundef -Wvla -Wwrite-strings -Wno-error=extra -Wno-error=deprecated-declarations -Wno-error=sign-compare -Wno-error=strict-aliasing -Wno-error=type-limits -Wno-unused-parameter -Wno-error=unused-variable -Wno-error=undef -Wno-error=uninitialized -Wno-deprecated-register") if(NOT MINGW) - set(WARNINGS "${WARNINGS} -Werror") + set(WARNINGS "${WARNINGS} -Werror") # to allow pedantic but not stop compilation endif() if(CMAKE_C_COMPILER_ID STREQUAL "Clang") set(WARNINGS "${WARNINGS} -Wno-error=mismatched-tags -Wno-error=null-conversion -Wno-overloaded-shift-op-parentheses -Wno-error=shift-count-overflow -Wno-error=tautological-constant-out-of-range-compare -Wno-error=unused-private-field -Wno-error=unneeded-internal-declaration") @@ -180,8 +238,27 @@ else() else() set(STATIC_ASSERT_FLAG "-Dstatic_assert=_Static_assert") endif() - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11 -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_FLAG} ${WARNINGS} ${C_WARNINGS} ${ARCH_FLAG} -maes") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -D_GNU_SOURCE ${MINGW_FLAG} ${WARNINGS} ${CXX_WARNINGS} ${ARCH_FLAG} -maes") + + option(NO_AES "Explicitly disable AES support" ${NO_AES}) + + if (NO_AES) + message(STATUS "Disabling AES support") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11 -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_FLAG} ${WARNINGS} ${C_WARNINGS} ${ARCH_FLAG}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -D_GNU_SOURCE ${MINGW_FLAG} ${WARNINGS} ${CXX_WARNINGS} ${ARCH_FLAG}") + else() + message(STATUS "Enabling AES support") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11 -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_FLAG} ${WARNINGS} ${C_WARNINGS} ${ARCH_FLAG} -maes") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -D_GNU_SOURCE ${MINGW_FLAG} ${WARNINGS} ${CXX_WARNINGS} ${ARCH_FLAG} -maes") + endif() + + string(SUBSTRING ${ARCH} 0 3 ARM_TEST) + string(TOLOWER ${ARM_TEST} ARM_TEST) + if(${ARM_TEST} STREQUAL "arm") + message(STATUS "Setting ARM C and C++ flags") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mfpu=vfp -mfloat-abi=hard") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mfpu=vfp -mfloat-abi=hard") + endif() + if(APPLE) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DGTEST_HAS_TR1_TUPLE=0") endif() @@ -223,16 +300,22 @@ else() endif() endif() -if (BOOST_IGNORE_SYSTEM_PATHS) +if (${BOOST_IGNORE_SYSTEM_PATHS} STREQUAL "ON") set(Boost_NO_SYSTEM_PATHS TRUE) endif() +set(OLD_LIB_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) if(STATIC) + if(MINGW) + set(CMAKE_FIND_LIBRARY_SUFFIXES .a) + endif() + set(Boost_USE_STATIC_LIBS ON) set(Boost_USE_STATIC_RUNTIME ON) endif() find_package(Boost 1.53 QUIET REQUIRED COMPONENTS system filesystem thread date_time chrono regex serialization program_options) +set(CMAKE_FIND_LIBRARY_SUFFIXES ${OLD_LIB_SUFFIXES}) if(NOT Boost_FOUND) die("Could not find Boost libraries, please make sure you have installed Boost or libboost-all-dev (1.53 or 1.55+) or the equivalent") endif() @@ -253,8 +336,34 @@ endif() include(version.cmake) +add_subdirectory(contrib) add_subdirectory(src) if(BUILD_TESTS) add_subdirectory(tests) endif() + + + +if(BUILD_DOCUMENTATION) + set(DOC_GRAPHS "YES" CACHE STRING "Create dependency graphs (needs graphviz)") + set(DOC_FULLGRAPHS "NO" CACHE STRING "Create call/callee graphs (large)") + + find_program(DOT_PATH dot) + + if (DOT_PATH STREQUAL "DOT_PATH-NOTFOUND") + message("Doxygen: graphviz not found - graphs disabled") + set(DOC_GRAPHS "NO") + endif() + + find_package(Doxygen) + if(DOXYGEN_FOUND) + configure_file("cmake/Doxyfile.in" "Doxyfile" @ONLY) + configure_file("cmake/Doxygen.extra.css.in" "Doxygen.extra.css" @ONLY) + add_custom_target(doc + ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Generating API documentation with Doxygen.." VERBATIM) + endif() +endif() + @@ -1,17 +1,17 @@ -all: all-release +all: release-all cmake-debug: mkdir -p build/debug cd build/debug && cmake -D CMAKE_BUILD_TYPE=Debug ../.. -build-debug: cmake-debug +debug: cmake-debug cd build/debug && $(MAKE) -test-debug: build-debug +debug-test: debug mkdir -p build/debug cd build/debug && cmake -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=Debug ../.. && $(MAKE) test -all-debug: +debug-all: mkdir -p build/debug cd build/debug && cmake -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=Debug ../.. && $(MAKE) @@ -19,20 +19,38 @@ cmake-release: mkdir -p build/release cd build/release && cmake -D CMAKE_BUILD_TYPE=Release ../.. -build-release: cmake-release +release: cmake-release cd build/release && $(MAKE) -test-release: build-release +release-test: release mkdir -p build/release cd build/release && cmake -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=release ../.. && $(MAKE) test -all-release: +release-all: mkdir -p build/release cd build/release && cmake -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=release ../.. && $(MAKE) -release-static: +release-static: release-static-64 + +release-static-64: + mkdir -p build/release + cd build/release && cmake -D STATIC=ON -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_BUILD_TYPE=release ../.. && $(MAKE) + +release-static-32: + mkdir -p build/release + cd build/release && cmake -D STATIC=ON -D ARCH="i686" -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=release ../.. && $(MAKE) + +release-static-arm6: + mkdir -p build/release + cd build/release && cmake -D STATIC=ON -D ARCH="armv6zk" -D BUILD_64=OFF -D NO_AES=ON -D CMAKE_BUILD_TYPE=release ../.. && $(MAKE) + +release-static-win64: + mkdir -p build/release + cd build/release && cmake -G "MSYS Makefiles" -D STATIC=ON -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_BUILD_TYPE=Release -D CMAKE_TOOLCHAIN_FILE=../../cmake/64-bit-toolchain.cmake -D MSYS2_FOLDER=c:/msys64 ../.. && $(MAKE) + +release-static-win32: mkdir -p build/release - cd build/release && cmake -D STATIC=ON -D CMAKE_BUILD_TYPE=release ../.. && $(MAKE) + cd build/release && cmake -G "MSYS Makefiles" -D STATIC=ON -D ARCH="i686" -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=Release -D CMAKE_TOOLCHAIN_FILE=../../cmake/32-bit-toolchain.cmake -D MSYS2_FOLDER=c:/msys32 ../.. && $(MAKE) clean: @echo "WARNING: Back-up your wallet if it exists within ./build!" ; \ @@ -43,4 +61,4 @@ clean: tags: ctags -R --sort=1 --c++-kinds=+p --fields=+iaS --extra=+q --language-force=C++ src contrib tests/gtest -.PHONY: all cmake-debug build-debug test-debug all-debug cmake-release build-release test-release all-release clean tags +.PHONY: all cmake-debug debug debug-test debug-all cmake-release release release-test release-all clean tags @@ -1,11 +1,12 @@ # Monero -Copyright (c) 2014, The Monero Project +Copyright (c) 2014-2015, The Monero Project ## Development Resources -Web: [monero.cc](http://monero.cc) -Mail: [dev@monero.cc](mailto:dev@monero.cc) +Web: [getmonero.org](https://getmonero.org) +Forum: [forum.getmonero.org](https://forum.getmonero.org) +Mail: [dev@getmonero.org](mailto:dev@getmonero.org) Github (staging): [https://github.com/monero-project/bitmonero](https://github.com/monero-project/bitmonero) Github (development): [http://github.com/monero-project/bitmonero/tree/development](http://github.com/monero-project/bitmonero/tree/development) IRC: [#monero-dev on Freenode](irc://chat.freenode.net/#monero-dev) @@ -28,9 +29,25 @@ As with many development projects, the repository on Github is considered to be Anyone is able to contribute to Monero. If you have a fix or code change, feel free to submit is as a pull request directly to the "development" branch. In cases where the change is relatively small or does not affect other parts of the codebase it may be merged in immediately by any one of the collaborators. On the other hand, if the change is particularly large or complex, it is expected that it will be discussed at length either well in advance of the pull request being submitted, or even directly on the pull request. +## Supporting the Project + +Monero development can be supported directly through donations. + +Both Monero and Bitcoin donations can be made to donate.getmonero.org if using a client that supports the [OpenAlias](https://openalias.org) standard + +The Monero donation address is: 46BeWrHpwXmHDpDEUmZBWZfoQpdc6HaERCNmx1pEYL2rAcuwufPN9rXHHtyUA4QVy66qeFQkn6sfK8aHYjA3jk3o1Bv16em (viewkey: e422831985c9205238ef84daf6805526c14d96fd7b059fe68c7ab98e495e5703) + +The Bitcoin donation address is: 1FhnVJi2V1k4MqXm2nHoEbY5LV7FPai7bb + +Core development funding and/or some supporting services are also graciously provided by sponsors: + +[](https://mymonero.com) [](http://dome9.com) + +There are also several mining pools that kindly donate a portion of their fees, [a list of them can be found on our Bitcointalk post](https://bitcointalk.org/index.php?topic=583449.0). + ## License -Copyright (c) 2014, The Monero Project +Copyright (c) 2014-2015, The Monero Project All rights reserved. @@ -56,14 +73,14 @@ Static Build Additional Dependencies: ldns 1.6.17 or later, expat 1.1 or later, **Basic Process:** * To build, change to the root of the source code directory, and run `make`. -* The resulting executables can be found in `build/release/src` or `build/debug/src`, depending on what you're building. +* The resulting executables can be found in `build/release/bin` or `build/debug/bin`, depending on what you're building. **Advanced options:** * Parallel build: run `make -j<number of threads>` instead of `make`. * Statically linked release build: run `make release-static`. -* Debug build: run `make build-debug`. -* Test suite: run `make test-release` to run tests in addition to building. Running `make test-debug` will do the same to the debug version. +* Debug build: run `make debug`. +* Test suite: run `make release-test` to run tests in addition to building. Running `make debug-test` will do the same to the debug version. * Building with Clang: it may be possible to use Clang instead of GCC, but this may not work everywhere. To build, run `export CC=clang CXX=clang++` before running `make`. ### On OS X: @@ -74,7 +91,7 @@ Alternatively, it can be built in an easier and more automated fashion using Hom * Ensure Homebrew is installed, it can be found at http://brew.sh * Add the repository to brew: `brew tap sammy007/cryptonight` -* Build Monero: `brew install --HEAD bitmonero` +* Build Monero: `brew install bitmonero --build-from-source` ### On Windows: @@ -111,7 +128,7 @@ cmake -G "MSYS Makefiles" -D CMAKE_BUILD_TYPE=Release -D CMAKE_TOOLCHAIN_FILE=.. cmake -G "MSYS Makefiles" -D CMAKE_BUILD_TYPE=Release -D CMAKE_TOOLCHAIN_FILE=../cmake/32-bit-toolchain.cmake -D MSYS2_FOLDER=c:/msys32 .. ``` * You can now run `make` to have it build -* The resulting executables can be found in `build/release/src` or `build/debug/src`, depending on what you're building. +* The resulting executables can be found in `build/release/bin` or `build/debug/bin`, depending on what you're building. If you installed MSYS2 in a folder other than c:/msys64, make the appropriate substitution above. @@ -119,8 +136,8 @@ If you installed MSYS2 in a folder other than c:/msys64, make the appropriate su * Parallel build: run `make -j<number of threads>` instead of `make`. * Statically linked release build: run `make release-static`. -* Debug build: run `make build-debug`. -* Test suite: run `make test-release` to run tests in addition to building. Running `make test-debug` will do the same to the debug version. +* Debug build: run `make debug`. +* Test suite: run `make release-test` to run tests in addition to building. Running `make debug-test` will do the same to the debug version. ### On FreeBSD: @@ -136,4 +153,4 @@ Dependencies: Doxygen 1.8.0 or later, Graphviz 2.28 or later (optional). * To build, change to the root of the source code directory, and run `doxygen Doxyfile` * If you have installed Graphviz, you can also generate in-doc diagrams by instead running `HAVE_DOT=YES doxygen Doxyfile` -* The output will be built in doc/html/
\ No newline at end of file +* The output will be built in doc/html/ diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt deleted file mode 100644 index b2a3e0081..000000000 --- a/ReleaseNotes.txt +++ /dev/null @@ -1,47 +0,0 @@ -Monero - -Release notes 0.8.8 - -- JSON RPC v2.0 compatibility -- JSON RPC over TCP added- Further optimizations -- Fixed a bug with checkpoints behavior -- UI improvements for daemon -- Fixed COIN value (10^12) -- Raised minimum fee to 5 * (10^9) -- Temporary fix for block reward DoS attack - -Release notes 0.8.7 - -- Slow hash optimizations - -Release notes 0.8.6 - -- Simplwallet can set extra for transfers -- Improvements in JSON RPC for wallet -- UX improvements in simplewallet -- Win32 compilation -- Mac OSX compilation - -Release notes 0.8.5 - -- Port mapping for daemon using UPnP -- Improvements for daemon JSON RPC API - -Release notes 0.8.4 - -- Formalization of high level API -- Improvements for wallet JSON RPC API - -Release notes 0.8.3 - -- JSON RPC for wallet -- Fixed bug with blockchain storing -- Fixed bug with correct display of transfer's change -- Bug fix in simple wallet - -Release notes 0.8.2 - -- Now wallet is still working when daemon stores blockchain and can't serve clients -- No-console option for a daemon -- Fixed problem with network synchronization -- Major bug fix in simple wallet diff --git a/cmake/Doxyfile.in b/cmake/Doxyfile.in new file mode 100644 index 000000000..35a4911f8 --- /dev/null +++ b/cmake/Doxyfile.in @@ -0,0 +1,1803 @@ +# Doxyfile 1.8.1.2 + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for a project +# +# All text after a hash (#) is considered a comment and will be ignored +# The format is: +# TAG = value [value, ...] +# For lists items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (" ") + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- + +# This tag specifies the encoding used for all characters in the config file +# that follow. The default is UTF-8 which is also the encoding used for all +# text before the first occurrence of this tag. Doxygen uses libiconv (or the +# iconv built into libc) for the transcoding. See +# http://www.gnu.org/software/libiconv for the list of possible encodings. + +DOXYFILE_ENCODING = UTF-8 + +# The PROJECT_NAME tag is a single word (or sequence of words) that should +# identify the project. Note that if you do not use Doxywizard you need +# to put quotes around the project name if it contains spaces. + +PROJECT_NAME = Monero + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. +# This could be handy for archiving the generated documentation or +# if some version control system is used. + +PROJECT_NUMBER = @VERSION_STRING@ + +# Using the PROJECT_BRIEF tag one can provide an optional one line description +# for a project that appears at the top of each page and should give viewer +# a quick idea about the purpose of the project. Keep the description short. + +PROJECT_BRIEF = @CPACK_PACKAGE_DESCRIPTION_SUMMARY@ + +# With the PROJECT_LOGO tag one can specify an logo or icon that is +# included in the documentation. The maximum height of the logo should not +# exceed 55 pixels and the maximum width should not exceed 200 pixels. +# Doxygen will copy the logo to the output directory. + +PROJECT_LOGO = + +# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) +# base path where the generated documentation will be put. +# If a relative path is entered, it will be relative to the location +# where doxygen was started. If left blank the current directory will be used. + +OUTPUT_DIRECTORY = ./docs + +# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create +# 4096 sub-directories (in 2 levels) under the output directory of each output +# format and will distribute the generated files over these directories. +# Enabling this option can be useful when feeding doxygen a huge amount of +# source files, where putting all generated files in the same directory would +# otherwise cause performance problems for the file system. + +CREATE_SUBDIRS = YES + +# The OUTPUT_LANGUAGE tag is used to specify the language in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all constant output in the proper language. +# The default language is English, other supported languages are: +# Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional, +# Croatian, Czech, Danish, Dutch, Esperanto, Farsi, Finnish, French, German, +# Greek, Hungarian, Italian, Japanese, Japanese-en (Japanese with English +# messages), Korean, Korean-en, Lithuanian, Norwegian, Macedonian, Persian, +# Polish, Portuguese, Romanian, Russian, Serbian, Serbian-Cyrillic, Slovak, +# Slovene, Spanish, Swedish, Ukrainian, and Vietnamese. + +OUTPUT_LANGUAGE = English + +# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will +# include brief member descriptions after the members that are listed in +# the file and class documentation (similar to JavaDoc). +# Set to NO to disable this. + +BRIEF_MEMBER_DESC = YES + +# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend +# the brief description of a member or function before the detailed description. +# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the +# brief descriptions will be completely suppressed. + +REPEAT_BRIEF = YES + +# This tag implements a quasi-intelligent brief description abbreviator +# that is used to form the text in various listings. Each string +# in this list, if found as the leading text of the brief description, will be +# stripped from the text and the result after processing the whole list, is +# used as the annotated text. Otherwise, the brief description is used as-is. +# If left blank, the following values are used ("$name" is automatically +# replaced with the name of the entity): "The $name class" "The $name widget" +# "The $name file" "is" "provides" "specifies" "contains" +# "represents" "a" "an" "the" + +ABBREVIATE_BRIEF = + +# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then +# Doxygen will generate a detailed section even if there is only a brief +# description. + +ALWAYS_DETAILED_SEC = NO + +# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all +# inherited members of a class in the documentation of that class as if those +# members were ordinary class members. Constructors, destructors and assignment +# operators of the base classes will not be shown. + +INLINE_INHERITED_MEMB = NO + +# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full +# path before files name in the file list and in the header files. If set +# to NO the shortest path that makes the file name unique will be used. + +FULL_PATH_NAMES = YES + +# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag +# can be used to strip a user-defined part of the path. Stripping is +# only done if one of the specified strings matches the left-hand part of +# the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the +# path to strip. + +STRIP_FROM_PATH = + +# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of +# the path mentioned in the documentation of a class, which tells +# the reader which header file to include in order to use a class. +# If left blank only the name of the header file containing the class +# definition is used. Otherwise one should specify the include paths that +# are normally passed to the compiler using the -I flag. + +STRIP_FROM_INC_PATH = + +# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter +# (but less readable) file names. This can be useful if your file system +# doesn't support long names like on DOS, Mac, or CD-ROM. + +SHORT_NAMES = NO + +# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen +# will interpret the first line (until the first dot) of a JavaDoc-style +# comment as the brief description. If set to NO, the JavaDoc +# comments will behave just like regular Qt-style comments +# (thus requiring an explicit @brief command for a brief description.) + +JAVADOC_AUTOBRIEF = NO + +# If the QT_AUTOBRIEF tag is set to YES then Doxygen will +# interpret the first line (until the first dot) of a Qt-style +# comment as the brief description. If set to NO, the comments +# will behave just like regular Qt-style comments (thus requiring +# an explicit \brief command for a brief description.) + +QT_AUTOBRIEF = NO + +# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen +# treat a multi-line C++ special comment block (i.e. a block of //! or /// +# comments) as a brief description. This used to be the default behaviour. +# The new default is to treat a multi-line C++ comment block as a detailed +# description. Set this tag to YES if you prefer the old behaviour instead. + +MULTILINE_CPP_IS_BRIEF = NO + +# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented +# member inherits the documentation from any documented member that it +# re-implements. + +INHERIT_DOCS = YES + +# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce +# a new page for each member. If set to NO, the documentation of a member will +# be part of the file/class/namespace that contains it. + +SEPARATE_MEMBER_PAGES = NO + +# The TAB_SIZE tag can be used to set the number of spaces in a tab. +# Doxygen uses this value to replace tabs by spaces in code fragments. + +TAB_SIZE = 4 + +# This tag can be used to specify a number of aliases that acts +# as commands in the documentation. An alias has the form "name=value". +# For example adding "sideeffect=\par Side Effects:\n" will allow you to +# put the command \sideeffect (or @sideeffect) in the documentation, which +# will result in a user-defined paragraph with heading "Side Effects:". +# You can put \n's in the value part of an alias to insert newlines. + +ALIASES = + +# This tag can be used to specify a number of word-keyword mappings (TCL only). +# A mapping has the form "name=value". For example adding +# "class=itcl::class" will allow you to use the command class in the +# itcl::class meaning. + +TCL_SUBST = + +# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C +# sources only. Doxygen will then generate output that is more tailored for C. +# For instance, some of the names that are used will be different. The list +# of all members will be omitted, etc. + +OPTIMIZE_OUTPUT_FOR_C = NO + +# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java +# sources only. Doxygen will then generate output that is more tailored for +# Java. For instance, namespaces will be presented as packages, qualified +# scopes will look different, etc. + +OPTIMIZE_OUTPUT_JAVA = NO + +# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran +# sources only. Doxygen will then generate output that is more tailored for +# Fortran. + +OPTIMIZE_FOR_FORTRAN = NO + +# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL +# sources. Doxygen will then generate output that is tailored for +# VHDL. + +OPTIMIZE_OUTPUT_VHDL = NO + +# Doxygen selects the parser to use depending on the extension of the files it +# parses. With this tag you can assign which parser to use for a given extension. +# Doxygen has a built-in mapping, but you can override or extend it using this +# tag. The format is ext=language, where ext is a file extension, and language +# is one of the parsers supported by doxygen: IDL, Java, Javascript, CSharp, C, +# C++, D, PHP, Objective-C, Python, Fortran, VHDL, C, C++. For instance to make +# doxygen treat .inc files as Fortran files (default is PHP), and .f files as C +# (default is Fortran), use: inc=Fortran f=C. Note that for custom extensions +# you also need to set FILE_PATTERNS otherwise the files are not read by doxygen. + +EXTENSION_MAPPING = + +# If MARKDOWN_SUPPORT is enabled (the default) then doxygen pre-processes all +# comments according to the Markdown format, which allows for more readable +# documentation. See http://daringfireball.net/projects/markdown/ for details. +# The output of markdown processing is further processed by doxygen, so you +# can mix doxygen, HTML, and XML commands with Markdown formatting. +# Disable only in case of backward compatibilities issues. + +MARKDOWN_SUPPORT = YES + +# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want +# to include (a tag file for) the STL sources as input, then you should +# set this tag to YES in order to let doxygen match functions declarations and +# definitions whose arguments contain STL classes (e.g. func(std::string); v.s. +# func(std::string) {}). This also makes the inheritance and collaboration +# diagrams that involve STL classes more complete and accurate. + +BUILTIN_STL_SUPPORT = YES + +# If you use Microsoft's C++/CLI language, you should set this option to YES to +# enable parsing support. + +CPP_CLI_SUPPORT = NO + +# Set the SIP_SUPPORT tag to YES if your project consists of sip sources only. +# Doxygen will parse them like normal C++ but will assume all classes use public +# instead of private inheritance when no explicit protection keyword is present. + +SIP_SUPPORT = NO + +# For Microsoft's IDL there are propget and propput attributes to indicate getter +# and setter methods for a property. Setting this option to YES (the default) +# will make doxygen replace the get and set methods by a property in the +# documentation. This will only work if the methods are indeed getting or +# setting a simple type. If this is not the case, or you want to show the +# methods anyway, you should set this option to NO. + +IDL_PROPERTY_SUPPORT = YES + +# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC +# tag is set to YES, then doxygen will reuse the documentation of the first +# member in the group (if any) for the other members of the group. By default +# all members of a group must be documented explicitly. + +DISTRIBUTE_GROUP_DOC = NO + +# Set the SUBGROUPING tag to YES (the default) to allow class member groups of +# the same type (for instance a group of public functions) to be put as a +# subgroup of that type (e.g. under the Public Functions section). Set it to +# NO to prevent subgrouping. Alternatively, this can be done per class using +# the \nosubgrouping command. + +SUBGROUPING = YES + +# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and +# unions are shown inside the group in which they are included (e.g. using +# @ingroup) instead of on a separate page (for HTML and Man pages) or +# section (for LaTeX and RTF). + +INLINE_GROUPED_CLASSES = NO + +# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and +# unions with only public data fields will be shown inline in the documentation +# of the scope in which they are defined (i.e. file, namespace, or group +# documentation), provided this scope is documented. If set to NO (the default), +# structs, classes, and unions are shown on a separate page (for HTML and Man +# pages) or section (for LaTeX and RTF). + +INLINE_SIMPLE_STRUCTS = NO + +# When TYPEDEF_HIDES_STRUCT is enabled, a typedef of a struct, union, or enum +# is documented as struct, union, or enum with the name of the typedef. So +# typedef struct TypeS {} TypeT, will appear in the documentation as a struct +# with name TypeT. When disabled the typedef will appear as a member of a file, +# namespace, or class. And the struct will be named TypeS. This can typically +# be useful for C code in case the coding convention dictates that all compound +# types are typedef'ed and only the typedef is referenced, never the tag name. + +TYPEDEF_HIDES_STRUCT = NO + +# The SYMBOL_CACHE_SIZE determines the size of the internal cache use to +# determine which symbols to keep in memory and which to flush to disk. +# When the cache is full, less often used symbols will be written to disk. +# For small to medium size projects (<1000 input files) the default value is +# probably good enough. For larger projects a too small cache size can cause +# doxygen to be busy swapping symbols to and from disk most of the time +# causing a significant performance penalty. +# If the system has enough physical memory increasing the cache will improve the +# performance by keeping more symbols in memory. Note that the value works on +# a logarithmic scale so increasing the size by one will roughly double the +# memory usage. The cache size is given by this formula: +# 2^(16+SYMBOL_CACHE_SIZE). The valid range is 0..9, the default is 0, +# corresponding to a cache size of 2^16 = 65536 symbols. + +SYMBOL_CACHE_SIZE = 0 + +# Similar to the SYMBOL_CACHE_SIZE the size of the symbol lookup cache can be +# set using LOOKUP_CACHE_SIZE. This cache is used to resolve symbols given +# their name and scope. Since this can be an expensive process and often the +# same symbol appear multiple times in the code, doxygen keeps a cache of +# pre-resolved symbols. If the cache is too small doxygen will become slower. +# If the cache is too large, memory is wasted. The cache size is given by this +# formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range is 0..9, the default is 0, +# corresponding to a cache size of 2^16 = 65536 symbols. + +LOOKUP_CACHE_SIZE = 0 + +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- + +# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in +# documentation are documented, even if no documentation was available. +# Private class members and static file members will be hidden unless +# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES + +EXTRACT_ALL = YES + +# If the EXTRACT_PRIVATE tag is set to YES all private members of a class +# will be included in the documentation. + +EXTRACT_PRIVATE = NO + +# If the EXTRACT_PACKAGE tag is set to YES all members with package or internal +# scope will be included in the documentation. + +EXTRACT_PACKAGE = NO + +# If the EXTRACT_STATIC tag is set to YES all static members of a file +# will be included in the documentation. + +EXTRACT_STATIC = NO + +# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) +# defined locally in source files will be included in the documentation. +# If set to NO only classes defined in header files are included. + +EXTRACT_LOCAL_CLASSES = YES + +# This flag is only useful for Objective-C code. When set to YES local +# methods, which are defined in the implementation section but not in +# the interface are included in the documentation. +# If set to NO (the default) only methods in the interface are included. + +EXTRACT_LOCAL_METHODS = NO + +# If this flag is set to YES, the members of anonymous namespaces will be +# extracted and appear in the documentation as a namespace called +# 'anonymous_namespace{file}', where file will be replaced with the base +# name of the file that contains the anonymous namespace. By default +# anonymous namespaces are hidden. + +EXTRACT_ANON_NSPACES = NO + +# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all +# undocumented members of documented classes, files or namespaces. +# If set to NO (the default) these members will be included in the +# various overviews, but no documentation section is generated. +# This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_MEMBERS = NO + +# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all +# undocumented classes that are normally visible in the class hierarchy. +# If set to NO (the default) these classes will be included in the various +# overviews. This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_CLASSES = NO + +# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all +# friend (class|struct|union) declarations. +# If set to NO (the default) these declarations will be included in the +# documentation. + +HIDE_FRIEND_COMPOUNDS = NO + +# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any +# documentation blocks found inside the body of a function. +# If set to NO (the default) these blocks will be appended to the +# function's detailed documentation block. + +HIDE_IN_BODY_DOCS = NO + +# The INTERNAL_DOCS tag determines if documentation +# that is typed after a \internal command is included. If the tag is set +# to NO (the default) then the documentation will be excluded. +# Set it to YES to include the internal documentation. + +INTERNAL_DOCS = NO + +# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate +# file names in lower-case letters. If set to YES upper-case letters are also +# allowed. This is useful if you have classes or files whose names only differ +# in case and if your file system supports case sensitive file names. Windows +# and Mac users are advised to set this option to NO. + +CASE_SENSE_NAMES = NO + +# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen +# will show members with their full class and namespace scopes in the +# documentation. If set to YES the scope will be hidden. + +HIDE_SCOPE_NAMES = NO + +# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen +# will put a list of the files that are included by a file in the documentation +# of that file. + +SHOW_INCLUDE_FILES = YES + +# If the FORCE_LOCAL_INCLUDES tag is set to YES then Doxygen +# will list include files with double quotes in the documentation +# rather than with sharp brackets. + +FORCE_LOCAL_INCLUDES = NO + +# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] +# is inserted in the documentation for inline members. + +INLINE_INFO = YES + +# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen +# will sort the (detailed) documentation of file and class members +# alphabetically by member name. If set to NO the members will appear in +# declaration order. + +SORT_MEMBER_DOCS = YES + +# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the +# brief documentation of file, namespace and class members alphabetically +# by member name. If set to NO (the default) the members will appear in +# declaration order. + +SORT_BRIEF_DOCS = NO + +# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen +# will sort the (brief and detailed) documentation of class members so that +# constructors and destructors are listed first. If set to NO (the default) +# the constructors will appear in the respective orders defined by +# SORT_MEMBER_DOCS and SORT_BRIEF_DOCS. +# This tag will be ignored for brief docs if SORT_BRIEF_DOCS is set to NO +# and ignored for detailed docs if SORT_MEMBER_DOCS is set to NO. + +SORT_MEMBERS_CTORS_1ST = NO + +# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the +# hierarchy of group names into alphabetical order. If set to NO (the default) +# the group names will appear in their defined order. + +SORT_GROUP_NAMES = NO + +# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be +# sorted by fully-qualified names, including namespaces. If set to +# NO (the default), the class list will be sorted only by class name, +# not including the namespace part. +# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. +# Note: This option applies only to the class list, not to the +# alphabetical list. + +SORT_BY_SCOPE_NAME = NO + +# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to +# do proper type resolution of all parameters of a function it will reject a +# match between the prototype and the implementation of a member function even +# if there is only one candidate or it is obvious which candidate to choose +# by doing a simple string match. By disabling STRICT_PROTO_MATCHING doxygen +# will still accept a match between prototype and implementation in such cases. + +STRICT_PROTO_MATCHING = NO + +# The GENERATE_TODOLIST tag can be used to enable (YES) or +# disable (NO) the todo list. This list is created by putting \todo +# commands in the documentation. + +GENERATE_TODOLIST = YES + +# The GENERATE_TESTLIST tag can be used to enable (YES) or +# disable (NO) the test list. This list is created by putting \test +# commands in the documentation. + +GENERATE_TESTLIST = YES + +# The GENERATE_BUGLIST tag can be used to enable (YES) or +# disable (NO) the bug list. This list is created by putting \bug +# commands in the documentation. + +GENERATE_BUGLIST = YES + +# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or +# disable (NO) the deprecated list. This list is created by putting +# \deprecated commands in the documentation. + +GENERATE_DEPRECATEDLIST= YES + +# The ENABLED_SECTIONS tag can be used to enable conditional +# documentation sections, marked by \if sectionname ... \endif. + +ENABLED_SECTIONS = + +# The MAX_INITIALIZER_LINES tag determines the maximum number of lines +# the initial value of a variable or macro consists of for it to appear in +# the documentation. If the initializer consists of more lines than specified +# here it will be hidden. Use a value of 0 to hide initializers completely. +# The appearance of the initializer of individual variables and macros in the +# documentation can be controlled using \showinitializer or \hideinitializer +# command in the documentation regardless of this setting. + +MAX_INITIALIZER_LINES = 30 + +# Set the SHOW_USED_FILES tag to NO to disable the list of files generated +# at the bottom of the documentation of classes and structs. If set to YES the +# list will mention the files that were used to generate the documentation. + +SHOW_USED_FILES = YES + +# Set the SHOW_FILES tag to NO to disable the generation of the Files page. +# This will remove the Files entry from the Quick Index and from the +# Folder Tree View (if specified). The default is YES. + +SHOW_FILES = YES + +# Set the SHOW_NAMESPACES tag to NO to disable the generation of the +# Namespaces page. This will remove the Namespaces entry from the Quick Index +# and from the Folder Tree View (if specified). The default is YES. + +SHOW_NAMESPACES = YES + +# The FILE_VERSION_FILTER tag can be used to specify a program or script that +# doxygen should invoke to get the current version for each file (typically from +# the version control system). Doxygen will invoke the program by executing (via +# popen()) the command <command> <input-file>, where <command> is the value of +# the FILE_VERSION_FILTER tag, and <input-file> is the name of an input file +# provided by doxygen. Whatever the program writes to standard output +# is used as the file version. See the manual for examples. + +FILE_VERSION_FILTER = + +# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed +# by doxygen. The layout file controls the global structure of the generated +# output files in an output format independent way. To create the layout file +# that represents doxygen's defaults, run doxygen with the -l option. +# You can optionally specify a file name after the option, if omitted +# DoxygenLayout.xml will be used as the name of the layout file. + +LAYOUT_FILE = + +# The CITE_BIB_FILES tag can be used to specify one or more bib files +# containing the references data. This must be a list of .bib files. The +# .bib extension is automatically appended if omitted. Using this command +# requires the bibtex tool to be installed. See also +# http://en.wikipedia.org/wiki/BibTeX for more info. For LaTeX the style +# of the bibliography can be controlled using LATEX_BIB_STYLE. To use this +# feature you need bibtex and perl available in the search path. + +CITE_BIB_FILES = + +#--------------------------------------------------------------------------- +# configuration options related to warning and progress messages +#--------------------------------------------------------------------------- + +# The QUIET tag can be used to turn on/off the messages that are generated +# by doxygen. Possible values are YES and NO. If left blank NO is used. + +QUIET = NO + +# The WARNINGS tag can be used to turn on/off the warning messages that are +# generated by doxygen. Possible values are YES and NO. If left blank +# NO is used. + +WARNINGS = YES + +# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings +# for undocumented members. If EXTRACT_ALL is set to YES then this flag will +# automatically be disabled. + +WARN_IF_UNDOCUMENTED = YES + +# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for +# potential errors in the documentation, such as not documenting some +# parameters in a documented function, or documenting parameters that +# don't exist or using markup commands wrongly. + +WARN_IF_DOC_ERROR = YES + +# The WARN_NO_PARAMDOC option can be enabled to get warnings for +# functions that are documented, but have no documentation for their parameters +# or return value. If set to NO (the default) doxygen will only warn about +# wrong or incomplete parameter documentation, but not about the absence of +# documentation. + +WARN_NO_PARAMDOC = NO + +# The WARN_FORMAT tag determines the format of the warning messages that +# doxygen can produce. The string should contain the $file, $line, and $text +# tags, which will be replaced by the file and line number from which the +# warning originated and the warning text. Optionally the format may contain +# $version, which will be replaced by the version of the file (if it could +# be obtained via FILE_VERSION_FILTER) + +WARN_FORMAT = "$file:$line: $text" + +# The WARN_LOGFILE tag can be used to specify a file to which warning +# and error messages should be written. If left blank the output is written +# to stderr. + +WARN_LOGFILE = + +#--------------------------------------------------------------------------- +# configuration options related to the input files +#--------------------------------------------------------------------------- + +# The INPUT tag can be used to specify the files and/or directories that contain +# documented source files. You may enter file names like "myfile.cpp" or +# directories like "/usr/src/myproject". Separate the files or directories +# with spaces. + +INPUT = ../README.md \ + ../contrib/ \ + ../external/ \ + ../include/ \ + ../tests/ \ + ../src/ + +# This tag can be used to specify the character encoding of the source files +# that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is +# also the default input encoding. Doxygen uses libiconv (or the iconv built +# into libc) for the transcoding. See http://www.gnu.org/software/libiconv for +# the list of possible encodings. + +INPUT_ENCODING = UTF-8 + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank the following patterns are tested: +# *.c *.cc *.cxx *.cpp *.c++ *.d *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh +# *.hxx *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.dox *.py +# *.f90 *.f *.for *.vhd *.vhdl + +FILE_PATTERNS = *.cpp \ + *.h \ + *.hpp + +# The RECURSIVE tag can be used to turn specify whether or not subdirectories +# should be searched for input files as well. Possible values are YES and NO. +# If left blank NO is used. + +RECURSIVE = YES + +# The EXCLUDE tag can be used to specify files and/or directories that should be +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. +# Note that relative paths are relative to the directory from which doxygen is +# run. + +EXCLUDE = + +# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or +# directories that are symbolic links (a Unix file system feature) are excluded +# from the input. + +EXCLUDE_SYMLINKS = NO + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. Note that the wildcards are matched +# against the file with absolute path, so to exclude all test directories +# for example use the pattern */test/* + +EXCLUDE_PATTERNS = *.pb.* \ + tinythread.* \ + fast_mutex.* \ + anyoption.* \ + stacktrace.* \ + */simpleini/* \ + ExportWrapper.* \ + WinsockWrapper.* \ + otapicli.* \ + test*.* \ + irrXML.* \ + */chaiscript/* \ + */zmq/* + +# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names +# (namespaces, classes, functions, etc.) that should be excluded from the +# output. The symbol name can be a fully qualified name, a word, or if the +# wildcard * is used, a substring. Examples: ANamespace, AClass, +# AClass::ANamespace, ANamespace::*Test + +EXCLUDE_SYMBOLS = tinythread + +# The EXAMPLE_PATH tag can be used to specify one or more files or +# directories that contain example code fragments that are included (see +# the \include command). + +EXAMPLE_PATH = + +# If the value of the EXAMPLE_PATH tag contains directories, you can use the +# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank all files are included. + +EXAMPLE_PATTERNS = + +# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be +# searched for input files to be used with the \include or \dontinclude +# commands irrespective of the value of the RECURSIVE tag. +# Possible values are YES and NO. If left blank NO is used. + +EXAMPLE_RECURSIVE = NO + +# The IMAGE_PATH tag can be used to specify one or more files or +# directories that contain image that are included in the documentation (see +# the \image command). + +IMAGE_PATH = + +# The INPUT_FILTER tag can be used to specify a program that doxygen should +# invoke to filter for each input file. Doxygen will invoke the filter program +# by executing (via popen()) the command <filter> <input-file>, where <filter> +# is the value of the INPUT_FILTER tag, and <input-file> is the name of an +# input file. Doxygen will then use the output that the filter program writes +# to standard output. If FILTER_PATTERNS is specified, this tag will be +# ignored. + +INPUT_FILTER = + +# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern +# basis. Doxygen will compare the file name with each pattern and apply the +# filter if there is a match. The filters are a list of the form: +# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further +# info on how filters are used. If FILTER_PATTERNS is empty or if +# non of the patterns match the file name, INPUT_FILTER is applied. + +FILTER_PATTERNS = + +# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using +# INPUT_FILTER) will be used to filter the input files when producing source +# files to browse (i.e. when SOURCE_BROWSER is set to YES). + +FILTER_SOURCE_FILES = NO + +# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file +# pattern. A pattern will override the setting for FILTER_PATTERN (if any) +# and it is also possible to disable source filtering for a specific pattern +# using *.ext= (so without naming a filter). This option only has effect when +# FILTER_SOURCE_FILES is enabled. + +FILTER_SOURCE_PATTERNS = + +#--------------------------------------------------------------------------- +# configuration options related to source browsing +#--------------------------------------------------------------------------- + +# If the SOURCE_BROWSER tag is set to YES then a list of source files will +# be generated. Documented entities will be cross-referenced with these sources. +# Note: To get rid of all source code in the generated output, make sure also +# VERBATIM_HEADERS is set to NO. + +SOURCE_BROWSER = YES + +# Setting the INLINE_SOURCES tag to YES will include the body +# of functions and classes directly in the documentation. + +INLINE_SOURCES = YES + +# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct +# doxygen to hide any special comment blocks from generated source code +# fragments. Normal C, C++ and Fortran comments will always remain visible. + +STRIP_CODE_COMMENTS = YES + +# If the REFERENCED_BY_RELATION tag is set to YES +# then for each documented function all documented +# functions referencing it will be listed. + +REFERENCED_BY_RELATION = NO + +# If the REFERENCES_RELATION tag is set to YES +# then for each documented function all documented entities +# called/used by that function will be listed. + +REFERENCES_RELATION = NO + +# If the REFERENCES_LINK_SOURCE tag is set to YES (the default) +# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from +# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will +# link to the source code. Otherwise they will link to the documentation. + +REFERENCES_LINK_SOURCE = YES + +# If the USE_HTAGS tag is set to YES then the references to source code +# will point to the HTML generated by the htags(1) tool instead of doxygen +# built-in source browser. The htags tool is part of GNU's global source +# tagging system (see http://www.gnu.org/software/global/global.html). You +# will need version 4.8.6 or higher. + +USE_HTAGS = NO + +# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen +# will generate a verbatim copy of the header file for each class for +# which an include is specified. Set to NO to disable this. + +VERBATIM_HEADERS = YES + +#--------------------------------------------------------------------------- +# configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- + +# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index +# of all compounds will be generated. Enable this if the project +# contains a lot of classes, structs, unions or interfaces. + +ALPHABETICAL_INDEX = YES + +# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then +# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns +# in which this list will be split (can be a number in the range [1..20]) + +COLS_IN_ALPHA_INDEX = 5 + +# In case all classes in a project start with a common prefix, all +# classes will be put under the same header in the alphabetical index. +# The IGNORE_PREFIX tag can be used to specify one or more prefixes that +# should be ignored while generating the index headers. + +IGNORE_PREFIX = + +#--------------------------------------------------------------------------- +# configuration options related to the HTML output +#--------------------------------------------------------------------------- + +# If the GENERATE_HTML tag is set to YES (the default) Doxygen will +# generate HTML output. + +GENERATE_HTML = YES + +# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `html' will be used as the default path. + +HTML_OUTPUT = html + +# The HTML_FILE_EXTENSION tag can be used to specify the file extension for +# each generated HTML page (for example: .htm,.php,.asp). If it is left blank +# doxygen will generate files with .html extension. + +HTML_FILE_EXTENSION = .html + +# The HTML_HEADER tag can be used to specify a personal HTML header for +# each generated HTML page. If it is left blank doxygen will generate a +# standard header. Note that when using a custom header you are responsible +# for the proper inclusion of any scripts and style sheets that doxygen +# needs, which is dependent on the configuration options used. +# It is advised to generate a default header using "doxygen -w html +# header.html footer.html stylesheet.css YourConfigFile" and then modify +# that header. Note that the header is subject to change so you typically +# have to redo this when upgrading to a newer version of doxygen or when +# changing the value of configuration settings such as GENERATE_TREEVIEW! + +HTML_HEADER = + +# The HTML_FOOTER tag can be used to specify a personal HTML footer for +# each generated HTML page. If it is left blank doxygen will generate a +# standard footer. + +HTML_FOOTER = + +# The HTML_STYLESHEET tag can be used to specify a user-defined cascading +# style sheet that is used by each HTML page. It can be used to +# fine-tune the look of the HTML output. If the tag is left blank doxygen +# will generate a default style sheet. Note that doxygen will try to copy +# the style sheet file to the HTML output directory, so don't put your own +# style sheet in the HTML output directory as well, or it will be erased! + +HTML_STYLESHEET = + +# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or +# other source files which should be copied to the HTML output directory. Note +# that these files will be copied to the base HTML output directory. Use the +# $relpath$ marker in the HTML_HEADER and/or HTML_FOOTER files to load these +# files. In the HTML_STYLESHEET file, use the file name only. Also note that +# the files will be copied as-is; there are no commands or markers available. + +HTML_EXTRA_FILES = + +# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. +# Doxygen will adjust the colors in the style sheet and background images +# according to this color. Hue is specified as an angle on a colorwheel, +# see http://en.wikipedia.org/wiki/Hue for more information. +# For instance the value 0 represents red, 60 is yellow, 120 is green, +# 180 is cyan, 240 is blue, 300 purple, and 360 is red again. +# The allowed range is 0 to 359. + +HTML_COLORSTYLE_HUE = 220 + +# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of +# the colors in the HTML output. For a value of 0 the output will use +# grayscales only. A value of 255 will produce the most vivid colors. + +HTML_COLORSTYLE_SAT = 100 + +# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to +# the luminance component of the colors in the HTML output. Values below +# 100 gradually make the output lighter, whereas values above 100 make +# the output darker. The value divided by 100 is the actual gamma applied, +# so 80 represents a gamma of 0.8, The value 220 represents a gamma of 2.2, +# and 100 does not change the gamma. + +HTML_COLORSTYLE_GAMMA = 80 + +# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML +# page will contain the date and time when the page was generated. Setting +# this to NO can help when comparing the output of multiple runs. + +HTML_TIMESTAMP = YES + +# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML +# documentation will contain sections that can be hidden and shown after the +# page has loaded. + +HTML_DYNAMIC_SECTIONS = NO + +# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of +# entries shown in the various tree structured indices initially; the user +# can expand and collapse entries dynamically later on. Doxygen will expand +# the tree to such a level that at most the specified number of entries are +# visible (unless a fully collapsed tree already exceeds this amount). +# So setting the number of entries 1 will produce a full collapsed tree by +# default. 0 is a special value representing an infinite number of entries +# and will result in a full expanded tree by default. + +HTML_INDEX_NUM_ENTRIES = 100 + +# If the GENERATE_DOCSET tag is set to YES, additional index files +# will be generated that can be used as input for Apple's Xcode 3 +# integrated development environment, introduced with OSX 10.5 (Leopard). +# To create a documentation set, doxygen will generate a Makefile in the +# HTML output directory. Running make will produce the docset in that +# directory and running "make install" will install the docset in +# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find +# it at startup. +# See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html +# for more information. + +GENERATE_DOCSET = NO + +# When GENERATE_DOCSET tag is set to YES, this tag determines the name of the +# feed. A documentation feed provides an umbrella under which multiple +# documentation sets from a single provider (such as a company or product suite) +# can be grouped. + +DOCSET_FEEDNAME = "Doxygen generated docs" + +# When GENERATE_DOCSET tag is set to YES, this tag specifies a string that +# should uniquely identify the documentation set bundle. This should be a +# reverse domain-name style string, e.g. com.mycompany.MyDocSet. Doxygen +# will append .docset to the name. + +DOCSET_BUNDLE_ID = org.doxygen.Project + +# When GENERATE_PUBLISHER_ID tag specifies a string that should uniquely identify +# the documentation publisher. This should be a reverse domain-name style +# string, e.g. com.mycompany.MyDocSet.documentation. + +DOCSET_PUBLISHER_ID = org.doxygen.Publisher + +# The GENERATE_PUBLISHER_NAME tag identifies the documentation publisher. + +DOCSET_PUBLISHER_NAME = Publisher + +# If the GENERATE_HTMLHELP tag is set to YES, additional index files +# will be generated that can be used as input for tools like the +# Microsoft HTML help workshop to generate a compiled HTML help file (.chm) +# of the generated HTML documentation. + +GENERATE_HTMLHELP = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can +# be used to specify the file name of the resulting .chm file. You +# can add a path in front of the file if the result should not be +# written to the html output directory. + +CHM_FILE = + +# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can +# be used to specify the location (absolute path including file name) of +# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run +# the HTML help compiler on the generated index.hhp. + +HHC_LOCATION = + +# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag +# controls if a separate .chi index file is generated (YES) or that +# it should be included in the master .chm file (NO). + +GENERATE_CHI = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the CHM_INDEX_ENCODING +# is used to encode HtmlHelp index (hhk), content (hhc) and project file +# content. + +CHM_INDEX_ENCODING = + +# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag +# controls whether a binary table of contents is generated (YES) or a +# normal table of contents (NO) in the .chm file. + +BINARY_TOC = NO + +# The TOC_EXPAND flag can be set to YES to add extra items for group members +# to the contents of the HTML help documentation and to the tree view. + +TOC_EXPAND = NO + +# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and +# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated +# that can be used as input for Qt's qhelpgenerator to generate a +# Qt Compressed Help (.qch) of the generated HTML documentation. + +GENERATE_QHP = NO + +# If the QHG_LOCATION tag is specified, the QCH_FILE tag can +# be used to specify the file name of the resulting .qch file. +# The path specified is relative to the HTML output folder. + +QCH_FILE = + +# The QHP_NAMESPACE tag specifies the namespace to use when generating +# Qt Help Project output. For more information please see +# http://doc.trolltech.com/qthelpproject.html#namespace + +QHP_NAMESPACE = org.doxygen.Project + +# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating +# Qt Help Project output. For more information please see +# http://doc.trolltech.com/qthelpproject.html#virtual-folders + +QHP_VIRTUAL_FOLDER = doc + +# If QHP_CUST_FILTER_NAME is set, it specifies the name of a custom filter to +# add. For more information please see +# http://doc.trolltech.com/qthelpproject.html#custom-filters + +QHP_CUST_FILTER_NAME = + +# The QHP_CUST_FILT_ATTRS tag specifies the list of the attributes of the +# custom filter to add. For more information please see +# <a href="http://doc.trolltech.com/qthelpproject.html#custom-filters"> +# Qt Help Project / Custom Filters</a>. + +QHP_CUST_FILTER_ATTRS = + +# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this +# project's +# filter section matches. +# <a href="http://doc.trolltech.com/qthelpproject.html#filter-attributes"> +# Qt Help Project / Filter Attributes</a>. + +QHP_SECT_FILTER_ATTRS = + +# If the GENERATE_QHP tag is set to YES, the QHG_LOCATION tag can +# be used to specify the location of Qt's qhelpgenerator. +# If non-empty doxygen will try to run qhelpgenerator on the generated +# .qhp file. + +QHG_LOCATION = + +# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files +# will be generated, which together with the HTML files, form an Eclipse help +# plugin. To install this plugin and make it available under the help contents +# menu in Eclipse, the contents of the directory containing the HTML and XML +# files needs to be copied into the plugins directory of eclipse. The name of +# the directory within the plugins directory should be the same as +# the ECLIPSE_DOC_ID value. After copying Eclipse needs to be restarted before +# the help appears. + +GENERATE_ECLIPSEHELP = NO + +# A unique identifier for the eclipse help plugin. When installing the plugin +# the directory name containing the HTML and XML files should also have +# this name. + +ECLIPSE_DOC_ID = org.doxygen.Project + +# The DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) +# at top of each HTML page. The value NO (the default) enables the index and +# the value YES disables it. Since the tabs have the same information as the +# navigation tree you can set this option to NO if you already set +# GENERATE_TREEVIEW to YES. + +DISABLE_INDEX = NO + +# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index +# structure should be generated to display hierarchical information. +# If the tag value is set to YES, a side panel will be generated +# containing a tree-like index structure (just like the one that +# is generated for HTML Help). For this to work a browser that supports +# JavaScript, DHTML, CSS and frames is required (i.e. any modern browser). +# Windows users are probably better off using the HTML help feature. +# Since the tree basically has the same information as the tab index you +# could consider to set DISABLE_INDEX to NO when enabling this option. + +GENERATE_TREEVIEW = YES + +# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values +# (range [0,1..20]) that doxygen will group on one line in the generated HTML +# documentation. Note that a value of 0 will completely suppress the enum +# values from appearing in the overview section. + +ENUM_VALUES_PER_LINE = 4 + +# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be +# used to set the initial width (in pixels) of the frame in which the tree +# is shown. + +TREEVIEW_WIDTH = 250 + +# When the EXT_LINKS_IN_WINDOW option is set to YES doxygen will open +# links to external symbols imported via tag files in a separate window. + +EXT_LINKS_IN_WINDOW = YES + +# Use this tag to change the font size of Latex formulas included +# as images in the HTML documentation. The default is 10. Note that +# when you change the font size after a successful doxygen run you need +# to manually remove any form_*.png images from the HTML output directory +# to force them to be regenerated. + +FORMULA_FONTSIZE = 10 + +# Use the FORMULA_TRANPARENT tag to determine whether or not the images +# generated for formulas are transparent PNGs. Transparent PNGs are +# not supported properly for IE 6.0, but are supported on all modern browsers. +# Note that when changing this option you need to delete any form_*.png files +# in the HTML output before the changes have effect. + +FORMULA_TRANSPARENT = YES + +# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax +# (see http://www.mathjax.org) which uses client side Javascript for the +# rendering instead of using prerendered bitmaps. Use this if you do not +# have LaTeX installed or if you want to formulas look prettier in the HTML +# output. When enabled you may also need to install MathJax separately and +# configure the path to it using the MATHJAX_RELPATH option. + +USE_MATHJAX = NO + +# When MathJax is enabled you need to specify the location relative to the +# HTML output directory using the MATHJAX_RELPATH option. The destination +# directory should contain the MathJax.js script. For instance, if the mathjax +# directory is located at the same level as the HTML output directory, then +# MATHJAX_RELPATH should be ../mathjax. The default value points to +# the MathJax Content Delivery Network so you can quickly see the result without +# installing MathJax. However, it is strongly recommended to install a local +# copy of MathJax from http://www.mathjax.org before deployment. + +MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest + +# The MATHJAX_EXTENSIONS tag can be used to specify one or MathJax extension +# names that should be enabled during MathJax rendering. + +MATHJAX_EXTENSIONS = + +# When the SEARCHENGINE tag is enabled doxygen will generate a search box +# for the HTML output. The underlying search engine uses javascript +# and DHTML and should work on any modern browser. Note that when using +# HTML help (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets +# (GENERATE_DOCSET) there is already a search function so this one should +# typically be disabled. For large projects the javascript based search engine +# can be slow, then enabling SERVER_BASED_SEARCH may provide a better solution. + +SEARCHENGINE = YES + +# When the SERVER_BASED_SEARCH tag is enabled the search engine will be +# implemented using a PHP enabled web server instead of at the web client +# using Javascript. Doxygen will generate the search PHP script and index +# file to put on the web server. The advantage of the server +# based approach is that it scales better to large projects and allows +# full text search. The disadvantages are that it is more difficult to setup +# and does not have live searching capabilities. + +SERVER_BASED_SEARCH = NO + +#--------------------------------------------------------------------------- +# configuration options related to the LaTeX output +#--------------------------------------------------------------------------- + +# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will +# generate Latex output. + +GENERATE_LATEX = NO + +# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `latex' will be used as the default path. + +LATEX_OUTPUT = latex + +# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be +# invoked. If left blank `latex' will be used as the default command name. +# Note that when enabling USE_PDFLATEX this option is only used for +# generating bitmaps for formulas in the HTML output, but not in the +# Makefile that is written to the output directory. + +LATEX_CMD_NAME = latex + +# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to +# generate index for LaTeX. If left blank `makeindex' will be used as the +# default command name. + +MAKEINDEX_CMD_NAME = makeindex + +# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact +# LaTeX documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_LATEX = NO + +# The PAPER_TYPE tag can be used to set the paper type that is used +# by the printer. Possible values are: a4, letter, legal and +# executive. If left blank a4wide will be used. + +PAPER_TYPE = a4 + +# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX +# packages that should be included in the LaTeX output. + +EXTRA_PACKAGES = + +# The LATEX_HEADER tag can be used to specify a personal LaTeX header for +# the generated latex document. The header should contain everything until +# the first chapter. If it is left blank doxygen will generate a +# standard header. Notice: only use this tag if you know what you are doing! + +LATEX_HEADER = + +# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for +# the generated latex document. The footer should contain everything after +# the last chapter. If it is left blank doxygen will generate a +# standard footer. Notice: only use this tag if you know what you are doing! + +LATEX_FOOTER = + +# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated +# is prepared for conversion to pdf (using ps2pdf). The pdf file will +# contain links (just like the HTML output) instead of page references +# This makes the output suitable for online browsing using a pdf viewer. + +PDF_HYPERLINKS = YES + +# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of +# plain latex in the generated Makefile. Set this option to YES to get a +# higher quality PDF documentation. + +USE_PDFLATEX = YES + +# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. +# command to the generated LaTeX files. This will instruct LaTeX to keep +# running if errors occur, instead of asking the user for help. +# This option is also used when generating formulas in HTML. + +LATEX_BATCHMODE = NO + +# If LATEX_HIDE_INDICES is set to YES then doxygen will not +# include the index chapters (such as File Index, Compound Index, etc.) +# in the output. + +LATEX_HIDE_INDICES = NO + +# If LATEX_SOURCE_CODE is set to YES then doxygen will include +# source code with syntax highlighting in the LaTeX output. +# Note that which sources are shown also depends on other settings +# such as SOURCE_BROWSER. + +LATEX_SOURCE_CODE = NO + +# The LATEX_BIB_STYLE tag can be used to specify the style to use for the +# bibliography, e.g. plainnat, or ieeetr. The default style is "plain". See +# http://en.wikipedia.org/wiki/BibTeX for more info. + +LATEX_BIB_STYLE = plain + +#--------------------------------------------------------------------------- +# configuration options related to the RTF output +#--------------------------------------------------------------------------- + +# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output +# The RTF output is optimized for Word 97 and may not look very pretty with +# other RTF readers or editors. + +GENERATE_RTF = NO + +# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `rtf' will be used as the default path. + +RTF_OUTPUT = rtf + +# If the COMPACT_RTF tag is set to YES Doxygen generates more compact +# RTF documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_RTF = NO + +# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated +# will contain hyperlink fields. The RTF file will +# contain links (just like the HTML output) instead of page references. +# This makes the output suitable for online browsing using WORD or other +# programs which support those fields. +# Note: wordpad (write) and others do not support links. + +RTF_HYPERLINKS = NO + +# Load style sheet definitions from file. Syntax is similar to doxygen's +# config file, i.e. a series of assignments. You only have to provide +# replacements, missing definitions are set to their default value. + +RTF_STYLESHEET_FILE = + +# Set optional variables used in the generation of an rtf document. +# Syntax is similar to doxygen's config file. + +RTF_EXTENSIONS_FILE = + +#--------------------------------------------------------------------------- +# configuration options related to the man page output +#--------------------------------------------------------------------------- + +# If the GENERATE_MAN tag is set to YES (the default) Doxygen will +# generate man pages + +GENERATE_MAN = NO + +# The MAN_OUTPUT tag is used to specify where the man pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `man' will be used as the default path. + +MAN_OUTPUT = man + +# The MAN_EXTENSION tag determines the extension that is added to +# the generated man pages (default is the subroutine's section .3) + +MAN_EXTENSION = .3 + +# If the MAN_LINKS tag is set to YES and Doxygen generates man output, +# then it will generate one additional man file for each entity +# documented in the real man page(s). These additional files +# only source the real man page, but without them the man command +# would be unable to find the correct page. The default is NO. + +MAN_LINKS = NO + +#--------------------------------------------------------------------------- +# configuration options related to the XML output +#--------------------------------------------------------------------------- + +# If the GENERATE_XML tag is set to YES Doxygen will +# generate an XML file that captures the structure of +# the code including all documentation. + +GENERATE_XML = NO + +# The XML_OUTPUT tag is used to specify where the XML pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `xml' will be used as the default path. + +XML_OUTPUT = xml + +# The XML_SCHEMA tag can be used to specify an XML schema, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_SCHEMA = + +# The XML_DTD tag can be used to specify an XML DTD, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_DTD = + +# If the XML_PROGRAMLISTING tag is set to YES Doxygen will +# dump the program listings (including syntax highlighting +# and cross-referencing information) to the XML output. Note that +# enabling this will significantly increase the size of the XML output. + +XML_PROGRAMLISTING = YES + +#--------------------------------------------------------------------------- +# configuration options for the AutoGen Definitions output +#--------------------------------------------------------------------------- + +# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will +# generate an AutoGen Definitions (see autogen.sf.net) file +# that captures the structure of the code including all +# documentation. Note that this feature is still experimental +# and incomplete at the moment. + +GENERATE_AUTOGEN_DEF = NO + +#--------------------------------------------------------------------------- +# configuration options related to the Perl module output +#--------------------------------------------------------------------------- + +# If the GENERATE_PERLMOD tag is set to YES Doxygen will +# generate a Perl module file that captures the structure of +# the code including all documentation. Note that this +# feature is still experimental and incomplete at the +# moment. + +GENERATE_PERLMOD = NO + +# If the PERLMOD_LATEX tag is set to YES Doxygen will generate +# the necessary Makefile rules, Perl scripts and LaTeX code to be able +# to generate PDF and DVI output from the Perl module output. + +PERLMOD_LATEX = NO + +# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be +# nicely formatted so it can be parsed by a human reader. This is useful +# if you want to understand what is going on. On the other hand, if this +# tag is set to NO the size of the Perl module output will be much smaller +# and Perl will parse it just the same. + +PERLMOD_PRETTY = YES + +# The names of the make variables in the generated doxyrules.make file +# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. +# This is useful so different doxyrules.make files included by the same +# Makefile don't overwrite each other's variables. + +PERLMOD_MAKEVAR_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the preprocessor +#--------------------------------------------------------------------------- + +# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will +# evaluate all C-preprocessor directives found in the sources and include +# files. + +ENABLE_PREPROCESSING = YES + +# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro +# names in the source code. If set to NO (the default) only conditional +# compilation will be performed. Macro expansion can be done in a controlled +# way by setting EXPAND_ONLY_PREDEF to YES. + +MACRO_EXPANSION = NO + +# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES +# then the macro expansion is limited to the macros specified with the +# PREDEFINED and EXPAND_AS_DEFINED tags. + +EXPAND_ONLY_PREDEF = NO + +# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files +# pointed to by INCLUDE_PATH will be searched when a #include is found. + +SEARCH_INCLUDES = YES + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by +# the preprocessor. + +INCLUDE_PATH = + +# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard +# patterns (like *.h and *.hpp) to filter out the header-files in the +# directories. If left blank, the patterns specified with FILE_PATTERNS will +# be used. + +INCLUDE_FILE_PATTERNS = + +# The PREDEFINED tag can be used to specify one or more macro names that +# are defined before the preprocessor is started (similar to the -D option of +# gcc). The argument of the tag is a list of macros of the form: name +# or name=definition (no spaces). If the definition and the = are +# omitted =1 is assumed. To prevent a macro definition from being +# undefined via #undef or recursively expanded use the := operator +# instead of the = operator. + +PREDEFINED = OT_CRYPTO_USING_OPENSSL \ + OT_CASH_USING_LUCRE + +# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then +# this tag can be used to specify a list of macro names that should be expanded. +# The macro definition that is found in the sources will be used. +# Use the PREDEFINED tag if you want to use a different macro definition that +# overrules the definition found in the source code. + +EXPAND_AS_DEFINED = + +# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then +# doxygen's preprocessor will remove all references to function-like macros +# that are alone on a line, have an all uppercase name, and do not end with a +# semicolon, because these will confuse the parser if not removed. + +SKIP_FUNCTION_MACROS = YES + +#--------------------------------------------------------------------------- +# Configuration::additions related to external references +#--------------------------------------------------------------------------- + +# The TAGFILES option can be used to specify one or more tagfiles. For each +# tag file the location of the external documentation should be added. The +# format of a tag file without this location is as follows: +# TAGFILES = file1 file2 ... +# Adding location for the tag files is done as follows: +# TAGFILES = file1=loc1 "file2 = loc2" ... +# where "loc1" and "loc2" can be relative or absolute paths +# or URLs. Note that each tag file must have a unique name (where the name does +# NOT include the path). If a tag file is not located in the directory in which +# doxygen is run, you must also specify the path to the tagfile here. + +TAGFILES = + +# When a file name is specified after GENERATE_TAGFILE, doxygen will create +# a tag file that is based on the input files it reads. + +GENERATE_TAGFILE = + +# If the ALLEXTERNALS tag is set to YES all external classes will be listed +# in the class index. If set to NO only the inherited external classes +# will be listed. + +ALLEXTERNALS = NO + +# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed +# in the modules index. If set to NO, only the current project's groups will +# be listed. + +EXTERNAL_GROUPS = YES + +# The PERL_PATH should be the absolute path and name of the perl script +# interpreter (i.e. the result of `which perl'). + +PERL_PATH = /usr/bin/perl + +#--------------------------------------------------------------------------- +# Configuration options related to the dot tool +#--------------------------------------------------------------------------- + +# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will +# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base +# or super classes. Setting the tag to NO turns the diagrams off. Note that +# this option also works with HAVE_DOT disabled, but it is recommended to +# install and use dot, since it yields more powerful graphs. + +CLASS_DIAGRAMS = NO + +# You can define message sequence charts within doxygen comments using the \msc +# command. Doxygen will then run the mscgen tool (see +# http://www.mcternan.me.uk/mscgen/) to produce the chart and insert it in the +# documentation. The MSCGEN_PATH tag allows you to specify the directory where +# the mscgen tool resides. If left empty the tool is assumed to be found in the +# default search path. + +MSCGEN_PATH = + +# If set to YES, the inheritance and collaboration graphs will hide +# inheritance and usage relations if the target is undocumented +# or is not a class. + +HIDE_UNDOC_RELATIONS = YES + +# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is +# available from the path. This tool is part of Graphviz, a graph visualization +# toolkit from AT&T and Lucent Bell Labs. The other options in this section +# have no effect if this option is set to NO (the default) + +HAVE_DOT = YES + +# The DOT_NUM_THREADS specifies the number of dot invocations doxygen is +# allowed to run in parallel. When set to 0 (the default) doxygen will +# base this on the number of processors available in the system. You can set it +# explicitly to a value larger than 0 to get control over the balance +# between CPU load and processing speed. + +DOT_NUM_THREADS = 0 + +# By default doxygen will use the Helvetica font for all dot files that +# doxygen generates. When you want a differently looking font you can specify +# the font name using DOT_FONTNAME. You need to make sure dot is able to find +# the font, which can be done by putting it in a standard location or by setting +# the DOTFONTPATH environment variable or by setting DOT_FONTPATH to the +# directory containing the font. + +DOT_FONTNAME = Helvetica + +# The DOT_FONTSIZE tag can be used to set the size of the font of dot graphs. +# The default size is 10pt. + +DOT_FONTSIZE = 10 + +# By default doxygen will tell dot to use the Helvetica font. +# If you specify a different font using DOT_FONTNAME you can use DOT_FONTPATH to +# set the path where dot can find it. + +DOT_FONTPATH = + +# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect inheritance relations. Setting this tag to YES will force the +# CLASS_DIAGRAMS tag to NO. + +CLASS_GRAPH = YES + +# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect implementation dependencies (inheritance, containment, and +# class references variables) of the class with other documented classes. + +COLLABORATION_GRAPH = YES + +# If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for groups, showing the direct groups dependencies + +GROUP_GRAPHS = YES + +# If the UML_LOOK tag is set to YES doxygen will generate inheritance and +# collaboration diagrams in a style similar to the OMG's Unified Modeling +# Language. + +UML_LOOK = NO + +# If the UML_LOOK tag is enabled, the fields and methods are shown inside +# the class node. If there are many fields or methods and many nodes the +# graph may become too big to be useful. The UML_LIMIT_NUM_FIELDS +# threshold limits the number of items for each type to make the size more +# managable. Set this to 0 for no limit. Note that the threshold may be +# exceeded by 50% before the limit is enforced. + +UML_LIMIT_NUM_FIELDS = 10 + +# If set to YES, the inheritance and collaboration graphs will show the +# relations between templates and their instances. + +TEMPLATE_RELATIONS = NO + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT +# tags are set to YES then doxygen will generate a graph for each documented +# file showing the direct and indirect include dependencies of the file with +# other documented files. + +INCLUDE_GRAPH = YES + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and +# HAVE_DOT tags are set to YES then doxygen will generate a graph for each +# documented header file showing the documented files that directly or +# indirectly include this file. + +INCLUDED_BY_GRAPH = YES + +# If the CALL_GRAPH and HAVE_DOT options are set to YES then +# doxygen will generate a call dependency graph for every global function +# or class method. Note that enabling this option will significantly increase +# the time of a run. So in most cases it will be better to enable call graphs +# for selected functions only using the \callgraph command. + +CALL_GRAPH = YES + +# If the CALLER_GRAPH and HAVE_DOT tags are set to YES then +# doxygen will generate a caller dependency graph for every global function +# or class method. Note that enabling this option will significantly increase +# the time of a run. So in most cases it will be better to enable caller +# graphs for selected functions only using the \callergraph command. + +CALLER_GRAPH = YES + +# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen +# will generate a graphical hierarchy of all classes instead of a textual one. + +GRAPHICAL_HIERARCHY = YES + +# If the DIRECTORY_GRAPH and HAVE_DOT tags are set to YES +# then doxygen will show the dependencies a directory has on other directories +# in a graphical way. The dependency relations are determined by the #include +# relations between the files in the directories. + +DIRECTORY_GRAPH = YES + +# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images +# generated by dot. Possible values are svg, png, jpg, or gif. +# If left blank png will be used. If you choose svg you need to set +# HTML_FILE_EXTENSION to xhtml in order to make the SVG files +# visible in IE 9+ (other browsers do not have this requirement). + +DOT_IMAGE_FORMAT = png + +# If DOT_IMAGE_FORMAT is set to svg, then this option can be set to YES to +# enable generation of interactive SVG images that allow zooming and panning. +# Note that this requires a modern browser other than Internet Explorer. +# Tested and working are Firefox, Chrome, Safari, and Opera. For IE 9+ you +# need to set HTML_FILE_EXTENSION to xhtml in order to make the SVG files +# visible. Older versions of IE do not have SVG support. + +INTERACTIVE_SVG = NO + +# The tag DOT_PATH can be used to specify the path where the dot tool can be +# found. If left blank, it is assumed the dot tool can be found in the path. + +DOT_PATH = + +# The DOTFILE_DIRS tag can be used to specify one or more directories that +# contain dot files that are included in the documentation (see the +# \dotfile command). + +DOTFILE_DIRS = + +# The MSCFILE_DIRS tag can be used to specify one or more directories that +# contain msc files that are included in the documentation (see the +# \mscfile command). + +MSCFILE_DIRS = + +# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of +# nodes that will be shown in the graph. If the number of nodes in a graph +# becomes larger than this value, doxygen will truncate the graph, which is +# visualized by representing a node as a red box. Note that doxygen if the +# number of direct children of the root node in a graph is already larger than +# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note +# that the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH. + +DOT_GRAPH_MAX_NODES = 50 + +# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the +# graphs generated by dot. A depth value of 3 means that only nodes reachable +# from the root by following a path via at most 3 edges will be shown. Nodes +# that lay further from the root node will be omitted. Note that setting this +# option to 1 or 2 may greatly reduce the computation time needed for large +# code bases. Also note that the size of a graph can be further restricted by +# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction. + +MAX_DOT_GRAPH_DEPTH = 0 + +# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent +# background. This is disabled by default, because dot on Windows does not +# seem to support this out of the box. Warning: Depending on the platform used, +# enabling this option may lead to badly anti-aliased labels on the edges of +# a graph (i.e. they become hard to read). + +DOT_TRANSPARENT = NO + +# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output +# files in one run (i.e. multiple -o and -T options on the command line). This +# makes dot run faster, but since only newer versions of dot (>1.8.10) +# support this, this feature is disabled by default. + +DOT_MULTI_TARGETS = NO + +# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will +# generate a legend page explaining the meaning of the various boxes and +# arrows in the dot generated graphs. + +GENERATE_LEGEND = YES + +# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will +# remove the intermediate dot files that are used to generate +# the various graphs. + +DOT_CLEANUP = YES diff --git a/cmake/Doxygen.extra.css.in b/cmake/Doxygen.extra.css.in new file mode 100644 index 000000000..022974fac --- /dev/null +++ b/cmake/Doxygen.extra.css.in @@ -0,0 +1,14 @@ +/* increase vertical space */ +#titlearea, #nav-path { + display: none; + height: 0px; +} + + +/* uncomment these lines for some extra vertical space */ + +/* +.tablist li { + line-height: 26px; +} +*/ diff --git a/cmake/FindBerkeleyDB.cmake b/cmake/FindBerkeleyDB.cmake new file mode 100644 index 000000000..916d8f9cc --- /dev/null +++ b/cmake/FindBerkeleyDB.cmake @@ -0,0 +1,25 @@ +# - Try to find Berkeley DB +# Once done this will define +# +# BERKELEY_DB_FOUND - system has Berkeley DB +# BERKELEY_DB_INCLUDE_DIR - the Berkeley DB include directory +# BERKELEY_DB_LIBRARIES - Link these to use Berkeley DB +# BERKELEY_DB_DEFINITIONS - Compiler switches required for using Berkeley DB + +# Copyright (c) 2006, Alexander Dymo, <adymo@kdevelop.org> +# +# Redistribution and use is allowed according to the terms of the BSD license. +# For details see the accompanying COPYING-CMAKE-SCRIPTS file. + +find_path(BERKELEY_DB_INCLUDE_DIR db_cxx.h + /usr/include/db4 + /usr/local/include/db4 +) + +find_library(BERKELEY_DB_LIBRARIES NAMES db_cxx ) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Berkeley "Could not find Berkeley DB >= 4.1" BERKELEY_DB_INCLUDE_DIR BERKELEY_DB_LIBRARIES) +# show the BERKELEY_DB_INCLUDE_DIR and BERKELEY_DB_LIBRARIES variables only in the advanced view +mark_as_advanced(BERKELEY_DB_INCLUDE_DIR BERKELEY_DB_LIBRARIES ) + diff --git a/cmake/FindMiniupnpc.cmake b/cmake/FindMiniupnpc.cmake index 97a0280fe..ab525b3a7 100644 --- a/cmake/FindMiniupnpc.cmake +++ b/cmake/FindMiniupnpc.cmake @@ -182,4 +182,4 @@ else () endif () MARK_AS_ADVANCED(MINIUPNP_INCLUDE_DIR MINIUPNP_LIBRARY) -# --------------------------------- FindMiniupnpc End ---------------------------------
\ No newline at end of file +# --------------------------------- FindMiniupnpc End --------------------------------- diff --git a/cmake/FindUnbound.cmake b/cmake/FindUnbound.cmake index 9728cd2f0..ff6de8d98 100644 --- a/cmake/FindUnbound.cmake +++ b/cmake/FindUnbound.cmake @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # All rights reserved. # # Redistribution and use in source and binary forms, with or without modification, are diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt new file mode 100644 index 000000000..18402a61a --- /dev/null +++ b/contrib/CMakeLists.txt @@ -0,0 +1,3 @@ + +add_subdirectory(otshell_utils) + diff --git a/contrib/epee/LICENSE.txt b/contrib/epee/LICENSE.txt index 4a6b529e5..9835c2f69 100644 --- a/contrib/epee/LICENSE.txt +++ b/contrib/epee/LICENSE.txt @@ -22,4 +22,4 @@ DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/contrib/epee/README.md b/contrib/epee/README.md index a69884f57..8157d3e56 100644 --- a/contrib/epee/README.md +++ b/contrib/epee/README.md @@ -1 +1 @@ -epee - is a small library of helpers, wrappers, tools and and so on, used to make my life easier.
\ No newline at end of file +epee - is a small library of helpers, wrappers, tools and and so on, used to make my life easier. diff --git a/contrib/epee/demo/generate_vc_proj.bat b/contrib/epee/demo/generate_vc_proj.bat index 111405981..7d83ced6f 100644 --- a/contrib/epee/demo/generate_vc_proj.bat +++ b/contrib/epee/demo/generate_vc_proj.bat @@ -4,4 +4,4 @@ cd build cmake "-DBoost_USE_STATIC_LIBS=TRUE" -G "Visual Studio 11 Win64" .. cd .. -pause
\ No newline at end of file +pause diff --git a/contrib/epee/demo/iface/transport_defs.h b/contrib/epee/demo/iface/transport_defs.h index 8463eb903..746694aa3 100644 --- a/contrib/epee/demo/iface/transport_defs.h +++ b/contrib/epee/demo/iface/transport_defs.h @@ -218,4 +218,4 @@ namespace demo s.m_storage_entry_string = epee::serialization::storage_entry(std::string("sdsvsdvs")); return s; } -}
\ No newline at end of file +} diff --git a/contrib/epee/include/console_handler.h b/contrib/epee/include/console_handler.h index ab3cf67c6..fc8cc3841 100644 --- a/contrib/epee/include/console_handler.h +++ b/contrib/epee/include/console_handler.h @@ -26,11 +26,13 @@ #pragma once +#include "misc_log_ex.h" #include <atomic> #include <condition_variable> #include <functional> #include <mutex> #include <thread> +#include <iostream> namespace epee { @@ -267,17 +269,19 @@ namespace epee string_tools::trim(command); LOG_PRINT_L2("Read command: " << command); - if(0 == command.compare("exit") || 0 == command.compare("q")) - { - continue_handle = false; - }else if (command.empty()) + if (command.empty()) { continue; } else if(cmd_handler(command)) { continue; - } else + } + else if(0 == command.compare("exit") || 0 == command.compare("q")) + { + continue_handle = false; + } + else { std::cout << "unknown command: " << command << std::endl; std::cout << usage; @@ -290,7 +294,7 @@ namespace epee private: async_stdin_reader m_stdin_reader; - bool m_running = true; + std::atomic<bool> m_running = {true}; }; @@ -350,17 +354,11 @@ namespace epee return true; }*/ - /************************************************************************/ - /* */ - /************************************************************************/ - class console_handlers_binder - { - typedef boost::function<bool (const std::vector<std::string> &)> console_command_handler; - typedef std::map<std::string, std::pair<console_command_handler, std::string> > command_handlers_map; - std::unique_ptr<boost::thread> m_console_thread; - command_handlers_map m_command_handlers; - async_console_handler m_console_handler; + class command_handler { public: + typedef boost::function<bool (const std::vector<std::string> &)> callback; + typedef std::map<std::string, std::pair<callback, std::string> > lookup; + std::string get_usage() { std::stringstream ss; @@ -376,12 +374,14 @@ namespace epee } return ss.str(); } - void set_handler(const std::string& cmd, const console_command_handler& hndlr, const std::string& usage = "") + + void set_handler(const std::string& cmd, const callback& hndlr, const std::string& usage = "") { - command_handlers_map::mapped_type & vt = m_command_handlers[cmd]; + lookup::mapped_type & vt = m_command_handlers[cmd]; vt.first = hndlr; vt.second = usage; } + bool process_command_vec(const std::vector<std::string>& cmd) { if(!cmd.size()) @@ -399,14 +399,20 @@ namespace epee boost::split(cmd_v,cmd,boost::is_any_of(" "), boost::token_compress_on); return process_command_vec(cmd_v); } + private: + lookup m_command_handlers; + }; - /*template<class t_srv> - bool start_handling(t_srv& srv, const std::string& usage_string = "") - { - start_default_console_handler_no_srv_param(&srv, boost::bind(&console_handlers_binder::process_command_str, this, _1)); - return true; - }*/ - + /************************************************************************/ + /* */ + /************************************************************************/ + class console_handlers_binder : public command_handler + { + typedef command_handler::callback console_command_handler; + typedef command_handler::lookup command_handlers_map; + std::unique_ptr<boost::thread> m_console_thread; + async_console_handler m_console_handler; + public: bool start_handling(const std::string& prompt, const std::string& usage_string = "") { m_console_thread.reset(new boost::thread(boost::bind(&console_handlers_binder::run_handling, this, prompt, usage_string))); @@ -423,40 +429,33 @@ namespace epee { return m_console_handler.run(boost::bind(&console_handlers_binder::process_command_str, this, _1), prompt, usage_string); } - - /*template<class t_srv> - bool run_handling(t_srv& srv, const std::string& usage_string) - { - return run_default_console_handler_no_srv_param(&srv, boost::bind<bool>(&console_handlers_binder::process_command_str, this, _1), usage_string); - }*/ }; - /* work around because of broken boost bind */ - template<class t_server> - class srv_console_handlers_binder: public console_handlers_binder - { - bool process_command_str(t_server* /*psrv*/, const std::string& cmd) - { - return console_handlers_binder::process_command_str(cmd); - } - public: - bool start_handling(t_server* psrv, const std::string& prompt, const std::string& usage_string = "") - { - boost::thread(boost::bind(&srv_console_handlers_binder<t_server>::run_handling, this, psrv, prompt, usage_string)).detach(); - return true; - } - - bool run_handling(t_server* psrv, const std::string& prompt, const std::string& usage_string) - { - return m_console_handler.run(psrv, boost::bind(&srv_console_handlers_binder<t_server>::process_command_str, this, _1, _2), prompt, usage_string); - } - - void stop_handling() - { - m_console_handler.stop(); - } - - private: - async_console_handler m_console_handler; - }; + ///* work around because of broken boost bind */ + //template<class t_server> + //class srv_console_handlers_binder: public command_handler + //{ + // async_console_handler m_console_handler; + //public: + // bool start_handling(t_server* psrv, const std::string& prompt, const std::string& usage_string = "") + // { + // boost::thread(boost::bind(&srv_console_handlers_binder<t_server>::run_handling, this, psrv, prompt, usage_string)).detach(); + // return true; + // } + + // bool run_handling(t_server* psrv, const std::string& prompt, const std::string& usage_string) + // { + // return m_console_handler.run(psrv, boost::bind(&srv_console_handlers_binder<t_server>::process_command_str, this, _1, _2), prompt, usage_string); + // } + + // void stop_handling() + // { + // m_console_handler.stop(); + // } + //private: + // bool process_command_str(t_server* /*psrv*/, const std::string& cmd) + // { + // return console_handlers_binder::process_command_str(cmd); + // } + //}; } diff --git a/contrib/epee/include/copyable_atomic.h b/contrib/epee/include/copyable_atomic.h index 6b5691ab1..410b4b4ff 100644 --- a/contrib/epee/include/copyable_atomic.h +++ b/contrib/epee/include/copyable_atomic.h @@ -51,4 +51,4 @@ namespace epee return std::atomic<uint32_t>::operator++(fake); } }; -}
\ No newline at end of file +} diff --git a/contrib/epee/include/math_helper.h b/contrib/epee/include/math_helper.h index 349d6d822..9b8765e60 100644 --- a/contrib/epee/include/math_helper.h +++ b/contrib/epee/include/math_helper.h @@ -269,4 +269,4 @@ PRAGMA_WARNING_POP time_t m_interval; }; } -}
\ No newline at end of file +} diff --git a/contrib/epee/include/md5_l.inl b/contrib/epee/include/md5_l.inl index c3da1a3b0..2aa881092 100644 --- a/contrib/epee/include/md5_l.inl +++ b/contrib/epee/include/md5_l.inl @@ -560,4 +560,4 @@ namespace md5 MD5Final(digest, &context); /* finish up 2nd pass */ } -}
\ No newline at end of file +} diff --git a/contrib/epee/include/net/abstract_tcp_server2.h b/contrib/epee/include/net/abstract_tcp_server2.h index 6c613c5d5..8043c0876 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.h +++ b/contrib/epee/include/net/abstract_tcp_server2.h @@ -1,3 +1,9 @@ +/** +@file +@author from CrypoNote (see copyright below; Andrey N. Sabelnikov) +@monero rfree +@brief the connection templated-class for one peer connection +*/ // Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net // All rights reserved. // @@ -26,7 +32,7 @@ -#ifndef _ABSTRACT_TCP_SERVER2_H_ +#ifndef _ABSTRACT_TCP_SERVER2_H_ #define _ABSTRACT_TCP_SERVER2_H_ @@ -36,6 +42,8 @@ #include <boost/noncopyable.hpp> #include <boost/shared_ptr.hpp> #include <atomic> +#include <map> +#include <memory> #include <boost/asio.hpp> #include <boost/array.hpp> @@ -46,7 +54,9 @@ #include <boost/thread/thread.hpp> #include "net_utils_base.h" #include "syncobj.h" - +#include "../../../../src/p2p/connection_basic.hpp" +#include "../../../../contrib/otshell_utils/utils.hpp" +#include "../../../../src/p2p/network_throttle-detail.hpp" #define ABSTRACT_SERVER_SEND_QUE_MAX_COUNT 1000 @@ -61,6 +71,7 @@ namespace net_utils protected: virtual ~i_connection_filter(){} }; + /************************************************************************/ /* */ @@ -70,13 +81,19 @@ namespace net_utils class connection : public boost::enable_shared_from_this<connection<t_protocol_handler> >, private boost::noncopyable, - public i_service_endpoint + public i_service_endpoint, + public connection_basic { public: typedef typename t_protocol_handler::connection_context t_connection_context; /// Construct a connection with the given io_service. - explicit connection(boost::asio::io_service& io_service, - typename t_protocol_handler::config_type& config, volatile uint32_t& sock_count, i_connection_filter * &pfilter); + + explicit connection( boost::asio::io_service& io_service, + typename t_protocol_handler::config_type& config, + std::atomic<long> &ref_sock_count, // the ++/-- counter + std::atomic<long> &sock_number, // the only increasing ++ number generator + i_connection_filter * &pfilter + ,t_connection_type connection_type); virtual ~connection(); /// Get the socket associated with the connection. @@ -88,9 +105,16 @@ namespace net_utils void get_context(t_connection_context& context_){context_ = context;} void call_back_starter(); + + void save_dbg_log(); + + + bool speed_limit_is_enabled() const; ///< tells us should we be sleeping here (e.g. do not sleep on RPC connections) + private: //----------------- i_service_endpoint --------------------- - virtual bool do_send(const void* ptr, size_t cb); + virtual bool do_send(const void* ptr, size_t cb); ///< (see do_send from i_service_endpoint) + virtual bool do_send_chunk(const void* ptr, size_t cb); ///< will send (or queue) a part of data virtual bool close(); virtual bool call_run_once_service_io(); virtual bool request_callback(); @@ -107,29 +131,31 @@ namespace net_utils /// Handle completion of a write operation. void handle_write(const boost::system::error_code& e, size_t cb); - /// Strand to ensure the connection's handlers are not called concurrently. - boost::asio::io_service::strand strand_; - - /// Socket for the connection. - boost::asio::ip::tcp::socket socket_; - /// Buffer for incoming data. boost::array<char, 8192> buffer_; + //boost::array<char, 1024> buffer_; t_connection_context context; - volatile uint32_t m_want_close_connection; - std::atomic<bool> m_was_shutdown; - critical_section m_send_que_lock; - std::list<std::string> m_send_que; - volatile uint32_t& m_ref_sockets_count; i_connection_filter* &m_pfilter; - volatile bool m_is_multithreaded; + // TODO what do they mean about wait on destructor?? --rfree : //this should be the last one, because it could be wait on destructor, while other activities possible on other threads t_protocol_handler m_protocol_handler; //typename t_protocol_handler::config_type m_dummy_config; std::list<boost::shared_ptr<connection<t_protocol_handler> > > m_self_refs; // add_ref/release support critical_section m_self_refs_lock; + critical_section m_chunking_lock; // held while we add small chunks of the big do_send() to small do_send_chunk() + + t_connection_type m_connection_type; + + // for calculate speed (last 60 sec) + network_throttle m_throttle_speed_in; + network_throttle m_throttle_speed_out; + std::mutex m_throttle_speed_in_mutex; + std::mutex m_throttle_speed_out_mutex; + + public: + void setRpcStation(); }; @@ -145,9 +171,13 @@ namespace net_utils typedef typename t_protocol_handler::connection_context t_connection_context; /// Construct the server to listen on the specified TCP address and port, and /// serve up files from the given directory. - boosted_tcp_server(); - explicit boosted_tcp_server(boost::asio::io_service& external_io_service); + + boosted_tcp_server(t_connection_type connection_type); + explicit boosted_tcp_server(boost::asio::io_service& external_io_service, t_connection_type connection_type); ~boosted_tcp_server(); + + std::map<std::string, t_connection_type> server_type_map; + void create_server_type_map(); bool init_server(uint32_t port, const std::string address = "0.0.0.0"); bool init_server(const std::string port, const std::string& address = "0.0.0.0"); @@ -254,22 +284,30 @@ namespace net_utils /// Acceptor used to listen for incoming connections. boost::asio::ip::tcp::acceptor acceptor_; - /// The next connection to be accepted. - connection_ptr new_connection_; std::atomic<bool> m_stop_signal_sent; uint32_t m_port; - volatile uint32_t m_sockets_count; + std::atomic<long> m_sock_count; + std::atomic<long> m_sock_number; std::string m_address; - std::string m_thread_name_prefix; + std::string m_thread_name_prefix; //TODO: change to enum server_type, now used size_t m_threads_count; i_connection_filter* m_pfilter; std::vector<boost::shared_ptr<boost::thread> > m_threads; boost::thread::id m_main_thread_id; critical_section m_threads_lock; - volatile uint32_t m_thread_index; - }; -} -} + volatile uint32_t m_thread_index; // TODO change to std::atomic + void detach_threads(); + + t_connection_type m_connection_type; + + /// The next connection to be accepted + connection_ptr new_connection_; + + }; // class <>boosted_tcp_server + + +} // namespace +} // namespace #include "abstract_tcp_server2.inl" diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index db3f9e322..612e2b417 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -1,3 +1,9 @@ +/** +@file +@author from CrypoNote (see copyright below; Andrey N. Sabelnikov) +@monero rfree +@brief the connection templated-class for one peer connection +*/ // Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net // All rights reserved. // @@ -26,7 +32,7 @@ -#include "net_utils_base.h" +//#include "net_utils_base.h" #include <boost/lambda/bind.hpp> #include <boost/foreach.hpp> #include <boost/lambda/lambda.hpp> @@ -34,9 +40,21 @@ #include <boost/chrono.hpp> #include <boost/utility/value_init.hpp> #include <boost/asio/deadline_timer.hpp> +#include <boost/date_time/posix_time/posix_time.hpp> // TODO +#include <boost/thread/thread.hpp> // TODO #include "misc_language.h" #include "pragma_comp_defs.h" +#include <sstream> +#include <iomanip> +#include <algorithm> + +#include "../../../../src/cryptonote_core/cryptonote_core.h" // e.g. for the send_stop_signal() + +#include "../../../../contrib/otshell_utils/utils.hpp" +#include "../../../../src/p2p/data_logger.hpp" +using namespace nOT::nUtils; // TODO + PRAGMA_WARNING_PUSH namespace epee { @@ -48,17 +66,22 @@ namespace net_utils PRAGMA_WARNING_DISABLE_VS(4355) template<class t_protocol_handler> - connection<t_protocol_handler>::connection(boost::asio::io_service& io_service, - typename t_protocol_handler::config_type& config, volatile uint32_t& sock_count, i_connection_filter* &pfilter) - : strand_(io_service), - socket_(io_service), - m_want_close_connection(0), - m_was_shutdown(0), - m_ref_sockets_count(sock_count), - m_pfilter(pfilter), - m_protocol_handler(this, config, context) + connection<t_protocol_handler>::connection( boost::asio::io_service& io_service, + typename t_protocol_handler::config_type& config, + std::atomic<long> &ref_sock_count, // the ++/-- counter + std::atomic<long> &sock_number, // the only increasing ++ number generator + i_connection_filter* &pfilter + ,t_connection_type connection_type + ) + : + connection_basic(io_service, ref_sock_count, sock_number), + m_protocol_handler(this, config, context), + m_pfilter( pfilter ), + m_connection_type( connection_type ), + m_throttle_speed_in("speed_in", "throttle_speed_in"), + m_throttle_speed_out("speed_out", "throttle_speed_out") { - boost::interprocess::ipcdetail::atomic_inc32(&m_ref_sockets_count); + _info_c("net/sleepRPC", "test, connection constructor set m_connection_type="<<m_connection_type); } PRAGMA_WARNING_DISABLE_VS(4355) //--------------------------------------------------------------------------------- @@ -67,12 +90,11 @@ PRAGMA_WARNING_DISABLE_VS(4355) { if(!m_was_shutdown) { - LOG_PRINT_L3("[sock " << socket_.native_handle() << "] Socket destroyed without shutdown."); + _dbg3("[sock " << socket_.native_handle() << "] Socket destroyed without shutdown."); shutdown(); } - LOG_PRINT_L3("[sock " << socket_.native_handle() << "] Socket destroyed"); - boost::interprocess::ipcdetail::atomic_dec32(&m_ref_sockets_count); + _dbg3("[sock " << socket_.native_handle() << "] Socket destroyed"); } //--------------------------------------------------------------------------------- template<class t_protocol_handler> @@ -118,13 +140,13 @@ PRAGMA_WARNING_DISABLE_VS(4355) long ip_ = boost::asio::detail::socket_ops::host_to_network_long(remote_ep.address().to_v4().to_ulong()); context.set_details(boost::uuids::random_generator()(), ip_, remote_ep.port(), is_income); - LOG_PRINT_L3("[sock " << socket_.native_handle() << "] new connection from " << print_connection_context_short(context) << + _dbg3("[sock " << socket_.native_handle() << "] new connection from " << print_connection_context_short(context) << " to " << local_ep.address().to_string() << ':' << local_ep.port() << - ", total sockets objects " << m_ref_sockets_count); + ", total sockets objects " << m_ref_sock_count); if(m_pfilter && !m_pfilter->is_remote_ip_allowed(context.m_remote_ip)) { - LOG_PRINT_L2("[sock " << socket_.native_handle() << "] ip denied " << string_tools::get_ip_string_from_int32(context.m_remote_ip) << ", shutdowning connection"); + _dbg2("[sock " << socket_.native_handle() << "] ip denied " << string_tools::get_ip_string_from_int32(context.m_remote_ip) << ", shutdowning connection"); close(); return false; } @@ -136,7 +158,17 @@ PRAGMA_WARNING_DISABLE_VS(4355) boost::bind(&connection<t_protocol_handler>::handle_read, self, boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred))); - + + //set ToS flag + int tos = get_tos_flag(); + boost::asio::detail::socket_option::integer< IPPROTO_IP, IP_TOS > + optionTos( tos ); + socket_.set_option( optionTos ); + //_dbg1("Set ToS flag to " << tos); + + boost::asio::ip::tcp::no_delay noDelayOption(false); + socket_.set_option(noDelayOption); + return true; CATCH_ENTRY_L0("connection<t_protocol_handler>::start()", false); @@ -146,7 +178,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) bool connection<t_protocol_handler>::request_callback() { TRY_ENTRY(); - LOG_PRINT_L2("[" << print_connection_context_short(context) << "] request_callback"); + _dbg2("[" << print_connection_context_short(context) << "] request_callback"); // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted auto self = safe_shared_from_this(); if(!self) @@ -167,8 +199,9 @@ PRAGMA_WARNING_DISABLE_VS(4355) bool connection<t_protocol_handler>::add_ref() { TRY_ENTRY(); - LOG_PRINT_L4("[sock " << socket_.native_handle() << "] add_ref"); + //_dbg3("[sock " << socket_.native_handle() << "] add_ref, m_peer_number=" << mI->m_peer_number); CRITICAL_REGION_LOCAL(m_self_refs_lock); + //_dbg3("[sock " << socket_.native_handle() << "] add_ref 2, m_peer_number=" << mI->m_peer_number); // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted auto self = safe_shared_from_this(); @@ -201,27 +234,69 @@ PRAGMA_WARNING_DISABLE_VS(4355) void connection<t_protocol_handler>::call_back_starter() { TRY_ENTRY(); - LOG_PRINT_L2("[" << print_connection_context_short(context) << "] fired_callback"); + _dbg2("[" << print_connection_context_short(context) << "] fired_callback"); m_protocol_handler.handle_qued_callback(); CATCH_ENTRY_L0("connection<t_protocol_handler>::call_back_starter()", void()); } //--------------------------------------------------------------------------------- template<class t_protocol_handler> + void connection<t_protocol_handler>::save_dbg_log() + { + _mark_c("net/kind" , + " connection type " << to_string( m_connection_type ) << " " + << socket_.local_endpoint().address().to_string() << ":" << socket_.local_endpoint().port() + << " <--> " << socket_.remote_endpoint().address().to_string() << ":" << socket_.remote_endpoint().port() + ); + } + //--------------------------------------------------------------------------------- + template<class t_protocol_handler> void connection<t_protocol_handler>::handle_read(const boost::system::error_code& e, std::size_t bytes_transferred) { TRY_ENTRY(); - LOG_PRINT_L4("[sock " << socket_.native_handle() << "] Async read calledback."); + //_info("[sock " << socket_.native_handle() << "] Async read calledback."); if (!e) { - LOG_PRINT("[sock " << socket_.native_handle() << "] RECV " << bytes_transferred, LOG_LEVEL_4); + { + CRITICAL_REGION_LOCAL(m_throttle_speed_in_mutex); + m_throttle_speed_in.handle_trafic_exact(bytes_transferred); + context.m_current_speed_down = m_throttle_speed_in.get_current_speed(); + } + + { + CRITICAL_REGION_LOCAL( epee::net_utils::network_throttle_manager::network_throttle_manager::m_lock_get_global_throttle_in ); + epee::net_utils::network_throttle_manager::network_throttle_manager::get_global_throttle_in().handle_trafic_exact(bytes_transferred * 1024); + } + + double delay=0; // will be calculated - how much we should sleep to obey speed limit etc + + + if (speed_limit_is_enabled()) { + do // keep sleeping if we should sleep + { + { //_scope_dbg1("CRITICAL_REGION_LOCAL"); + CRITICAL_REGION_LOCAL( epee::net_utils::network_throttle_manager::m_lock_get_global_throttle_in ); + delay = epee::net_utils::network_throttle_manager::get_global_throttle_in().get_sleep_time_after_tick( bytes_transferred ); // decission from global throttle + } + + delay *= 0.5; + if (delay > 0) { + long int ms = (long int)(delay * 100); + epee::net_utils::data_logger::get_instance().add_data("sleep_down", ms); + std::this_thread::sleep_for(std::chrono::milliseconds(ms)); + } + } while(delay > 0); + } // any form of sleeping + + //_info("[sock " << socket_.native_handle() << "] RECV " << bytes_transferred); + logger_handle_net_read(bytes_transferred); context.m_last_recv = time(NULL); context.m_recv_cnt += bytes_transferred; bool recv_res = m_protocol_handler.handle_recv(buffer_.data(), bytes_transferred); if(!recv_res) { - LOG_PRINT("[sock " << socket_.native_handle() << "] protocol_want_close", LOG_LEVEL_4); + //_info("[sock " << socket_.native_handle() << "] protocol_want_close"); //some error in protocol, protocol handler ask to close connection boost::interprocess::ipcdetail::atomic_write32(&m_want_close_connection, 1); @@ -239,14 +314,14 @@ PRAGMA_WARNING_DISABLE_VS(4355) boost::bind(&connection<t_protocol_handler>::handle_read, connection<t_protocol_handler>::shared_from_this(), boost::asio::placeholders::error, boost::asio::placeholders::bytes_transferred))); - LOG_PRINT_L4("[sock " << socket_.native_handle() << "]Async read requested."); + //_info("[sock " << socket_.native_handle() << "]Async read requested."); } }else { - LOG_PRINT_L3("[sock " << socket_.native_handle() << "] Some not success at read: " << e.message() << ':' << e.value()); + _dbg3("[sock " << socket_.native_handle() << "] Some not success at read: " << e.message() << ':' << e.value()); if(e.value() != 2) { - LOG_PRINT_L3("[sock " << socket_.native_handle() << "] Some problems at read: " << e.message() << ':' << e.value()); + _dbg3("[sock " << socket_.native_handle() << "] Some problems at read: " << e.message() << ':' << e.value()); shutdown(); } } @@ -283,8 +358,91 @@ PRAGMA_WARNING_DISABLE_VS(4355) CATCH_ENTRY_L0("connection<t_protocol_handler>::call_run_once_service_io", false); } //--------------------------------------------------------------------------------- + template<class t_protocol_handler> + bool connection<t_protocol_handler>::do_send(const void* ptr, size_t cb) { + TRY_ENTRY(); + + // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted + auto self = safe_shared_from_this(); + if (!self) return false; + if (m_was_shutdown) return false; + // TODO avoid copy + + const double factor = 32; // TODO config + typedef long long signed int t_safe; // my t_size to avoid any overunderflow in arithmetic + const t_safe chunksize_good = (t_safe)( 1024 * std::max(1.0,factor) ); + const t_safe chunksize_max = chunksize_good * 2 ; + const bool allow_split = (m_connection_type == e_connection_type_RPC) ? false : true; // do not split RPC data + + ASRT(! (chunksize_max<0) ); // make sure it is unsigned before removin sign with cast: + long long unsigned int chunksize_max_unsigned = static_cast<long long unsigned int>( chunksize_max ) ; + + if (allow_split && (cb > chunksize_max_unsigned)) { + { // LOCK: chunking + epee::critical_region_t<decltype(m_chunking_lock)> send_guard(m_chunking_lock); // *** critical *** + + _dbg3_c("net/out/size", "do_send() will SPLIT into small chunks, from packet="<<cb<<" B for ptr="<<ptr); + t_safe all = cb; // all bytes to send + t_safe pos = 0; // current sending position + // 01234567890 + // ^^^^ (pos=0, len=4) ; pos:=pos+len, pos=4 + // ^^^^ (pos=4, len=4) ; pos:=pos+len, pos=8 + // ^^^ (pos=8, len=4) ; + + // const size_t bufsize = chunksize_good; // TODO safecast + // char* buf = new char[ bufsize ]; + + bool all_ok = true; + while (pos < all) { + t_safe lenall = all-pos; // length from here to end + t_safe len = std::min( chunksize_good , lenall); // take a smaller part + ASRT(len<=chunksize_good); + // pos=8; len=4; all=10; len=3; + + ASRT(! (len<0) ); // check before we cast away sign: + unsigned long long int len_unsigned = static_cast<long long int>( len ); + ASRT(len>0); // (redundand) + ASRT(len_unsigned < std::numeric_limits<size_t>::max()); // yeap we want strong < then max size, to be sure + + void *chunk_start = ((char*)ptr) + pos; + _fact_c("net/out/size","chunk_start="<<chunk_start<<" ptr="<<ptr<<" pos="<<pos); + ASRT(chunk_start >= ptr); // not wrapped around address? + //std::memcpy( (void*)buf, chunk_start, len); + + _dbg3_c("net/out/size", "part of " << lenall << ": pos="<<pos << " len="<<len); + + bool ok = do_send_chunk(chunk_start, len); // <====== *** + + all_ok = all_ok && ok; + if (!all_ok) { + _dbg1_c("net/out/size", "do_send() DONE ***FAILED*** from packet="<<cb<<" B for ptr="<<ptr); + _dbg1("do_send() SEND was aborted in middle of big package - this is mostly harmless " + << " (e.g. peer closed connection) but if it causes trouble tell us at #monero-dev. " << cb); + return false; // partial failure in sending + } + pos = pos+len; ASRT(pos >0); + + // (in catch block, or uniq pointer) delete buf; + } // each chunk + + _dbg3_c("net/out/size", "do_send() DONE SPLIT from packet="<<cb<<" B for ptr="<<ptr); + _dbg3 ( "do_send() DONE SPLIT from packet="<<cb<<" B for ptr="<<ptr); + + _info_c("net/sleepRPC", "do_send() m_connection_type = " << m_connection_type); + + return all_ok; // done - e.g. queued - all the chunks of current do_send call + } // LOCK: chunking + } // a big block (to be chunked) - all chunks + else { // small block + return do_send_chunk(ptr,cb); // just send as 1 big chunk + } + + CATCH_ENTRY_L0("connection<t_protocol_handler>::do_send", false); + } // do_send() + + //--------------------------------------------------------------------------------- template<class t_protocol_handler> - bool connection<t_protocol_handler>::do_send(const void* ptr, size_t cb) + bool connection<t_protocol_handler>::do_send_chunk(const void* ptr, size_t cb) { TRY_ENTRY(); // Use safe_shared_from_this, because of this is public method and it can be called on the object being deleted @@ -293,50 +451,90 @@ PRAGMA_WARNING_DISABLE_VS(4355) return false; if(m_was_shutdown) return false; + { + CRITICAL_REGION_LOCAL(m_throttle_speed_out_mutex); + m_throttle_speed_out.handle_trafic_exact(cb); + context.m_current_speed_up = m_throttle_speed_out.get_current_speed(); + } - LOG_PRINT("[sock " << socket_.native_handle() << "] SEND " << cb, LOG_LEVEL_4); + //_info("[sock " << socket_.native_handle() << "] SEND " << cb); context.m_last_send = time(NULL); context.m_send_cnt += cb; //some data should be wrote to stream //request complete - epee::critical_region_t<decltype(m_send_que_lock)> send_guard(m_send_que_lock); - if(m_send_que.size() > ABSTRACT_SERVER_SEND_QUE_MAX_COUNT) + if (speed_limit_is_enabled()) { + sleep_before_packet(cb, 1, 1); + } + + epee::critical_region_t<decltype(m_send_que_lock)> send_guard(m_send_que_lock); // *** critical *** + long int retry=0; + const long int retry_limit = 5*4; + while (m_send_que.size() > ABSTRACT_SERVER_SEND_QUE_MAX_COUNT) { - send_guard.unlock(); - LOG_PRINT_L2("send que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection"); - close(); - return false; + retry++; + + /* if ( ::cryptonote::core::get_is_stopping() ) { // TODO re-add fast stop + _fact("ABORT queue wait due to stopping"); + return false; // aborted + }*/ + + long int ms = 250 + (rand()%50); + _info_c("net/sleep", "Sleeping because QUEUE is FULL, in " << __FUNCTION__ << " for " << ms << " ms before packet_size="<<cb); // XXX debug sleep + boost::this_thread::sleep(boost::posix_time::milliseconds( ms ) ); + _dbg1("sleep for queue: " << ms); + + if (retry > retry_limit) { + send_guard.unlock(); + _erro("send que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection"); + // _dbg1_c("net/sleep", "send que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection"); + close(); + return false; + } } m_send_que.resize(m_send_que.size()+1); m_send_que.back().assign((const char*)ptr, cb); if(m_send_que.size() > 1) - { - //active operation should be in progress, nothing to do, just wait last operation callback - }else - { - //no active operation - if(m_send_que.size()!=1) - { - LOG_ERROR("Looks like no active operations, but send que size != 1!!"); - return false; - } - - boost::asio::async_write(socket_, boost::asio::buffer(m_send_que.front().data(), m_send_que.front().size()), - //strand_.wrap( - boost::bind(&connection<t_protocol_handler>::handle_write, self, _1, _2) - //) - ); + { // active operation should be in progress, nothing to do, just wait last operation callback + auto size_now = cb; + _info_c("net/out/size", "do_send() NOW just queues: packet="<<size_now<<" B, is added to queue-size="<<m_send_que.size()); + //do_send_handler_delayed( ptr , size_now ); // (((H))) // empty function LOG_PRINT_L4("[sock " << socket_.native_handle() << "] Async send requested " << m_send_que.front().size()); } + else + { // no active operation + + if(m_send_que.size()!=1) + { + _erro("Looks like no active operations, but send que size != 1!!"); + return false; + } + + auto size_now = m_send_que.front().size(); + _dbg1_c("net/out/size", "do_send() NOW SENSD: packet="<<size_now<<" B"); + if (speed_limit_is_enabled()) + do_send_handler_write( ptr , size_now ); // (((H))) + + ASRT( size_now == m_send_que.front().size() ); + boost::asio::async_write(socket_, boost::asio::buffer(m_send_que.front().data(), size_now ) , + //strand_.wrap( + boost::bind(&connection<t_protocol_handler>::handle_write, self, _1, _2) + //) + ); + //_dbg3("(chunk): " << size_now); + //logger_handle_net_write(size_now); + //_info("[sock " << socket_.native_handle() << "] Async send requested " << m_send_que.front().size()); + } + + //do_send_handler_stop( ptr , cb ); // empty function return true; CATCH_ENTRY_L0("connection<t_protocol_handler>::do_send", false); - } + } // do_send_chunk //--------------------------------------------------------------------------------- template<class t_protocol_handler> bool connection<t_protocol_handler>::shutdown() @@ -353,7 +551,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) bool connection<t_protocol_handler>::close() { TRY_ENTRY(); - LOG_PRINT_L4("[sock " << socket_.native_handle() << "] Que Shutdown called."); + //_info("[sock " << socket_.native_handle() << "] Que Shutdown called."); size_t send_que_size = 0; CRITICAL_REGION_BEGIN(m_send_que_lock); send_que_size = m_send_que.size(); @@ -367,6 +565,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) return true; CATCH_ENTRY_L0("connection<t_protocol_handler>::close", false); } + //--------------------------------------------------------------------------------- template<class t_protocol_handler> void connection<t_protocol_handler>::handle_write(const boost::system::error_code& e, size_t cb) @@ -376,16 +575,21 @@ PRAGMA_WARNING_DISABLE_VS(4355) if (e) { - LOG_PRINT_L1("[sock " << socket_.native_handle() << "] Some problems at write: " << e.message() << ':' << e.value()); + _dbg1("[sock " << socket_.native_handle() << "] Some problems at write: " << e.message() << ':' << e.value()); shutdown(); return; } + logger_handle_net_write(cb); + + if (speed_limit_is_enabled()) { + sleep_before_packet(cb, 1, 1); + } bool do_shutdown = false; CRITICAL_REGION_BEGIN(m_send_que_lock); if(m_send_que.empty()) { - LOG_ERROR("[sock " << socket_.native_handle() << "] m_send_que.size() == 0 at handle_write!"); + _erro("[sock " << socket_.native_handle() << "] m_send_que.size() == 0 at handle_write!"); return; } @@ -399,10 +603,17 @@ PRAGMA_WARNING_DISABLE_VS(4355) }else { //have more data to send - boost::asio::async_write(socket_, boost::asio::buffer(m_send_que.front().data(), m_send_que.front().size()), - //strand_.wrap( - boost::bind(&connection<t_protocol_handler>::handle_write, connection<t_protocol_handler>::shared_from_this(), _1, _2)); - //); + auto size_now = m_send_que.front().size(); + _dbg1_c("net/out/size", "handle_write() NOW SENDS: packet="<<size_now<<" B" <<", from queue size="<<m_send_que.size()); + if (speed_limit_is_enabled()) + do_send_handler_write_from_queue(e, m_send_que.front().size() , m_send_que.size()); // (((H))) + ASRT( size_now == m_send_que.front().size() ); + boost::asio::async_write(socket_, boost::asio::buffer(m_send_que.front().data(), size_now) , + // strand_.wrap( + boost::bind(&connection<t_protocol_handler>::handle_write, connection<t_protocol_handler>::shared_from_this(), _1, _2) + // ) + ); + //_dbg3("(normal)" << size_now); } CRITICAL_REGION_END(); @@ -412,27 +623,52 @@ PRAGMA_WARNING_DISABLE_VS(4355) } CATCH_ENTRY_L0("connection<t_protocol_handler>::handle_write", void()); } + + //--------------------------------------------------------------------------------- + template<class t_protocol_handler> + void connection<t_protocol_handler>::setRpcStation() + { + m_connection_type = e_connection_type_RPC; + _fact_c("net/sleepRPC", "set m_connection_type = RPC "); + _info_c("net/kind", "set m_connection_type = RPC "); + } + + + template<class t_protocol_handler> + bool connection<t_protocol_handler>::speed_limit_is_enabled() const { + return m_connection_type != e_connection_type_RPC ; + } + /************************************************************************/ /* */ /************************************************************************/ + template<class t_protocol_handler> - boosted_tcp_server<t_protocol_handler>::boosted_tcp_server(): + boosted_tcp_server<t_protocol_handler>::boosted_tcp_server( t_connection_type connection_type ) : m_io_service_local_instance(new boost::asio::io_service()), io_service_(*m_io_service_local_instance.get()), acceptor_(io_service_), - new_connection_(new connection<t_protocol_handler>(io_service_, m_config, m_sockets_count, m_pfilter)), - m_stop_signal_sent(false), m_port(0), m_sockets_count(0), m_threads_count(0), m_pfilter(NULL), m_thread_index(0) + m_stop_signal_sent(false), m_port(0), + m_sock_count(0), m_sock_number(0), m_threads_count(0), + m_pfilter(NULL), m_thread_index(0), + m_connection_type( connection_type ), + new_connection_(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter, m_connection_type)) { + create_server_type_map(); m_thread_name_prefix = "NET"; } template<class t_protocol_handler> - boosted_tcp_server<t_protocol_handler>::boosted_tcp_server(boost::asio::io_service& extarnal_io_service): + boosted_tcp_server<t_protocol_handler>::boosted_tcp_server(boost::asio::io_service& extarnal_io_service, t_connection_type connection_type) : io_service_(extarnal_io_service), acceptor_(io_service_), - new_connection_(new connection<t_protocol_handler>(io_service_, m_config, m_sockets_count, m_pfilter)), - m_stop_signal_sent(false), m_port(0), m_sockets_count(0), m_threads_count(0), m_pfilter(NULL), m_thread_index(0) + m_stop_signal_sent(false), m_port(0), + m_sock_count(0), m_sock_number(0), m_threads_count(0), + m_pfilter(NULL), m_thread_index(0), + m_connection_type(connection_type), + new_connection_(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter, connection_type)) { + create_server_type_map(); m_thread_name_prefix = "NET"; } //--------------------------------------------------------------------------------- @@ -444,6 +680,14 @@ PRAGMA_WARNING_DISABLE_VS(4355) } //--------------------------------------------------------------------------------- template<class t_protocol_handler> + void boosted_tcp_server<t_protocol_handler>::create_server_type_map() + { + server_type_map["NET"] = e_connection_type_NET; + server_type_map["RPC"] = e_connection_type_RPC; + server_type_map["P2P"] = e_connection_type_P2P; + } + //--------------------------------------------------------------------------------- + template<class t_protocol_handler> bool boosted_tcp_server<t_protocol_handler>::init_server(uint32_t port, const std::string address) { TRY_ENTRY(); @@ -460,6 +704,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) acceptor_.listen(); boost::asio::ip::tcp::endpoint binded_endpoint = acceptor_.local_endpoint(); m_port = binded_endpoint.port(); + _fact_c("net/RPClog", "start accept"); acceptor_.async_accept(new_connection_->socket(), boost::bind(&boosted_tcp_server<t_protocol_handler>::handle_accept, this, boost::asio::placeholders::error)); @@ -491,6 +736,7 @@ POP_WARNINGS std::string thread_name = std::string("[") + m_thread_name_prefix; thread_name += boost::to_string(local_thr_index) + "]"; log_space::log_singletone::set_thread_log_prefix(thread_name); + // _fact("Thread name: " << m_thread_name_prefix); while(!m_stop_signal_sent) { try @@ -499,14 +745,14 @@ POP_WARNINGS } catch(const std::exception& ex) { - LOG_ERROR("Exception at server worker thread, what=" << ex.what()); + _erro("Exception at server worker thread, what=" << ex.what()); } catch(...) { - LOG_ERROR("Exception at server worker thread, unknown execption"); + _erro("Exception at server worker thread, unknown execption"); } } - LOG_PRINT_L4("Worker thread finished"); + //_info("Worker thread finished"); return true; CATCH_ENTRY_L0("boosted_tcp_server<t_protocol_handler>::worker_thread", false); } @@ -515,6 +761,11 @@ POP_WARNINGS void boosted_tcp_server<t_protocol_handler>::set_threads_prefix(const std::string& prefix_name) { m_thread_name_prefix = prefix_name; + auto it = server_type_map.find(m_thread_name_prefix); + if (it==server_type_map.end()) throw std::runtime_error("Unknown prefix/server type:" + std::string(prefix_name)); + auto connection_type = it->second; // the value of type + _info_c("net/RPClog", "Set server type to: " << connection_type << " from name: " << m_thread_name_prefix); + _info_c("net/RPClog", "prefix_name = " << prefix_name); } //--------------------------------------------------------------------------------- template<class t_protocol_handler> @@ -539,32 +790,38 @@ POP_WARNINGS { boost::shared_ptr<boost::thread> thread(new boost::thread( attrs, boost::bind(&boosted_tcp_server<t_protocol_handler>::worker_thread, this))); + _note("Run server thread name: " << m_thread_name_prefix); m_threads.push_back(thread); } CRITICAL_REGION_END(); // Wait for all threads in the pool to exit. - if(wait) + if (wait) // && ! ::cryptonote::core::get_is_stopping()) // TODO fast_exit { - for (std::size_t i = 0; i < m_threads.size(); ++i) - m_threads[i]->join(); + _fact("JOINING all threads"); + for (std::size_t i = 0; i < m_threads.size(); ++i) { + m_threads[i]->join(); + } + _fact("JOINING all threads - almost"); m_threads.clear(); + _fact("JOINING all threads - DONE"); - }else - { + } + else { + _dbg1("Reiniting OK."); return true; } if(wait && !m_stop_signal_sent) { //some problems with the listening socket ?.. - LOG_PRINT_L0("Net service stopped without stop request, restarting..."); + _dbg1("Net service stopped without stop request, restarting..."); if(!this->init_server(m_port, m_address)) { - LOG_PRINT_L0("Reiniting service failed, exit."); + _dbg1("Reiniting service failed, exit."); return false; }else { - LOG_PRINT_L0("Reiniting OK."); + _dbg1("Reiniting OK."); } } } @@ -597,7 +854,7 @@ POP_WARNINGS { if(m_threads[i]->joinable() && !m_threads[i]->try_join_for(ms)) { - LOG_PRINT_L0("Interrupting thread " << m_threads[i]->native_handle()); + _dbg1("Interrupting thread " << m_threads[i]->native_handle()); m_threads[i]->interrupt(); } } @@ -608,6 +865,10 @@ POP_WARNINGS template<class t_protocol_handler> void boosted_tcp_server<t_protocol_handler>::send_stop_signal() { + if (::cryptonote::core::get_fast_exit() == true) + { + detach_threads(); + } m_stop_signal_sent = true; TRY_ENTRY(); io_service_.stop(); @@ -623,22 +884,28 @@ POP_WARNINGS template<class t_protocol_handler> void boosted_tcp_server<t_protocol_handler>::handle_accept(const boost::system::error_code& e) { + _fact_c("net/RPClog", "handle_accept"); TRY_ENTRY(); if (!e) { - connection_ptr conn(std::move(new_connection_)); - - new_connection_.reset(new connection<t_protocol_handler>(io_service_, m_config, m_sockets_count, m_pfilter)); + if (m_connection_type == e_connection_type_RPC) { + _note_c("net/rpc", "New server for RPC connections"); + _fact_c("net/RPClog", "New server for RPC connections"); + new_connection_->setRpcStation(); // hopefully this is not needed actually + } + connection_ptr conn(std::move(new_connection_)); + new_connection_.reset(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter, m_connection_type)); acceptor_.async_accept(new_connection_->socket(), boost::bind(&boosted_tcp_server<t_protocol_handler>::handle_accept, this, boost::asio::placeholders::error)); bool r = conn->start(true, 1 < m_threads_count); if (!r) - LOG_ERROR("[sock " << conn->socket().native_handle() << "] Failed to start connection, connections_count = " << m_sockets_count); + _erro("[sock " << conn->socket().native_handle() << "] Failed to start connection, connections_count = " << m_sock_count); + conn->save_dbg_log(); }else { - LOG_ERROR("Some problems at accept: " << e.message() << ", connections_count = " << m_sockets_count); + _erro("Some problems at accept: " << e.message() << ", connections_count = " << m_sock_count); } CATCH_ENTRY_L0("boosted_tcp_server<t_protocol_handler>::handle_accept", void()); } @@ -648,7 +915,7 @@ POP_WARNINGS { TRY_ENTRY(); - connection_ptr new_connection_l(new connection<t_protocol_handler>(io_service_, m_config, m_sockets_count, m_pfilter) ); + connection_ptr new_connection_l(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter, m_connection_type) ); boost::asio::ip::tcp::socket& sock_ = new_connection_l->socket(); ////////////////////////////////////////////////////////////////////////// @@ -658,7 +925,7 @@ POP_WARNINGS boost::asio::ip::tcp::resolver::iterator end; if(iterator == end) { - LOG_ERROR("Failed to resolve " << adr); + _erro("Failed to resolve " << adr); return false; } ////////////////////////////////////////////////////////////////////////// @@ -704,7 +971,7 @@ POP_WARNINGS { //timeout sock_.close(); - LOG_PRINT_L3("Failed to connect to " << adr << ":" << port << ", because of timeout (" << conn_timeout << ")"); + _dbg3("Failed to connect to " << adr << ":" << port << ", because of timeout (" << conn_timeout << ")"); return false; } } @@ -712,22 +979,24 @@ POP_WARNINGS if (ec || !sock_.is_open()) { - LOG_PRINT("Some problems at connect, message: " << ec.message(), LOG_LEVEL_3); + _dbg3("Some problems at connect, message: " << ec.message()); return false; } - LOG_PRINT_L3("Connected success to " << adr << ':' << port); + _dbg3("Connected success to " << adr << ':' << port); bool r = new_connection_l->start(false, 1 < m_threads_count); if (r) { new_connection_l->get_context(conn_context); - //new_connection_l.reset(new connection<t_protocol_handler>(io_service_, m_config, m_sockets_count, m_pfilter)); + //new_connection_l.reset(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_pfilter)); } else { - LOG_ERROR("[sock " << new_connection_->socket().native_handle() << "] Failed to start connection, connections_count = " << m_sockets_count); + _erro("[sock " << new_connection_->socket().native_handle() << "] Failed to start connection, connections_count = " << m_sock_count); } + + new_connection_l->save_dbg_log(); return r; @@ -738,7 +1007,7 @@ POP_WARNINGS bool boosted_tcp_server<t_protocol_handler>::connect_async(const std::string& adr, const std::string& port, uint32_t conn_timeout, t_callback cb, const std::string& bind_ip) { TRY_ENTRY(); - connection_ptr new_connection_l(new connection<t_protocol_handler>(io_service_, m_config, m_sockets_count, m_pfilter) ); + connection_ptr new_connection_l(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_sock_number, m_pfilter, m_connection_type) ); boost::asio::ip::tcp::socket& sock_ = new_connection_l->socket(); ////////////////////////////////////////////////////////////////////////// @@ -748,7 +1017,7 @@ POP_WARNINGS boost::asio::ip::tcp::resolver::iterator end; if(iterator == end) { - LOG_ERROR("Failed to resolve " << adr); + _erro("Failed to resolve " << adr); return false; } ////////////////////////////////////////////////////////////////////////// @@ -768,7 +1037,7 @@ POP_WARNINGS { if(error != boost::asio::error::operation_aborted) { - LOG_PRINT_L3("Failed to connect to " << adr << ':' << port << ", because of timeout (" << conn_timeout << ")"); + _dbg3("Failed to connect to " << adr << ':' << port << ", because of timeout (" << conn_timeout << ")"); new_connection_l->socket().close(); } }); @@ -785,7 +1054,7 @@ POP_WARNINGS cb(conn_context, boost::asio::error::operation_aborted);//this mean that deadline timer already queued callback with cancel operation, rare situation }else { - LOG_PRINT_L3("[sock " << new_connection_l->socket().native_handle() << "] Connected success to " << adr << ':' << port << + _dbg3("[sock " << new_connection_l->socket().native_handle() << "] Connected success to " << adr << ':' << port << " from " << lep.address().to_string() << ':' << lep.port()); bool r = new_connection_l->start(false, 1 < m_threads_count); if (r) @@ -795,13 +1064,13 @@ POP_WARNINGS } else { - LOG_PRINT_L3("[sock " << new_connection_l->socket().native_handle() << "] Failed to start connection to " << adr << ':' << port); + _dbg3("[sock " << new_connection_l->socket().native_handle() << "] Failed to start connection to " << adr << ':' << port); cb(conn_context, boost::asio::error::fault); } } }else { - LOG_PRINT_L3("[sock " << new_connection_l->socket().native_handle() << "] Failed to connect to " << adr << ':' << port << + _dbg3("[sock " << new_connection_l->socket().native_handle() << "] Failed to connect to " << adr << ':' << port << " from " << lep.address().to_string() << ':' << lep.port() << ": " << ec_.message() << ':' << ec_.value()); cb(conn_context, ec_); } @@ -809,6 +1078,15 @@ POP_WARNINGS return true; CATCH_ENTRY_L0("boosted_tcp_server<t_protocol_handler>::connect_async", false); } -} -} + //--------------------------------------------------------------------------------- + template<class t_protocol_handler> + void boosted_tcp_server<t_protocol_handler>::detach_threads() + { + for (auto thread : m_threads) + thread->detach(); + } + + +} // namespace +} // namespace PRAGMA_WARNING_POP diff --git a/contrib/epee/include/net/http_base.h b/contrib/epee/include/net/http_base.h index 49b0839b2..2a227cc70 100644 --- a/contrib/epee/include/net/http_base.h +++ b/contrib/epee/include/net/http_base.h @@ -181,4 +181,4 @@ namespace net_utils }; } } -}
\ No newline at end of file +} diff --git a/contrib/epee/include/net/http_client.h b/contrib/epee/include/net/http_client.h index 5897a017e..66b28384c 100644 --- a/contrib/epee/include/net/http_client.h +++ b/contrib/epee/include/net/http_client.h @@ -872,4 +872,4 @@ using namespace std; } } -}
\ No newline at end of file +} diff --git a/contrib/epee/include/net/http_client_base.h b/contrib/epee/include/net/http_client_base.h index 571e27f73..450d44823 100644 --- a/contrib/epee/include/net/http_client_base.h +++ b/contrib/epee/include/net/http_client_base.h @@ -70,4 +70,4 @@ namespace epee i_target_handler* m_powner_filter; }; } -}
\ No newline at end of file +} diff --git a/contrib/epee/include/net/http_client_via_api_helper.h b/contrib/epee/include/net/http_client_via_api_helper.h index 45a70993b..391c44964 100644 --- a/contrib/epee/include/net/http_client_via_api_helper.h +++ b/contrib/epee/include/net/http_client_via_api_helper.h @@ -174,4 +174,4 @@ namespace net_utils return final_res; } } -}
\ No newline at end of file +} diff --git a/contrib/epee/include/net/http_protocol_handler.inl b/contrib/epee/include/net/http_protocol_handler.inl index 78b46427e..2458af047 100644 --- a/contrib/epee/include/net/http_protocol_handler.inl +++ b/contrib/epee/include/net/http_protocol_handler.inl @@ -677,4 +677,4 @@ namespace net_utils //-------------------------------------------------------------------------------------------- //-------------------------------------------------------------------------------------------- -//--------------------------------------------------------------------------------------------
\ No newline at end of file +//-------------------------------------------------------------------------------------------- diff --git a/contrib/epee/include/net/http_server_handlers_map2.h b/contrib/epee/include/net/http_server_handlers_map2.h index 201460130..f812077f1 100644 --- a/contrib/epee/include/net/http_server_handlers_map2.h +++ b/contrib/epee/include/net/http_server_handlers_map2.h @@ -65,7 +65,7 @@ CHECK_AND_ASSERT_MES(parse_res, false, "Failed to parse json: \r\n" << query_info.m_body); \ uint64_t ticks1 = epee::misc_utils::get_tick_count(); \ boost::value_initialized<command_type::response> resp;\ - if(!callback_f(static_cast<command_type::request&>(req), static_cast<command_type::response&>(resp), m_conn_context)) \ + if(!callback_f(static_cast<command_type::request&>(req), static_cast<command_type::response&>(resp))) \ { \ LOG_ERROR("Failed to " << #callback_f << "()"); \ response_info.m_response_code = 500; \ @@ -90,7 +90,7 @@ CHECK_AND_ASSERT_MES(parse_res, false, "Failed to parse bin body data, body size=" << query_info.m_body.size()); \ uint64_t ticks1 = misc_utils::get_tick_count(); \ boost::value_initialized<command_type::response> resp;\ - if(!callback_f(static_cast<command_type::request&>(req), static_cast<command_type::response&>(resp), m_conn_context)) \ + if(!callback_f(static_cast<command_type::request&>(req), static_cast<command_type::response&>(resp))) \ { \ LOG_ERROR("Failed to " << #callback_f << "()"); \ response_info.m_response_code = 500; \ @@ -173,7 +173,7 @@ epee::json_rpc::error_response fail_resp = AUTO_VAL_INIT(fail_resp); \ fail_resp.jsonrpc = "2.0"; \ fail_resp.id = req.id; \ - if(!callback_f(req.params, resp.result, fail_resp.error, m_conn_context)) \ + if(!callback_f(req.params, resp.result, fail_resp.error)) \ { \ epee::serialization::store_t_to_json(static_cast<epee::json_rpc::error_response&>(fail_resp), response_info.m_body); \ return true; \ @@ -202,7 +202,7 @@ else if(callback_name == method_name) \ { \ PREPARE_OBJECTS_FROM_JSON(command_type) \ - if(!callback_f(req.params, resp.result, m_conn_context)) \ + if(!callback_f(req.params, resp.result)) \ { \ epee::json_rpc::error_response fail_resp = AUTO_VAL_INIT(fail_resp); \ fail_resp.jsonrpc = "2.0"; \ diff --git a/contrib/epee/include/net/http_server_impl_base.h b/contrib/epee/include/net/http_server_impl_base.h index c02475c34..10f74b9a8 100644 --- a/contrib/epee/include/net/http_server_impl_base.h +++ b/contrib/epee/include/net/http_server_impl_base.h @@ -45,7 +45,7 @@ namespace epee public: http_server_impl_base() - : m_net_server() + : m_net_server(epee::net_utils::e_connection_type_RPC) {} explicit http_server_impl_base(boost::asio::io_service& external_io_service) @@ -75,6 +75,7 @@ namespace epee { //go to loop LOG_PRINT("Run net_service loop( " << threads_count << " threads)...", LOG_LEVEL_0); + _fact_c("net/RPClog", "Run net_service loop( " << threads_count << " threads)..."); if(!m_net_server.run_server(threads_count, wait)) { LOG_ERROR("Failed to run net tcp server!"); @@ -109,4 +110,4 @@ namespace epee protected: net_utils::boosted_tcp_server<net_utils::http::http_custom_handler<t_connection_context> > m_net_server; }; -}
\ No newline at end of file +} diff --git a/contrib/epee/include/net/levin_client_async.h b/contrib/epee/include/net/levin_client_async.h index 9e76cd509..be5847c74 100644 --- a/contrib/epee/include/net/levin_client_async.h +++ b/contrib/epee/include/net/levin_client_async.h @@ -574,4 +574,4 @@ namespace levin }; } -}
\ No newline at end of file +} diff --git a/contrib/epee/include/net/levin_helper.h b/contrib/epee/include/net/levin_helper.h index a8406103c..53b19c9fa 100644 --- a/contrib/epee/include/net/levin_helper.h +++ b/contrib/epee/include/net/levin_helper.h @@ -134,4 +134,4 @@ namespace levin return true; } } -}
\ No newline at end of file +} diff --git a/contrib/epee/include/net/levin_protocol_handler_async.h b/contrib/epee/include/net/levin_protocol_handler_async.h index e79768b04..a7fbffb4b 100644 --- a/contrib/epee/include/net/levin_protocol_handler_async.h +++ b/contrib/epee/include/net/levin_protocol_handler_async.h @@ -34,6 +34,9 @@ #include "levin_base.h" #include "misc_language.h" +#include <random> +#include <chrono> + namespace epee { @@ -81,6 +84,7 @@ public: async_protocol_handler_config():m_pcommands_handler(NULL), m_max_packet_size(LEVIN_DEFAULT_MAX_PACKET_SIZE) {} + void del_out_connections(size_t count); }; @@ -669,6 +673,34 @@ void async_protocol_handler_config<t_connection_context>::del_connection(async_p } //------------------------------------------------------------------------------------------ template<class t_connection_context> +void async_protocol_handler_config<t_connection_context>::del_out_connections(size_t count) +{ + std::vector <boost::uuids::uuid> out_connections; + CRITICAL_REGION_BEGIN(m_connects_lock); + for (auto& c: m_connects) + { + if (!c.second->m_connection_context.m_is_income) + out_connections.push_back(c.first); + } + + if (out_connections.size() == 0) + return; + + // close random out connections + // TODO or better just keep removing random elements (performance) + unsigned seed = std::chrono::system_clock::now().time_since_epoch().count(); + shuffle(out_connections.begin(), out_connections.end(), std::default_random_engine(seed)); + while (count > 0 && out_connections.size() > 0) + { + close(*out_connections.begin()); + del_connection(m_connects.at(*out_connections.begin())); + --count; + } + + CRITICAL_REGION_END(); +} +//------------------------------------------------------------------------------------------ +template<class t_connection_context> void async_protocol_handler_config<t_connection_context>::add_connection(async_protocol_handler<t_connection_context>* pconn) { CRITICAL_REGION_BEGIN(m_connects_lock); diff --git a/contrib/epee/include/net/munin_connection_handler.h b/contrib/epee/include/net/munin_connection_handler.h index 8579339c5..ed2f59f22 100644 --- a/contrib/epee/include/net/munin_connection_handler.h +++ b/contrib/epee/include/net/munin_connection_handler.h @@ -373,4 +373,4 @@ namespace net_utils } } } -#endif//!_MUNIN_CONNECTION_HANDLER_H_
\ No newline at end of file +#endif//!_MUNIN_CONNECTION_HANDLER_H_ diff --git a/contrib/epee/include/net/munin_node_server.h b/contrib/epee/include/net/munin_node_server.h index 07637f550..e6df390cb 100644 --- a/contrib/epee/include/net/munin_node_server.h +++ b/contrib/epee/include/net/munin_node_server.h @@ -46,4 +46,4 @@ namespace net_utils } } } -#endif//!_MUNIN_NODE_SERVER_H_
\ No newline at end of file +#endif//!_MUNIN_NODE_SERVER_H_ diff --git a/contrib/epee/include/net/net_utils_base.h b/contrib/epee/include/net/net_utils_base.h index 90e352787..f963e7746 100644 --- a/contrib/epee/include/net/net_utils_base.h +++ b/contrib/epee/include/net/net_utils_base.h @@ -55,6 +55,8 @@ namespace net_utils time_t m_last_send; uint64_t m_recv_cnt; uint64_t m_send_cnt; + double m_current_speed_down; + double m_current_speed_up; connection_context_base(boost::uuids::uuid connection_id, long remote_ip, int remote_port, bool is_income, @@ -68,7 +70,9 @@ namespace net_utils m_last_recv(last_recv), m_last_send(last_send), m_recv_cnt(recv_cnt), - m_send_cnt(send_cnt) + m_send_cnt(send_cnt), + m_current_speed_down(0), + m_current_speed_up(0) {} connection_context_base(): m_connection_id(), @@ -79,7 +83,9 @@ namespace net_utils m_last_recv(0), m_last_send(0), m_recv_cnt(0), - m_send_cnt(0) + m_send_cnt(0), + m_current_speed_down(0), + m_current_speed_up(0) {} connection_context_base& operator=(const connection_context_base& a) diff --git a/contrib/epee/include/net/protocol_switcher.h b/contrib/epee/include/net/protocol_switcher.h index ca0ce6f97..3b153d19c 100644 --- a/contrib/epee/include/net/protocol_switcher.h +++ b/contrib/epee/include/net/protocol_switcher.h @@ -118,4 +118,4 @@ namespace net_utils } } } -#endif //_PROTOCOL_SWITCHER_H_
\ No newline at end of file +#endif //_PROTOCOL_SWITCHER_H_ diff --git a/contrib/epee/include/net/rpc_method_name.h b/contrib/epee/include/net/rpc_method_name.h index c226639c4..1c327bc31 100644 --- a/contrib/epee/include/net/rpc_method_name.h +++ b/contrib/epee/include/net/rpc_method_name.h @@ -28,4 +28,4 @@ #pragma once -#define RPC_METHOD_NAME(name) static inline const char* methodname(){return name;}
\ No newline at end of file +#define RPC_METHOD_NAME(name) static inline const char* methodname(){return name;} diff --git a/contrib/epee/include/net/smtp.h b/contrib/epee/include/net/smtp.h index d2e8598fd..5f2b842d5 100644 --- a/contrib/epee/include/net/smtp.h +++ b/contrib/epee/include/net/smtp.h @@ -178,4 +178,4 @@ namespace net_utils } } -//#include "smtp.inl"
\ No newline at end of file +//#include "smtp.inl" diff --git a/contrib/epee/include/net/smtp.inl b/contrib/epee/include/net/smtp.inl index d42c8b950..c16372c88 100644 --- a/contrib/epee/include/net/smtp.inl +++ b/contrib/epee/include/net/smtp.inl @@ -1566,4 +1566,4 @@ namespace net_utils } } -}
\ No newline at end of file +} diff --git a/contrib/epee/include/net/smtp_helper.h b/contrib/epee/include/net/smtp_helper.h index b8252e1cf..7827315a2 100644 --- a/contrib/epee/include/net/smtp_helper.h +++ b/contrib/epee/include/net/smtp_helper.h @@ -85,4 +85,4 @@ namespace net_utils } } } -}
\ No newline at end of file +} diff --git a/contrib/epee/include/serialization/enableable.h b/contrib/epee/include/serialization/enableable.h index ab1d799e6..e1be2a774 100644 --- a/contrib/epee/include/serialization/enableable.h +++ b/contrib/epee/include/serialization/enableable.h @@ -50,4 +50,4 @@ namespace epee { // construct from specified values } }; -}
\ No newline at end of file +} diff --git a/contrib/epee/include/soci_helper.h b/contrib/epee/include/soci_helper.h index 813edc1fc..1da5aa7e2 100644 --- a/contrib/epee/include/soci_helper.h +++ b/contrib/epee/include/soci_helper.h @@ -139,4 +139,4 @@ namespace soci std::string m_connection_string; }; } -/*}*/
\ No newline at end of file +/*}*/ diff --git a/contrib/epee/include/storages/crypted_storage.h b/contrib/epee/include/storages/crypted_storage.h index d6e6edcba..163728cfc 100644 --- a/contrib/epee/include/storages/crypted_storage.h +++ b/contrib/epee/include/storages/crypted_storage.h @@ -59,4 +59,4 @@ public: }; } -#endif //_CRYPTED_STORAGE_H_
\ No newline at end of file +#endif //_CRYPTED_STORAGE_H_ diff --git a/contrib/epee/include/storages/gzipped_inmemstorage.h b/contrib/epee/include/storages/gzipped_inmemstorage.h index 5c53fffa7..229a56da6 100644 --- a/contrib/epee/include/storages/gzipped_inmemstorage.h +++ b/contrib/epee/include/storages/gzipped_inmemstorage.h @@ -65,4 +65,4 @@ namespace StorageNamed } } -#endif
\ No newline at end of file +#endif diff --git a/contrib/epee/include/storages/levin_abstract_invoke2.h b/contrib/epee/include/storages/levin_abstract_invoke2.h index 1b32c51d1..73ede1b12 100644 --- a/contrib/epee/include/storages/levin_abstract_invoke2.h +++ b/contrib/epee/include/storages/levin_abstract_invoke2.h @@ -185,7 +185,7 @@ namespace epee } return res; - }; + } template<class t_owner, class t_in_type, class t_context, class callback_t> int buff_to_t_adapter(t_owner* powner, int command, const std::string& in_buff, callback_t cb, t_context& context) @@ -199,7 +199,7 @@ namespace epee boost::value_initialized<t_in_type> in_struct; static_cast<t_in_type&>(in_struct).load(strg); return cb(command, in_struct, context); - }; + } #define CHAIN_LEVIN_INVOKE_MAP2(context_type) \ int invoke(int command, const std::string& in_buff, std::string& buff_out, context_type& context) \ diff --git a/contrib/epee/include/storages/parserse_base_utils.h b/contrib/epee/include/storages/parserse_base_utils.h index baafb5623..c809392f4 100644 --- a/contrib/epee/include/storages/parserse_base_utils.h +++ b/contrib/epee/include/storages/parserse_base_utils.h @@ -257,4 +257,4 @@ namespace misc_utils } } } -}
\ No newline at end of file +} diff --git a/contrib/epee/include/storages/portable_storage_base.h b/contrib/epee/include/storages/portable_storage_base.h index 3f1637538..93132548b 100644 --- a/contrib/epee/include/storages/portable_storage_base.h +++ b/contrib/epee/include/storages/portable_storage_base.h @@ -157,4 +157,4 @@ namespace epee typedef section* hsection; typedef array_entry* harray; } -}
\ No newline at end of file +} diff --git a/contrib/epee/include/storages/portable_storage_from_bin.h b/contrib/epee/include/storages/portable_storage_from_bin.h index e9b7e2e6f..bc2fb1463 100644 --- a/contrib/epee/include/storages/portable_storage_from_bin.h +++ b/contrib/epee/include/storages/portable_storage_from_bin.h @@ -278,4 +278,4 @@ namespace epee m_count -= len; } } -}
\ No newline at end of file +} diff --git a/contrib/epee/include/storages/portable_storage_from_json.h b/contrib/epee/include/storages/portable_storage_from_json.h index 4e74fb7a5..41f270627 100644 --- a/contrib/epee/include/storages/portable_storage_from_json.h +++ b/contrib/epee/include/storages/portable_storage_from_json.h @@ -376,4 +376,4 @@ namespace epee } } } -}
\ No newline at end of file +} diff --git a/contrib/epee/include/storages/portable_storage_template_helper.h b/contrib/epee/include/storages/portable_storage_template_helper.h index 008f44321..88515b5c3 100644 --- a/contrib/epee/include/storages/portable_storage_template_helper.h +++ b/contrib/epee/include/storages/portable_storage_template_helper.h @@ -120,4 +120,4 @@ namespace epee return std::move(binary_buff); } } -}
\ No newline at end of file +} diff --git a/contrib/epee/include/storages/portable_storage_to_bin.h b/contrib/epee/include/storages/portable_storage_to_bin.h index baf90290a..5695143b0 100644 --- a/contrib/epee/include/storages/portable_storage_to_bin.h +++ b/contrib/epee/include/storages/portable_storage_to_bin.h @@ -209,4 +209,4 @@ namespace epee return true; } } -}
\ No newline at end of file +} diff --git a/contrib/epee/include/storages/portable_storage_to_json.h b/contrib/epee/include/storages/portable_storage_to_json.h index e3fdcec29..43e341291 100644 --- a/contrib/epee/include/storages/portable_storage_to_json.h +++ b/contrib/epee/include/storages/portable_storage_to_json.h @@ -178,4 +178,4 @@ namespace epee strm << make_indent(indent) << "}"; } } -}
\ No newline at end of file +} diff --git a/contrib/epee/include/storages/portable_storage_val_converters.h b/contrib/epee/include/storages/portable_storage_val_converters.h index 6ea505886..e9b91c82b 100644 --- a/contrib/epee/include/storages/portable_storage_val_converters.h +++ b/contrib/epee/include/storages/portable_storage_val_converters.h @@ -166,4 +166,4 @@ POP_WARNINGS convert_to_same<from_type, to_type, std::is_same<to_type, from_type>::value>::convert(from, to); } } -}
\ No newline at end of file +} diff --git a/contrib/epee/include/syncobj.h b/contrib/epee/include/syncobj.h index b7273da8e..b81eb43a9 100644 --- a/contrib/epee/include/syncobj.h +++ b/contrib/epee/include/syncobj.h @@ -35,10 +35,14 @@ #include <boost/thread/locks.hpp> #include <boost/thread/mutex.hpp> #include <boost/thread/recursive_mutex.hpp> +#include <thread> +#include <chrono> namespace epee { + extern unsigned int g_test_dbg_lock_sleep; + struct simple_event { simple_event() : m_rised(false) @@ -215,10 +219,10 @@ namespace epee #define SHARED_CRITICAL_REGION_BEGIN(x) { shared_guard critical_region_var(x) #define EXCLUSIVE_CRITICAL_REGION_BEGIN(x) { exclusive_guard critical_region_var(x) -#define CRITICAL_REGION_LOCAL(x) epee::critical_region_t<decltype(x)> critical_region_var(x) -#define CRITICAL_REGION_BEGIN(x) { epee::critical_region_t<decltype(x)> critical_region_var(x) -#define CRITICAL_REGION_LOCAL1(x) epee::critical_region_t<decltype(x)> critical_region_var1(x) -#define CRITICAL_REGION_BEGIN1(x) { epee::critical_region_t<decltype(x)> critical_region_var1(x) +#define CRITICAL_REGION_LOCAL(x) {std::this_thread::sleep_for(std::chrono::milliseconds(epee::g_test_dbg_lock_sleep));} epee::critical_region_t<decltype(x)> critical_region_var(x) +#define CRITICAL_REGION_BEGIN(x) { std::this_thread::sleep_for(std::chrono::milliseconds(epee::g_test_dbg_lock_sleep)); epee::critical_region_t<decltype(x)> critical_region_var(x) +#define CRITICAL_REGION_LOCAL1(x) {std::this_thread::sleep_for(std::chrono::milliseconds(epee::g_test_dbg_lock_sleep));} epee::critical_region_t<decltype(x)> critical_region_var1(x) +#define CRITICAL_REGION_BEGIN1(x) { std::this_thread::sleep_for(std::chrono::milliseconds(epee::g_test_dbg_lock_sleep)); epee::critical_region_t<decltype(x)> critical_region_var1(x) #define CRITICAL_REGION_END() } diff --git a/contrib/epee/include/time_helper.h b/contrib/epee/include/time_helper.h index 958176da6..244b35800 100644 --- a/contrib/epee/include/time_helper.h +++ b/contrib/epee/include/time_helper.h @@ -156,4 +156,4 @@ PRAGMA_WARNING_POP #endif } -}
\ No newline at end of file +} diff --git a/contrib/epee/include/warnings.h b/contrib/epee/include/warnings.h index 37d7a2900..df5c7d1db 100644 --- a/contrib/epee/include/warnings.h +++ b/contrib/epee/include/warnings.h @@ -27,4 +27,4 @@ #define DISABLE_GCC_AND_CLANG_WARNING(w) _Pragma(BOOST_PP_STRINGIZE(GCC diagnostic ignored BOOST_PP_STRINGIZE(-W##w))) -#endif
\ No newline at end of file +#endif diff --git a/contrib/epee/tests/data/storages/invalid_storage_1.bin b/contrib/epee/tests/data/storages/invalid_storage_1.bin Binary files differindex fac7b3e97..f64bef38e 100644 --- a/contrib/epee/tests/data/storages/invalid_storage_1.bin +++ b/contrib/epee/tests/data/storages/invalid_storage_1.bin diff --git a/contrib/epee/tests/data/storages/invalid_storage_3.bin b/contrib/epee/tests/data/storages/invalid_storage_3.bin index b5c31aa05..4233bf25c 100644 --- a/contrib/epee/tests/data/storages/invalid_storage_3.bin +++ b/contrib/epee/tests/data/storages/invalid_storage_3.bin @@ -1 +1 @@ -¢IMóÙ Ÿˆm_bo
\ No newline at end of file +¢IMóÙ Ÿˆm_bo diff --git a/contrib/epee/tests/data/storages/invalid_storage_4.bin b/contrib/epee/tests/data/storages/invalid_storage_4.bin Binary files differindex 4f8372d19..69017244a 100644 --- a/contrib/epee/tests/data/storages/invalid_storage_4.bin +++ b/contrib/epee/tests/data/storages/invalid_storage_4.bin diff --git a/contrib/epee/tests/data/storages/valid_storage.bin b/contrib/epee/tests/data/storages/valid_storage.bin Binary files differindex e13f780b1..2af0abf50 100644 --- a/contrib/epee/tests/data/storages/valid_storage.bin +++ b/contrib/epee/tests/data/storages/valid_storage.bin diff --git a/contrib/epee/tests/generate_vc_proj.bat b/contrib/epee/tests/generate_vc_proj.bat index a81bdce05..2b3fee953 100644 --- a/contrib/epee/tests/generate_vc_proj.bat +++ b/contrib/epee/tests/generate_vc_proj.bat @@ -2,4 +2,4 @@ mkdir build cd build cmake "-DBoost_USE_STATIC_LIBS=TRUE" -G "Visual Studio 11 Win64" ../src cd .. -pause
\ No newline at end of file +pause diff --git a/contrib/epee/tests/src/net/test_net.h b/contrib/epee/tests/src/net/test_net.h index 0b6dc1f73..5b21036bb 100644 --- a/contrib/epee/tests/src/net/test_net.h +++ b/contrib/epee/tests/src/net/test_net.h @@ -400,4 +400,4 @@ namespace tests } } -}
\ No newline at end of file +} diff --git a/contrib/epee/tests/src/storages/portable_storages_test.h b/contrib/epee/tests/src/storages/portable_storages_test.h index ecded8dad..89f217d95 100644 --- a/contrib/epee/tests/src/storages/portable_storages_test.h +++ b/contrib/epee/tests/src/storages/portable_storages_test.h @@ -229,4 +229,4 @@ namespace epee } } -}
\ No newline at end of file +} diff --git a/contrib/epee/tests/src/tests.cpp b/contrib/epee/tests/src/tests.cpp index ed045d833..8d61334cc 100644 --- a/contrib/epee/tests/src/tests.cpp +++ b/contrib/epee/tests/src/tests.cpp @@ -56,4 +56,4 @@ int main(int argc, char* argv[]) tests::test_portable_storages(tests_data_path); } return 1; -}
\ No newline at end of file +} diff --git a/contrib/otshell_utils/CMakeLists.txt b/contrib/otshell_utils/CMakeLists.txt new file mode 100644 index 000000000..7413e0dc5 --- /dev/null +++ b/contrib/otshell_utils/CMakeLists.txt @@ -0,0 +1,14 @@ +cmake_minimum_required (VERSION 2.6) +project (otshell CXX) + +# Add executable + +file(GLOB otshell_utils_sources # All files in directory: + "*.h" + "*.hpp" + "*.cpp" +) + +add_library (otshell_utils STATIC ${otshell_utils_sources}) +set_target_properties (otshell_utils PROPERTIES OUTPUT_NAME "otshell_utils") +#target_link_libraries (upnpc-static ${LDLIBS}) # to add used libs diff --git a/contrib/otshell_utils/LICENCE.txt b/contrib/otshell_utils/LICENCE.txt new file mode 100644 index 000000000..f351acf10 --- /dev/null +++ b/contrib/otshell_utils/LICENCE.txt @@ -0,0 +1,21 @@ + +This are some files also from OpenTransactions / otshell project, +developed thanks to the awesome OpenTransaction project, organization and developers :) + +Parts of code here was also developed thanks to the excellent Monero project, +thanks to Monero project, organization and developers :) + +[Some] files/code here (in external/otshell_utils) are under licence defined in +src/doc/LICENCE-otshell.txt ; +Others are from monero, with licence in src/doc/LICENCE-monero.txt ; + +For me (rfree) the licence seem compatbile so no problem, personally (as author of many parts of the code, +possibly not all) I do not worry who uses it how; I'am not a lawyer. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Please share :-) This licence can be used e.g. for parts of code that are usable in both open-source FOSS project +Monero and Open Transactions, to share and develop both faster. +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + + diff --git a/contrib/otshell_utils/ccolor.cpp b/contrib/otshell_utils/ccolor.cpp new file mode 100644 index 000000000..cd93e0de7 --- /dev/null +++ b/contrib/otshell_utils/ccolor.cpp @@ -0,0 +1,116 @@ +#include "ccolor.hpp" +#include <cstdarg> + +// from http://stackoverflow.com/questions/2616906/how-do-i-output-coloured-text-to-a-linux-terminal +// from http://wiznet.gr/src/ccolor.zip +// edited by rfree - as part of https://github.com/rfree/Open-Transactions/ + +using namespace std; + +#ifdef _MSC_VER + +#define snprintf c99_snprintf + +inline int c99_vsnprintf(char* str, size_t size, const char* format, va_list ap) { + int count = -1; + if (size != 0) + count = _vsnprintf_s(str, size, _TRUNCATE, format, ap); + if (count == -1) + count = _vscprintf(format, ap); + return count; +} + +inline int c99_snprintf(char* str, size_t size, const char* format, ...) { + int count; + va_list ap; + va_start(ap, format); + count = c99_vsnprintf(str, size, format, ap); + va_end(ap); + return count; +} +#endif // _MSC_VER + +#define CC_CONSOLE_COLOR_DEFAULT "\033[0m" +#define CC_FORECOLOR(C) "\033[" #C "m" +#define CC_BACKCOLOR(C) "\033[" #C "m" +#define CC_ATTR(A) "\033[" #A "m" + +namespace zkr +{ + enum Color + { + Black, + Red, + Green, + Yellow, + Blue, + Magenta, + Cyan, + White, + Default = 9 + }; + + enum Attributes + { + Reset, + Bright, + Dim, + Underline, + Blink, + Reverse, + Hidden + }; + + char * cc::color(int attr, int fg, int bg) + { + static const int size = 20; + static char command[size]; + + /* Command is the control command to the terminal */ + snprintf(command, size, "%c[%d;%d;%dm", 0x1B, attr, fg + 30, bg + 40); + return command; + } + + + const char *cc::console = CC_CONSOLE_COLOR_DEFAULT; + const char *cc::underline = CC_ATTR(4); + const char *cc::bold = CC_ATTR(1); + + const char *cc::fore::black = CC_FORECOLOR(30); + const char *cc::fore::blue = CC_FORECOLOR(34); + const char *cc::fore::red = CC_FORECOLOR(31); + const char *cc::fore::magenta = CC_FORECOLOR(35); + const char *cc::fore::green = CC_FORECOLOR(92); + const char *cc::fore::cyan = CC_FORECOLOR(36); + const char *cc::fore::yellow = CC_FORECOLOR(33); + const char *cc::fore::white = CC_FORECOLOR(37); + const char *cc::fore::console = CC_FORECOLOR(39); + + const char *cc::fore::lightblack = CC_FORECOLOR(90); + const char *cc::fore::lightblue = CC_FORECOLOR(94); + const char *cc::fore::lightred = CC_FORECOLOR(91); + const char *cc::fore::lightmagenta = CC_FORECOLOR(95); + const char *cc::fore::lightgreen = CC_FORECOLOR(92); + const char *cc::fore::lightcyan = CC_FORECOLOR(96); + const char *cc::fore::lightyellow = CC_FORECOLOR(93); + const char *cc::fore::lightwhite = CC_FORECOLOR(97); + + const char *cc::back::black = CC_BACKCOLOR(40); + const char *cc::back::blue = CC_BACKCOLOR(44); + const char *cc::back::red = CC_BACKCOLOR(41); + const char *cc::back::magenta = CC_BACKCOLOR(45); + const char *cc::back::green = CC_BACKCOLOR(42); + const char *cc::back::cyan = CC_BACKCOLOR(46); + const char *cc::back::yellow = CC_BACKCOLOR(43); + const char *cc::back::white = CC_BACKCOLOR(47); + const char *cc::back::console = CC_BACKCOLOR(49); + + const char *cc::back::lightblack = CC_BACKCOLOR(100); + const char *cc::back::lightblue = CC_BACKCOLOR(104); + const char *cc::back::lightred = CC_BACKCOLOR(101); + const char *cc::back::lightmagenta = CC_BACKCOLOR(105); + const char *cc::back::lightgreen = CC_BACKCOLOR(102); + const char *cc::back::lightcyan = CC_BACKCOLOR(106); + const char *cc::back::lightyellow = CC_BACKCOLOR(103); + const char *cc::back::lightwhite = CC_BACKCOLOR(107); +} diff --git a/contrib/otshell_utils/ccolor.hpp b/contrib/otshell_utils/ccolor.hpp new file mode 100644 index 000000000..bf5a601a2 --- /dev/null +++ b/contrib/otshell_utils/ccolor.hpp @@ -0,0 +1,73 @@ +// ccolor.hpp + +// from http://stackoverflow.com/questions/2616906/how-do-i-output-coloured-text-to-a-linux-terminal +// from http://wiznet.gr/src/ccolor.zip +// edited by rfree - as part of https://github.com/rfree/Open-Transactions/ + +#ifndef INCLUDE_OT_ccolor +#define INCLUDE_OT_ccolor + +#include <iostream> +#include <stdio.h> + +namespace zkr +{ + class cc + { + public: + + class fore + { + public: + static const char *black; + static const char *blue; + static const char *red; + static const char *magenta; + static const char *green; + static const char *cyan; + static const char *yellow; + static const char *white; + static const char *console; + + static const char *lightblack; + static const char *lightblue; + static const char *lightred; + static const char *lightmagenta; + static const char *lightgreen; + static const char *lightcyan; + static const char *lightyellow; + static const char *lightwhite; + }; + + class back + { + public: + static const char *black; + static const char *blue; + static const char *red; + static const char *magenta; + static const char *green; + static const char *cyan; + static const char *yellow; + static const char *white; + static const char *console; + + static const char *lightblack; + static const char *lightblue; + static const char *lightred; + static const char *lightmagenta; + static const char *lightgreen; + static const char *lightcyan; + static const char *lightyellow; + static const char *lightwhite; + }; + + static char *color(int attr, int fg, int bg); + static const char *console; + static const char *underline; + static const char *bold; + }; +} + +#endif + diff --git a/contrib/otshell_utils/lib_common1.hpp b/contrib/otshell_utils/lib_common1.hpp new file mode 100644 index 000000000..456e63fbb --- /dev/null +++ b/contrib/otshell_utils/lib_common1.hpp @@ -0,0 +1,52 @@ +/* See other files here for the LICENCE that applies here. */ + + +#ifndef INCLUDE_OT_NEWCLI_COMMON1 +#define INCLUDE_OT_NEWCLI_COMMON1 + +#include <string> +#include <cstring> +#include <vector> +#include <map> +#include <list> +#include <algorithm> +#include <iostream> +#include <fstream> +#include <sstream> +#include <set> +#include <iterator> +#include <stdexcept> + +#include <functional> +#include <memory> +#include <thread> +#include <atomic> +#include <mutex> + + +// list of thigs from libraries that we pull into namespace nOT::nNewcli +// we might still need to copy/paste it in few places to make IDEs pick it up correctly +#define INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 \ + using std::string; \ + using std::vector; \ + using std::vector; \ + using std::list; \ + using std::set; \ + using std::map; \ + using std::ostream; \ + using std::istream; \ + using std::cin; \ + using std::cerr; \ + using std::cout; \ + using std::cerr; \ + using std::endl; \ + using std::function; \ + using std::unique_ptr; \ + using std::shared_ptr; \ + using std::weak_ptr; \ + using std::enable_shared_from_this; \ + using std::mutex; \ + using std::lock_guard; \ + +#endif + diff --git a/contrib/otshell_utils/runoptions.cpp b/contrib/otshell_utils/runoptions.cpp new file mode 100644 index 000000000..ffd37eae4 --- /dev/null +++ b/contrib/otshell_utils/runoptions.cpp @@ -0,0 +1,69 @@ +/* See other files here for the LICENCE that applies here. */ +/* See header file .hpp for info */ + +#include "runoptions.hpp" + +#include "lib_common1.hpp" + +namespace nOT { + +INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 // <=== namespaces + +// (no debug - this is the default) +// +nodebug (no debug) +// +debug ...... --asdf +// +debug +debugcerr .... --asfs +// +debug +debugfile .... --asfs + +cRunOptions::cRunOptions() + : mRunMode(eRunModeCurrent), mDebug(false), mDebugSendToFile(false), mDebugSendToCerr(false) + ,mDoRunDebugshow(false) +{ } + +vector<string> cRunOptions::ExecuteRunoptionsAndRemoveThem(const vector<string> & args) { + vector<string> arg_clear; // will store only the arguments that are not removed + + for (auto arg : args) { + bool thisIsRunoption=false; + + if (arg.size()>0) { + if (arg.at(0) == '+') thisIsRunoption=true; + } + + if (thisIsRunoption) Exec(arg); // *** + if (! thisIsRunoption) arg_clear.push_back(arg); + } + + Normalize(); + + return arg_clear; +} + +void cRunOptions::Exec(const string & runoption) { // eg: Exec("+debug"); + if (runoption == "+nodebug") { mDebug=false; } + else if (runoption == "+debug") { mDebug=true; } + else if (runoption == "+debugcerr") { mDebug=true; mDebugSendToCerr=true; } + else if (runoption == "+debugfile") { mDebug=true; mDebugSendToFile=true; } + else if (runoption == "+demo") { mRunMode=eRunModeDemo; } + else if (runoption == "+normal") { mRunMode=eRunModeNormal; } + else if (runoption == "+current") { mRunMode=eRunModeCurrent; } + else if (runoption == "+debugshow") { mDebug=true; mDebugSendToCerr=true; mDoRunDebugshow=true; } + else { + cerr << "Unknown runoption in Exec: '" << runoption << "'" << endl; + throw std::runtime_error("Unknown runoption"); + } + // cerr<<"debug="<<mDebug<<endl; +} + +void cRunOptions::Normalize() { + if (mDebug) { + if (!( mDebugSendToFile || mDebugSendToCerr )) mDebugSendToCerr=true; // if debug is on then send to something, e.g. to cerr + } +} + + +cRunOptions gRunOptions; // (extern) + +} // namespace OT + + diff --git a/contrib/otshell_utils/runoptions.hpp b/contrib/otshell_utils/runoptions.hpp new file mode 100644 index 000000000..219d3b509 --- /dev/null +++ b/contrib/otshell_utils/runoptions.hpp @@ -0,0 +1,58 @@ +/* See other files here for the LICENCE that applies here. */ +/* +Template for new files, replace word "template" and later delete this line here. +*/ + +#ifndef INCLUDE_OT_NEWCLI_runoptions_hpp +#define INCLUDE_OT_NEWCLI_runoptions_hpp + +#include "lib_common1.hpp" + +namespace nOT { + +INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 // <=== namespaces + +/** Global options to run this program main() Eg used for developer's special options like +setdemo +setdebug. +This is NOT for all the other options that are parsed and executed by program. */ +class cRunOptions { + public: + enum tRunMode { ///< Type of run mode - is this normal, or demonstration etc. + eRunModeCurrent=1, ///< currently developed version + eRunModeDemo, ///< best currently available Demo of something nice + eRunModeNormal, ///< do the normal things that the program should do + }; + + private: + tRunMode mRunMode; ///< selected run mode + + bool mDebug; // turn debug on, Eg: +debug without it probably nothing will be written to debug (maybe just error etc) + bool mDebugSendToFile; // send to file, Eg: for +debugfile ; also turns on debug + bool mDebugSendToCerr; // send to cerr, Eg: for +debugcerr ; also turns on debug + // if debug is set but not any other DebugSend* then we will default to sending to debugcerr + + bool mDoRunDebugshow; + + public: + tRunMode getTRunMode() const { return mRunMode; } + bool getDebug() const { return mDebug; } + bool getDebugSendToFile() const { return mDebugSendToFile; } + bool getDebugSendToCerr() const { return mDebugSendToCerr; } + bool getDoRunDebugshow() const { return mDoRunDebugshow; } + + cRunOptions(); + + vector<string> ExecuteRunoptionsAndRemoveThem(const vector<string> & args); + void Exec(const string & runoption); // eg: Exec("+debug"); + + void Normalize(); +}; + +extern cRunOptions gRunOptions; + + +} // namespace nOT + + + +#endif + diff --git a/contrib/otshell_utils/utils.cpp b/contrib/otshell_utils/utils.cpp new file mode 100644 index 000000000..e1d7d9a14 --- /dev/null +++ b/contrib/otshell_utils/utils.cpp @@ -0,0 +1,804 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief various general utils taken from (and relate to) otshell project, including loggiang/debug + +/* See other files here for the LICENCE that applies here. */ +/* See header file .hpp for info */ + +#include <algorithm> +#include <functional> +#include <cctype> +#include <locale> +#include <fstream> +#include <iostream> +#include <iomanip> + +#include "utils.hpp" + +#include "ccolor.hpp" + +#include "lib_common1.hpp" + +#include "runoptions.hpp" + +#if defined(_WIN32) || defined(WIN32) || defined(_WIN64) || defined (WIN64) + #define OS_TYPE_WINDOWS +#elif defined(__unix__) || defined(__posix) || defined(__linux) || defined(__darwin) || defined(__APPLE__) || defined(__clang__) + #define OS_TYPE_POSIX +#else + #warning "Compiler/OS platform is not recognized. Just assuming it will work as POSIX then" + #define OS_TYPE_POSIX +#endif + +#if defined(OS_TYPE_WINDOWS) + #include <windows.h> +#elif defined(OS_TYPE_POSIX) + #include <sys/types.h> + #include <sys/stat.h> + #include <unistd.h> +#else + #error "Compiler/OS platform detection failed - not supported" +#endif + + +namespace nOT { +namespace nUtils { + +INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 // <=== namespaces + +// ==================================================================== + +// Numerical values of the debug levels - see hpp +const int _debug_level_nr_dbg3=20; +const int _debug_level_nr_dbg2=30; +const int _debug_level_nr_dbg1=40; +const int _debug_level_nr_info=50; +const int _debug_level_nr_note=60; +const int _debug_level_nr_fact=75; +const int _debug_level_nr_mark=80; +const int _debug_level_nr_warn=90; +const int _debug_level_nr_erro=100; + +// ==================================================================== + +myexception::myexception(const char * what) + : std::runtime_error(what) +{ } + +myexception::myexception(const std::string &what) + : std::runtime_error(what) +{ } + +void myexception::Report() const { + _erro("Error: " << what()); +} + +//myexception::~myexception() { } + +// ==================================================================== + +// text trimming +// http://stackoverflow.com/questions/216823/whats-the-best-way-to-trim-stdstring +std::string & ltrim(std::string &s) { + s.erase(s.begin(), std::find_if(s.begin(), s.end(), std::not1(std::ptr_fun<int, int>(std::isspace)))); + return s; +} + +std::string & rtrim(std::string &s) { + s.erase(std::find_if(s.rbegin(), s.rend(), std::not1(std::ptr_fun<int, int>(std::isspace))).base(), s.end()); + return s; +} + +std::string & trim(std::string &s) { + return ltrim(rtrim(s)); +} + +std::string get_current_time() { + std::chrono::system_clock::time_point now = std::chrono::system_clock::now(); + time_t time_now = std::chrono::system_clock::to_time_t(now); + std::chrono::high_resolution_clock::duration duration = now.time_since_epoch(); + int64_t micro = std::chrono::duration_cast<std::chrono::microseconds>(duration).count(); + + // std::localtime() - This function may not be thread-safe. + #ifdef OS_TYPE_WINDOWS + struct tm * tm_pointer = std::localtime( &time_now ); // thread-safe on mingw-w64 (thread local variable) and on MSVC btw + // http://stackoverflow.com/questions/18551409/localtime-r-support-on-mingw + // tm_pointer points to thread-local data, memory is owned/managed by the system/library + #else + // linux, freebsd, have this + struct tm tm_object; // automatic storage duration http://en.cppreference.com/w/cpp/language/storage_duration + struct tm * tm_pointer = & tm_object; // just point to our data + auto x = localtime_r( &time_now , tm_pointer ); // modifies our own (this thread) data in tm_object, this is safe http://linux.die.net/man/3/localtime_r + if (x != tm_pointer) return "(internal error in get_current_time)"; // redundant check in case of broken implementation of localtime_r + #endif + // tm_pointer now points to proper time data, and that memory is automatically managed + if (!tm_pointer) return "(internal error in get_current_time - NULL)"; // redundant check in case of broken implementation of used library methods + + std::stringstream stream; + stream << std::setfill('0') + << std::setw(2) << tm_pointer->tm_year+1900 + << '-' << std::setw(2) << tm_pointer->tm_mon+1 + << '-' << std::setw(2) << tm_pointer->tm_mday + << ' ' << std::setw(2) << tm_pointer->tm_hour + << ':' << std::setw(2) << tm_pointer->tm_min + << ':' << std::setw(2) << tm_pointer->tm_sec + << '.' << std::setw(6) << (micro%1000000); // 6 because microseconds + return stream.str(); +} + +cNullstream g_nullstream; // extern a stream that does nothing (eats/discards data) + +std::recursive_mutex gLoggerGuard; // extern +std::atomic<int> gLoggerGuardDepth; // extern + +std::atomic<int> & gLoggerGuardDepth_Get() { + // TODO std::once would be nicer here + + static bool once=0; + + if (!once) { // initialize it once + once=1; + gLoggerGuardDepth=0; + } + + return gLoggerGuardDepth; // global, atomic counter +} + + +// ==================================================================== + +namespace nDetail { + +const char* DbgShortenCodeFileName(const char *s) { + const char *p = s; + const char *a = s; + + bool inc=1; + while (*p) { + ++p; + if (inc && ('\0' != * p)) { a=p; inc=false; } // point to the current character (if valid) becasue previous one was slash + if ((*p)=='/') { a=p; inc=true; } // point at current slash (but set inc to try to point to next character) + } + return a; +} + +} + +// a workaround for MSVC compiler; e.g. see https://bugs.webkit.org/show_bug.cgi?format=multiple&id=125795 +#ifndef _MSC_VER +template<typename T, typename ...Args> +std::unique_ptr<T> make_unique( Args&& ...args ) +{ + return std::unique_ptr<T>( new T( std::forward<Args>(args)... ) ); +} +#else + using std::make_unique; +#endif +// ==================================================================== + +char cFilesystemUtils::GetDirSeparatorSys() { + // TODO nicer os detection? + #if defined(OS_TYPE_POSIX) + return '/'; + #elif defined(OS_TYPE_WINDOWS) + return '\\'; + #else + #error "Do not know how to compile this for your platform." + #endif +} + +char cFilesystemUtils::GetDirSeparatorInter() { + return '/'; +} + +string cFilesystemUtils::FileInternalToSystem(const std::string &name) { + string ret; + ret.resize(name.size()); + std::replace_copy(name.begin(), name.end(), ret.begin(), + GetDirSeparatorInter() , GetDirSeparatorSys()); + return ret; +} + +string cFilesystemUtils::FileSystemToInternal(const std::string &name) { + string ret; + ret.reserve(name.size()); + std::replace_copy(name.begin(), name.end(), ret.begin(), + GetDirSeparatorSys() , GetDirSeparatorInter()); + return ret; +} + +bool cFilesystemUtils::CreateDirTree(const std::string & dir, bool only_below) { + const bool dbg=false; + //struct stat st; + const char dirchS = cFilesystemUtils::GetDirSeparatorSys(); + const char dirchI = cFilesystemUtils::GetDirSeparatorInter(); + std::istringstream iss(dir); + string partI; // current par is in internal format (though it should not matter since it doesn't contain any slashes). eg "bar" + string sofarS=""; // sofarS - the so far created dir part is in SYSTEM format. eg "foo/bar" + if (dir.size()<1) return false; // illegal name + // dir[0] is valid from here + if ( only_below && ((dir[0]==dirchS) || (dir[0]==dirchI))) return false; // no jumping to top (on any os) + + while (getline(iss,partI,dirchI)) { // get new component eg "bar" into part + if (dbg) cout << '['<<partI<<']' << endl; + sofarS += partI; + if (partI.size()<1) return false; // bad format? + if ((only_below) && (partI=="..")) return false; // trying to go up + + if (dbg) cout << "test ["<<sofarS<<"]"<<endl; + // TODO nicer os detection? + #if defined(OS_TYPE_POSIX) + struct stat st; + bool exists = stat(sofarS.c_str() ,&st) == 0; // * + if (exists) { + if (! S_ISDIR(st.st_mode)) { + // std::cerr << "This exists, but as a file: [" << sofar << "]" << (size_t)st.st_ino << endl; + return false; // exists but is a file nor dir + } + } + #elif defined(OS_TYPE_WINDOWS) + DWORD dwAttrib = GetFileAttributesA(sofarS.c_str()); + bool exists = (dwAttrib != INVALID_FILE_ATTRIBUTES && (dwAttrib & FILE_ATTRIBUTE_DIRECTORY)); + #else + #error "Do not know how to compile this for your platform." + #endif + + if (!exists) { + if (dbg) cout << "mkdir ["<<sofarS<<"]"<<endl; + #if defined(OS_TYPE_POSIX) + bool ok = 0== mkdir(sofarS.c_str(), 0700); // *** + #elif defined(OS_TYPE_WINDOWS) + bool ok = (bool) CreateDirectoryA(sofarS.c_str(), NULL); // TODO use -W() after conversion to unicode UTF16 + #else + #error "Do not know how to compile this for your platform." + #endif + if (!ok) return false; + } + sofarS += dirchS; + } + return true; +} +// ==================================================================== + +namespace nDetail { + +struct channel_use_info { ///< feedback information about using (e.g. opening) given debug channel - used internally by logging system +/// TODO not yet used in code +/// e.g. used to write into channel net/in/all that given message was a first logged message from never-before-logged thread or PID etc + bool m_was_interesting; ///< anything interesting happened when using the channel? + std::vector<std::string> m_extra_msg; ///< any additional messages about this channel use +}; + +cDebugScopeGuard::cDebugScopeGuard() : mLevel(-1) { +} + +cDebugScopeGuard::~cDebugScopeGuard() { + if (mLevel != -1) { + gCurrentLogger.write_stream(mLevel,mChan) << mMsg << " ... end" << gCurrentLogger.endline() << std::flush; + } +} + +void cDebugScopeGuard::Assign(const string &chan, const int level, const string &msg) { + mChan=chan; + mLevel=level; + mMsg=msg; +} + +} // namespace nDetail + +// ==================================================================== + +cLogger::cLogger() : +mStream(NULL), +mStreamBrokenDebug(NULL), +mIsBroken(true), // before constructor finishes +mLevel(_debug_level_nr_warn), +mThread2Number_Biggest(0), // the CURRENT biggest value (no thread yet in map) +mPid2Number_Biggest(0) +{ + mStream = & std::cout; + mStreamBrokenDebug = & std::cerr; // the backup stream + *mStreamBrokenDebug << "Creating the logger system" << endl; + mIsBroken=false; // ok, constr. succeeded, so string is not broken now + + // this is here, because it could be using logging itself to log creation of first thread/PID etc + Thread2Number( std::this_thread::get_id() ); // convert current id to short number, useful to reserve a number so that main thread is usually called 1 + Pid2Number( getpid() ); // add this proces ID as first one +} + +cLogger::~cLogger() { + for (auto pair : mChannels) { + std::ofstream *ptr = pair.second; + delete ptr; + pair.second=NULL; + } +} + +void cLogger::SetStreamBroken() { + SetStreamBroken("(no additional details about this problem)"); +} + +void cLogger::SetStreamBroken(const std::string &msg) { + _dbg_dbg("Stream is broken (msg: " << msg << ")"); + if (!mIsBroken) { // if not already marked as broken + _dbg_dbg("(It was not broken before)"); + std::cerr << OT_CODE_STAMP << "WARNING: due to a problem in the debug/logging system itself ("<<msg<<") - we are switching back to fallback stream (e.g. cerr)" << std::endl; + if (mStreamBrokenDebug == nullptr) { + std::cerr << OT_CODE_STAMP << " ERROR: in addition, while reporting this problem, mStreamBrokenDebug stream is NULL: " << mStreamBrokenDebug << std::endl; + } else { + (*mStreamBrokenDebug) << OT_CODE_STAMP << "WARNING: due to debug stream problem ("<<msg<<") - switching back to fallback stream (e.g. cerr)" << std::endl; + } + mIsBroken = true; + } +} + +std::ostream & cLogger::write_stream(int level) { + return write_stream(level,""); +} + +std::ostream & cLogger::write_stream(int level, const std::string & channel ) { + _dbg_dbg("level="<<level<<" channel="<<channel); + if (level >= mLevel) { + if (mStream) { // TODO now disabling mStream also disables writting to any channel + _dbg_dbg("Selecting output..."); + ostream & output = SelectOutput(level,channel); + _dbg_dbg("Selecting output... done, output=" << (void*)(&output)); + #if defined(OS_TYPE_WINDOWS) + output << windows_stream(level); + #endif + output << icon(level) << ' '; + std::thread::id this_id = std::this_thread::get_id(); + output << "{" << Thread2Number(this_id) << "}"; + auto nicePid = Pid2Number(getpid()); + if (nicePid>0) output << " {p" << nicePid << "}"; + output << ' '; + return output; // <--- return + } else _dbg_dbg("Not writting: No mStream"); + } else _dbg_dbg("Not writting: Too low level level="<<level<<" not >= mLevel="<<mLevel); + return g_nullstream; +} + +std::string cLogger::GetLogBaseDir() const { + return "log"; +} + +void cLogger::OpenNewChannel(const std::string & channel) noexcept { + try { + _dbg_dbg("Openning channel for channel="<<channel); + OpenNewChannel_(channel); + } + catch (const std::exception &except) { + SetStreamBroken(OT_CODE_STAMP + " Got exception when opening debug channel: " + ToStr(except.what())); + } + catch (...) { + SetStreamBroken(OT_CODE_STAMP + " Got not-standard exception when opening debug channel."); + } +} + +void cLogger::OpenNewChannel_(const std::string & channel) { // channel=="net/sleep" + _dbg_dbg("Openning channel for channel="<<channel); + size_t last_split = channel.find_last_of(cFilesystemUtils::GetDirSeparatorInter()); + + string fname_system; // the full file name in system format + + if (last_split==string::npos) { // The channel name has no directory, eg channel=="test" + string dir = GetLogBaseDir(); + string basefile = channel + ".log"; + string fname = dir + cFilesystemUtils::GetDirSeparatorInter() + basefile; + fname_system = cFilesystemUtils::FileInternalToSystem(fname); // <- + } + else { // there is a directory eg channel=="net/sleep" + // net/sleep + // ^----- last_split + string dir = GetLogBaseDir() + cFilesystemUtils::GetDirSeparatorInter() + channel.substr(0, last_split); + string basefile = channel.substr(last_split+1) + ".log"; + string fname = dir + cFilesystemUtils::GetDirSeparatorInter() + basefile; + fname_system = cFilesystemUtils::FileInternalToSystem(fname); // <- + bool dirok = cFilesystemUtils::CreateDirTree(dir); + if (!dirok) { string err = "In logger failed to open directory (" + dir +") for channel (" + channel +")"; throw std::runtime_error(err); } + } + + _dbg_dbg("Openning fname_system="<<fname_system); + std::ofstream * thefile = new std::ofstream( fname_system.c_str() ); // file system + *thefile << "====== Log opened: " << fname_system << " (in " << ((void*)thefile) << ") ======" << endl; +// cerr << "====== Log opened: " << fname_system << " (in " << ((void*)thefile) << ") ======" << endl; + _dbg_dbg( "====== Log opened: " << fname_system << " (in " << ((void*)thefile) << ") ======" ); + mChannels.insert( std::pair<string,std::ofstream*>(channel , thefile ) ); // <- created the channel mapping +} + +std::ostream & cLogger::SelectOutput(int level, const std::string & channel) noexcept { + try { + if (mIsBroken) { + _dbg_dbg("The stream is broken mIsBroken="<<mIsBroken<<" so will return backup stream"); + return *mStreamBrokenDebug; + } + if (channel=="") { + _dbg_dbg("No channel given (channel="<<channel<<") so will return main stream"); + return *mStream; + } + + auto obj = mChannels.find(channel); + if (obj == mChannels.end()) { // not found - need to make new channel + _dbg_dbg("No stream openened for channel="<<channel<<" so will create it now"); + OpenNewChannel(channel); // <- create channel + obj = mChannels.find(channel); // find again + if (obj == mChannels.end()) { // still not found! something is wrong + SetStreamBroken( OT_CODE_STAMP + " WARNING: can not get stream for channel="+ToStr(channel)+" level="+ToStr(channel) ); + return *mStreamBrokenDebug; + } + } + auto the_stream_ptr = obj->second; + _dbg_dbg("Found the stream file for channel="<<channel<<" as the_stream_ptr="<<the_stream_ptr); + ASRT(the_stream_ptr); + return *the_stream_ptr; // <--- RETURN + } + catch (std::exception &except) { + SetStreamBroken( OT_CODE_STAMP + " Got exception: " + ToStr(except.what()) ); + _dbg_dbg("Exception! Returning broken stream"); + return *mStreamBrokenDebug; + } + catch (...) { + SetStreamBroken( OT_CODE_STAMP + " Got not-standard exception."); + _dbg_dbg("Exception! Returning broken stream"); + return *mStreamBrokenDebug; + } + + // dead code +} + +void cLogger::setOutStreamFile(const string &fname) { // switch to using this file + _mark("WILL SWITCH DEBUG NOW to file: " << fname); + mOutfile = make_unique<std::ofstream>(fname); + mStream = & (*mOutfile); + _mark("Started new debug, to file: " << fname); +} + +void cLogger::setOutStreamFromGlobalOptions() { + if ( gRunOptions.getDebug() ) { + if ( gRunOptions.getDebugSendToFile() ) { + mOutfile = make_unique<std::ofstream> ("debuglog.txt"); + mStream = & (*mOutfile); + } + else if ( gRunOptions.getDebugSendToCerr() ) { + mStream = & std::cerr; + } + else { + mStream = & g_nullstream; + } + } + else { + mStream = & g_nullstream; + } +} + +void cLogger::setDebugLevel(int level) { + bool note_before = (mLevel > level); // report the level change before or after the change? (on higher level) + if (note_before) _note("Setting debug level to "<<level); + mLevel = level; + if (!note_before) _note("Setting debug level to "<<level); +} + +std::string cLogger::icon(int level) const { + // TODO replan to avoid needles converting back and forth char*, string etc + + using namespace zkr; + #if defined(OS_TYPE_POSIX) + if (level >= 100) return cc::back::lightred + ToStr(cc::fore::lightyellow) + ToStr("ERROR ") + ToStr(cc::fore::lightyellow) + " " ; + if (level >= 90) return cc::back::lightyellow + ToStr(cc::fore::black) + ToStr("Warn ") + ToStr(cc::fore::red)+ " " ; + if (level >= 80) return cc::back::lightmagenta + ToStr(cc::fore::black) + ToStr("MARK "); //+ zkr::cc::console + ToStr(cc::fore::lightmagenta)+ " "; + if (level >= 75) return cc::back::lightyellow + ToStr(cc::fore::black) + ToStr("FACT ") + zkr::cc::console + ToStr(cc::fore::lightyellow)+ " "; + if (level >= 70) return cc::fore::green + ToStr("Note "); + if (level >= 50) return cc::fore::cyan + ToStr("info "); + if (level >= 40) return cc::fore::lightwhite + ToStr("dbg "); + if (level >= 30) return cc::fore::lightblue + ToStr("dbg "); + if (level >= 20) return cc::fore::blue + ToStr("dbg "); + + #elif defined(OS_TYPE_WINDOWS) + if (level >= 100) return ToStr("ERROR "); + if (level >= 90) return ToStr("Warn "); + if (level >= 80) return ToStr("MARK "); + if (level >= 75) return ToStr("FACT "); + if (level >= 70) return ToStr("Note "); + if (level >= 50) return ToStr("info "); + if (level >= 40) return ToStr("dbg "); + if (level >= 30) return ToStr("dbg "); + if (level >= 20) return ToStr("dbg "); + #endif + + return " "; +} + +std::string cLogger::endline() const { + #if defined(OS_TYPE_POSIX) + return ToStr("") + zkr::cc::console + ToStr("\n"); // TODO replan to avoid needles converting back and forth char*, string etc + #elif defined(OS_TYPE_WINDOWS) + return ToStr("\n"); + #endif +} + +int cLogger::Thread2Number(const std::thread::id id) { + auto found = mThread2Number.find( id ); + if (found == mThread2Number.end()) { // new one + mThread2Number_Biggest++; + mThread2Number[id] = mThread2Number_Biggest; + _info_c("dbg/main", "This is a new thread (used in debug), thread id="<<id); // can cause some recursion + return mThread2Number_Biggest; + } else { + return mThread2Number[id]; + } +} + +int cLogger::Pid2Number(const t_anypid id) { + auto found = mPid2Number.find( id ); + if (found == mPid2Number.end()) { // new one + mPid2Number_Biggest++; + mPid2Number[id] = mPid2Number_Biggest; + _info_c("dbg/main", "This is a new process (used in debug), process pid="<<id); // can cause some recursion + return mPid2Number_Biggest; + } else { + return mPid2Number[id]; + } +} + +// ==================================================================== +// object gCurrentLogger is defined later - in global namespace below + + +// ==================================================================== +// vector debug + +void DisplayStringEndl(std::ostream & out, const std::string text) { + out << text; + out << std::endl; +} + +std::string SpaceFromEscape(const std::string &s) { + std::ostringstream newStr; + for(size_t i = 0; i < s.length();i++) { + if(s[i] == '\\' && s[i+1] ==32) + newStr<<""; + else + newStr<<s[i]; + } + return newStr.str(); +} + +std::string EscapeFromSpace(const std::string &s) { + std::ostringstream newStr; + for(size_t i = 0; i < s.length();i++) { + if(s[i] == 32) + newStr << "\\" << " "; + else + newStr << s[i]; + } + return newStr.str(); +} + + +std::string EscapeString(const std::string &s) { + std::ostringstream newStr; + for(size_t i = 0; i < s.length();i++) { + if(s[i] >=32 && s[i] <= 126) + newStr<<s[i]; + else + newStr<<"\\"<< (int) s[i]; + } + + return newStr.str(); +} + + +bool CheckIfBegins(const std::string & beggining, const std::string & all) { + if (all.compare(0, beggining.length(), beggining) == 0) { + return 1; + } + else { + return 0; + } +} + +bool CheckIfEnds (std::string const & ending, std::string const & all){ + if (all.length() >= ending.length()) { + return (0 == all.compare (all.length() - ending.length(), ending.length(), ending)); + } else { + return false; + } +} + + +vector<string> WordsThatMatch(const std::string & sofar, const vector<string> & possib) { + vector<string> ret; + for ( auto rec : possib) { // check of possibilities + if (CheckIfBegins(sofar,rec)) { + rec = EscapeFromSpace(rec); + ret.push_back(rec); // this record matches + } + } + return ret; +} + +char GetLastChar(const std::string & str) { // TODO unicode? + auto s = str.length(); + if (s==0) throw std::runtime_error("Getting last character of empty string (" + ToStr(s) + ")" + OT_CODE_STAMP); + return str.at( s - 1); +} + +std::string GetLastCharIf(const std::string & str) { // TODO unicode? + auto s = str.length(); + if (s==0) return ""; // empty string signalizes ther is nothing to be returned + return std::string( 1 , str.at( s - 1) ); +} + +// ==================================================================== + +// ASRT - assert. Name like ASSERT() was too long, and ASS() was just... no. +// Use it like this: ASRT( x>y ); with the semicolon at end, a clever trick forces this syntax :) + +void Assert(bool result, const std::string &stamp, const std::string &condition) { + if (!result) { + _erro("Assert failed at "+stamp+": ASSERT( " << condition << ")"); + throw std::runtime_error("Assert failed at "+stamp+": ASSERT( " + condition + ")"); + } +} + +// ==================================================================== +// advanced string + +const std::string GetMultiline(string endLine) { + std::string result(""); // Taken from OT_CLI_ReadUntilEOF + while (true) { + std::string input_line(""); + if (std::getline(std::cin, input_line, '\n')) + { + input_line += "\n"; + if (input_line[0] == '~') + break; + result += input_line; + } + if (std::cin.eof() ) + { + std::cin.clear(); + break; + } + if (std::cin.fail() ) + { + std::cin.clear(); + break; + } + if (std::cin.bad()) + { + std::cin.clear(); + break; + } + } + return result; +} + +vector<string> SplitString(const string & str){ + std::istringstream iss(str); + vector<string> vec { std::istream_iterator<string>{iss}, std::istream_iterator<string>{} }; + return vec; +} + +bool checkPrefix(const string & str, char prefix) { + if (str.at(0) == prefix) + return true; + return false; +} + +// ==================================================================== +// operation on files + + +#ifdef __unix + +void cEnvUtils::GetTmpTextFile() { + // TODO make this name configurable (depending on project) + char filename[] = "/tmp/otshellutils_text.XXXXXX"; + fd = mkstemp(filename); + if (fd == -1) { + _erro("Can't create the file: " << filename); + return; + } + mFilename = filename; +} + +void cEnvUtils::CloseFile() { + close(fd); + unlink( mFilename.c_str() ); +} + +void cEnvUtils::OpenEditor() { + char* editor = std::getenv("OT_EDITOR"); //TODO Read editor from configuration file + if (editor == NULL) + editor = std::getenv("VISUAL"); + if (editor == NULL) + editor = std::getenv("EDITOR"); + + string command; + if (editor != NULL) + command = ToStr(editor) + " " + mFilename; + else + command = "/usr/bin/editor " + mFilename; + _dbg3("Opening editor with command: " << command); + if ( system( command.c_str() ) == -1 ) + _erro("Cannot execute system command: " << command); +} + +const string cEnvUtils::ReadFromTmpFile() { + std::ifstream ifs(mFilename); + string msg((std::istreambuf_iterator<char>(ifs)), std::istreambuf_iterator<char>()); + return msg; +} + +const string cEnvUtils::Compose() { + GetTmpTextFile(); + OpenEditor(); + string input = ReadFromTmpFile(); + CloseFile(); + return input; +} + +#endif + +const string cEnvUtils::ReadFromFile(const string path) { + std::ifstream ifs(path); + string msg((std::istreambuf_iterator<char>(ifs)), std::istreambuf_iterator<char>()); + return msg; +} + +void hintingToTxt(std::fstream & file, string command, vector<string> &commands) { + if(file.good()) { + file<<command<<"~"<<endl; + for (auto a: commands) { + file <<a<< " "; + file.flush(); + } + file<<endl; + } +} + +string stringToColor(const string &hash) { + // Generete vector with all possible light colors + vector <string> lightColors; + using namespace zkr; + lightColors.push_back(cc::fore::lightblue); + lightColors.push_back(cc::fore::lightred); + lightColors.push_back(cc::fore::lightmagenta); + lightColors.push_back(cc::fore::lightgreen); + lightColors.push_back(cc::fore::lightcyan); + lightColors.push_back(cc::fore::lightyellow); + lightColors.push_back(cc::fore::lightwhite); + + int sum=0; + + for (auto ch : hash) sum+=ch; + auto color = sum%(lightColors.size()-1); + + return lightColors.at( color ); +} + + +// ==================================================================== +// algorthms + + +} // namespace nUtil + + +} // namespace OT + +// global namespace + +const extern int _dbg_ignore = 0; // see description in .hpp + +std::string GetObjectName() { + //static std::string * name=nullptr; + //if (!name) name = new std::string("(global)"); + return ""; +} + +// ==================================================================== + +nOT::nUtils::cLogger gCurrentLogger; + diff --git a/contrib/otshell_utils/utils.hpp b/contrib/otshell_utils/utils.hpp new file mode 100644 index 000000000..297f13ac0 --- /dev/null +++ b/contrib/otshell_utils/utils.hpp @@ -0,0 +1,532 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief various general utils taken from (and relate to) otshell project, including loggiang/debug + +/* See other files here for the LICENCE that applies here. */ + +#include "ccolor.hpp" +#ifndef INCLUDE_OT_NEWCLI_UTILS +#define INCLUDE_OT_NEWCLI_UTILS + +#include "lib_common1.hpp" +#ifdef __unix + #include <unistd.h> +#endif + +#if defined(_WIN32) + #include"windows_stream.h" +#endif + +#ifndef CFG_WITH_TERMCOLORS + //#error "You requested to turn off terminal colors (CFG_WITH_TERMCOLORS), however currently they are hardcoded (this option to turn them off is not yet implemented)." +#endif + +///Macros related to automatic deduction of class name etc; +#define MAKE_CLASS_NAME(NAME) private: static std::string GetObjectName() { return #NAME; } +#define MAKE_STRUCT_NAME(NAME) private: static std::string GetObjectName() { return #NAME; } public: + +// define this to debug the debug system itself: +// #define opt_debug_debug + +#ifdef opt_debug_debug + #define _dbg_dbg(X) do { std::cerr<<"_dbg_dbg: " << OT_CODE_STAMP << " {thread=" << std::this_thread::get_id()<<"} " \ + << " {pid="<<getpid()<<"} " << ": " << X << std::endl; } while(0) +#else + #define _dbg_dbg(X) do { } while(0) +#endif + +namespace nOT { + +namespace nUtils { + +/// @brief general based for my runtime errors +class myexception : public std::runtime_error { + public: + myexception(const char * what); + myexception(const std::string &what); + //virtual ~myexception(); + virtual void Report() const; +}; + +/// @macro Use this macro INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 as a shortcut for various using std::string etc. +INJECT_OT_COMMON_USING_NAMESPACE_COMMON_1 // <=== namespaces + +// ====================================================================================== +/// text trimming functions (they do mutate the passes string); they trim based on std::isspace. also return it's reference again +/// http://stackoverflow.com/questions/216823/whats-the-best-way-to-trim-stdstring +std::string & trim(std::string &s); ///< trim text http://stackoverflow.com/questions/216823/whats-the-best-way-to-trim-stdstring +std::string & ltrim(std::string &s); ///< left trim +std::string & rtrim(std::string &s); ///< right trim + +// ====================================================================================== + +std::string get_current_time(); + +// string conversions +template <class T> +std::string ToStr(const T & obj) { + std::ostringstream oss; + oss << obj; + return oss.str(); +} + +struct cNullstream : std::ostream { + cNullstream() : std::ios(0), std::ostream(0) {} +}; +extern cNullstream g_nullstream; // a stream that does nothing (eats/discards data) +// ========== debug ========== +// _dbg_ignore is moved to global namespace (on purpose) + +// TODO make _dbg_ignore thread-safe everywhere + +extern std::recursive_mutex gLoggerGuard; // the mutex guarding logging/debugging code e.g. protecting streams, files, etc + +std::atomic<int> & gLoggerGuardDepth_Get(); // getter for the global singleton of counter (it guarantees initializing it to 0). This counter shows the current recursion (re-entrant) level of debug macros. + +// TODO more debug of the debug system: +// detect lock() error e.g. recursive limit +// detect stream e.g. operator<< error + +#define _debug_level(LEVEL,VAR) do { if (_dbg_ignore< LEVEL) { \ + _dbg_dbg("WRITE DEBUG: LEVEL="<<LEVEL<<" VAR: " << VAR ); \ + auto level=LEVEL; short int part=0; \ + try { \ + std::lock_guard<std::recursive_mutex> mutex_guard( nOT::nUtils::gLoggerGuard ); \ + part=1; \ + try { \ + ++nOT::nUtils::gLoggerGuardDepth_Get(); \ +/* int counter = nOT::nUtils::gLoggerGuardDepth_Get(); if (counter!=1) gCurrentLogger.write_stream(100,"")<<"DEBUG-ERROR: recursion, counter="<<counter<<gCurrentLogger.endline(); */ \ + gCurrentLogger.write_stream(LEVEL,"") << nOT::nUtils::get_current_time() << ' ' << OT_CODE_STAMP << ' ' << VAR << gCurrentLogger.endline() << std::flush; \ + part=9; \ + } catch(...) { \ + gCurrentLogger.write_stream(std::max(level,90),"") << nOT::nUtils::get_current_time() << ' ' << OT_CODE_STAMP << ' ' << "(ERROR IN DEBUG)" << gCurrentLogger.endline(); \ + --nOT::nUtils::gLoggerGuardDepth_Get(); throw ; \ + } \ + --nOT::nUtils::gLoggerGuardDepth_Get(); \ + } catch(...) { if (part<8) gCurrentLogger.write_stream(100,"")<<"DEBUG-ERROR: problem in debug mechanism e.g. in locking." <<gCurrentLogger.endline(); throw ; } \ + } } while(0) + +// info for code below: oss object is normal stack variable, using it does not need lock protection +#define _debug_level_c(CHANNEL,LEVEL,VAR) do { if (_dbg_ignore< LEVEL) { \ + _dbg_dbg("WRITE DEBUG: LEVEL="<<LEVEL<<" CHANNEL="<<CHANNEL<<" VAR: " << VAR ); \ + auto level=LEVEL; short int part=0; \ + try { \ + std::lock_guard<std::recursive_mutex> mutex_guard( nOT::nUtils::gLoggerGuard ); \ + part=1; \ + try { \ + ++nOT::nUtils::gLoggerGuardDepth_Get(); \ + std::ostringstream oss; \ + oss << nOT::nUtils::get_current_time() << ' ' << OT_CODE_STAMP << ' ' << VAR << gCurrentLogger.endline() << std::flush; \ + std::string as_string = oss.str(); \ + _dbg_dbg("START will write to log LEVEL="<<LEVEL<<" to CHANNEL="<<CHANNEL<<" as_string="<<as_string); \ +/* int counter = nOT::nUtils::gLoggerGuardDepth_Get(); if (counter!=1) gCurrentLogger.write_stream(100,"")<<"DEBUG-ERROR: recursion, counter="<<counter<<gCurrentLogger.endline(); */ \ + gCurrentLogger.write_stream(LEVEL,"" ) << as_string << gCurrentLogger.endline() << std::flush; \ + gCurrentLogger.write_stream(LEVEL,CHANNEL) << as_string << gCurrentLogger.endline() << std::flush; \ + _dbg_dbg("DONE will write to log LEVEL="<<LEVEL<<" to CHANNEL="<<CHANNEL<<" as_string="<<as_string); \ + part=9; \ + } catch(...) { \ + gCurrentLogger.write_stream(std::max(level,90),CHANNEL) << nOT::nUtils::get_current_time() << ' ' << OT_CODE_STAMP << ' ' << "(ERROR IN DEBUG)" << gCurrentLogger.endline(); \ + --nOT::nUtils::gLoggerGuardDepth_Get(); throw ; \ + } \ + --nOT::nUtils::gLoggerGuardDepth_Get(); \ + } catch(...) { if (part<8) gCurrentLogger.write_stream(100,CHANNEL)<<"DEBUG-ERROR: problem in debug mechanism e.g. in locking." <<gCurrentLogger.endline(); throw ; } \ + } } while(0) + +// Numerical values of the debug levels - are defined here as const ints. Full name (with namespace) given for clarity. +extern const int _debug_level_nr_dbg3; +extern const int _debug_level_nr_dbg2; +extern const int _debug_level_nr_dbg1; +extern const int _debug_level_nr_info; +extern const int _debug_level_nr_note; +extern const int _debug_level_nr_fact; +extern const int _debug_level_nr_mark; +extern const int _debug_level_nr_warn; +extern const int _debug_level_nr_erro; + +#define _dbg3(VAR) _debug_level( nOT::nUtils::_debug_level_nr_dbg3,VAR) // details - most detailed +#define _dbg2(VAR) _debug_level( nOT::nUtils::_debug_level_nr_dbg2,VAR) // details - a bit more important +#define _dbg1(VAR) _debug_level( nOT::nUtils::_debug_level_nr_dbg1,VAR) // details - more important +#define _info(VAR) _debug_level( nOT::nUtils::_debug_level_nr_info,VAR) // information +#define _note(VAR) _debug_level( nOT::nUtils::_debug_level_nr_note,VAR) // more interesting information +#define _fact(VAR) _debug_level( nOT::nUtils::_debug_level_nr_fact,VAR) // interesting events that could be interesting even for user, for logical/business things +#define _mark(VAR) _debug_level( nOT::nUtils::_debug_level_nr_mark,VAR) // marked actions +#define _warn(VAR) _debug_level( nOT::nUtils::_debug_level_nr_warn,VAR) // some problems +#define _erro(VAR) _debug_level( nOT::nUtils::_debug_level_nr_erro,VAR) // errors + +#define _dbg3_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_dbg3, VAR) // details - most detailed +#define _dbg2_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_dbg2, VAR) // details - a bit more important +#define _dbg1_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_dbg1, VAR) // details - more important +#define _info_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_info, VAR) // information +#define _note_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_note, VAR) // more interesting information +#define _fact_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_fact, VAR) // interesting events that could be interesting even for user, for logical/business things +#define _mark_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_mark, VAR) // marked actions +#define _warn_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_warn, VAR) // some problems +#define _erro_c(C,VAR) _debug_level_c(C, nOT::nUtils::_debug_level_nr_erro, VAR) // errors + +// lock // because of VAR +#define _scope_debug_level_c(CHANNEL,LEVEL,VAR) \ + std::ostringstream debug_detail_oss; \ + nOT::nUtils::gLoggerGuard.lock(); \ + debug_detail_oss << OT_CODE_STAMP << ' ' << VAR ; \ + nOT::nUtils::nDetail::cDebugScopeGuard debugScopeGuard; \ + if (_dbg_ignore<LEVEL) debugScopeGuard.Assign(CHANNEL,LEVEL, debug_detail_oss.str()); \ + if (_dbg_ignore<LEVEL) _debug_level_c(CHANNEL,LEVEL,debug_detail_oss.str() + " ... begin"); \ + nOT::nUtils::gLoggerGuard.unlock(); +#define _scope_debug_level(LEVEL,VAR) _scope_debug_level_c("",LEVEL,VAR) + +#define _scope_dbg1(VAR) _scope_debug_level( _debug_level_nr_dbg3, VAR) +#define _scope_dbg2(VAR) _scope_debug_level( _debug_level_nr_dbg2, VAR) +#define _scope_dbg3(VAR) _scope_debug_level( _debug_level_nr_dbg1, VAR) +#define _scope_info(VAR) _scope_debug_level( _debug_level_nr_info, VAR) +#define _scope_note(VAR) _scope_debug_level( _debug_level_nr_note, VAR) +#define _scope_fact(VAR) _scope_debug_level( _debug_level_nr_fact, VAR) +#define _scope_mark(VAR) _scope_debug_level( _debug_level_nr_mark, VAR) +#define _scope_warn(VAR) _scope_debug_level( _debug_level_nr_warn, VAR) +#define _scope_erro(VAR) _scope_debug_level( _debug_level_nr_erro, VAR) + +/*** +@brief do not use this namespace directly, it is implementation detail. +*/ +namespace nDetail { + +/*** +@brief a Debug scope-guard, to log a debug message when current scope is left. Do NOT use this directly, +only use it via the macros like _scope_dbg1 etc. +*/ +class cDebugScopeGuard { + protected: + string mMsg; + int mLevel; + string mChan; + public: + cDebugScopeGuard(); + ~cDebugScopeGuard(); + void Assign(const string &chan, const int level, const string &msg); +}; + +const char* DbgShortenCodeFileName(const char *s); ///< Returns a pointer to some part of the string that was given, skipping directory names, for log/debug + +} // namespace nDetail + +// ========== logger ========== + +namespace nDetail { + struct channel_use_info; +} // namespace nDetail + +/*** +@brief Class to write debug into. Used it by calling the debug macros _dbg1(...) _info(...) _erro(...) etc, NOT directly! +@author rfree (maintainer) +@thread this class is NOT thread safe and must used only by one thread at once (use it via ot_debug_macros like _info macro they do proper locking) +*/ +class cLogger { + public: + cLogger(); + ~cLogger(); + std::ostream & write_stream(int level); ///< starts a new message on given level (e.g. writes out the icon/tag) and returns stream to output to + std::ostream & write_stream(int level, const std::string & channel); ///< the same but with name of the debug channel + + void setOutStreamFromGlobalOptions(); // set debug level, file etc - according to global Options + void setOutStreamFile(const std::string &fname); // switch to using this file + void setDebugLevel(int level); // change the debug level e.g. to mute debug from now + + std::string icon(int level) const; ///< returns "icon" for given debug level. It is text, might include color controll characters + std::string endline() const; ///< returns string to be written at end of message + + protected: + typedef long int t_anypid; // a portable representation of PID. long int should cover all platforms + + void SetStreamBroken(); ///< call in case of internal error in logger (e.g. can not open a file) + void SetStreamBroken(const std::string &msg); ///< same but with error message + + unique_ptr<std::ofstream> mOutfile; + std::ostream * mStream; ///< pointing only! can point to our own mOutfile, or maye to global null stream + std::ostream * mStreamBrokenDebug; ///< pointing only! this is a pointer to some stream that should be used when normal debugging is broken eg std::cerr + bool mIsBroken; ///< is the debugging system broken (this should be set when internal problems occur and should cause fallback to std::cerr) + + std::map< std::string , std::ofstream * > mChannels; // the ofstream objects are owned by this class + + int mLevel; ///< current debug level + + std::ostream & SelectOutput(int level, const std::string & channel) noexcept; ///< returns a proper stream for this level and channel (always usable string) + void OpenNewChannel(const std::string & channel) noexcept; ///< tries to prepare this channel. does NOT guarantee to created mChannels[] entry! + void OpenNewChannel_(const std::string & channel); ///< internal function, will throw in case of problems + std::string GetLogBaseDir() const; + + std::map< std::thread::id , int > mThread2Number; ///< change long thread IDs into a short nice number to show + int mThread2Number_Biggest; ///< current biggest value held there (biggest key) - works as growing-only counter basically + int Thread2Number(const std::thread::id id); ///< convert the system's thread id into a nice short our id; make one if new thread + + std::map< t_anypid , int > mPid2Number; ///< change long proces PID into a short nice number to show + int mPid2Number_Biggest; ///< current biggest value held there (biggest key) - works as growing-only counter basically + int Pid2Number(const t_anypid id); ///< convert the system's PID id into a nice short our id; make one if new thread +}; + + + +// ==================================================================== +// vector debug + +template <class T> +std::string vectorToStr(const T & v) { + std::ostringstream oss; + for(auto rec: v) { + oss << rec <<","; + } + return oss.str(); +} + +template <class T> +void DisplayVector(std::ostream & out, const std::vector<T> &v, const std::string &delim=" ") { + std::copy( v.begin(), v.end(), std::ostream_iterator<T>(out, delim.c_str()) ); +} + +template <class T> +void EndlDisplayVector(std::ostream & out, const std::vector<T> &v, const std::string &delim=" ") { + out << std::endl; + DisplayVector(out,v,delim); +} + +template <class T> +void DisplayVectorEndl(std::ostream & out, const std::vector<T> &v, const std::string &delim=" ") { + DisplayVector(out,v,delim); + out << std::endl; +} + +template <class T> +void DbgDisplayVector(const std::vector<T> &v, const std::string &delim=" ") { + std::cerr << "["; + std::copy( v.begin(), v.end(), std::ostream_iterator<T>(std::cerr, delim.c_str()) ); + std::cerr << "]"; +} + +string stringToColor(const string &hash); +template <class T, class T2> +void DisplayMap(std::ostream & out, const std::map<T, T2> &m, const std::string &delim=" ") { + auto *no_color = zkr::cc::fore::console; + for(auto var : m) { + out << stringToColor(var.first) << var.first << delim << var.second << no_color << endl; + } + +} + +template <class T, class T2> +void EndlDisplayMap(std::ostream & out, const std::map<T, T2> &m, const std::string &delim=" ") { + out << endl; + for(auto var : m) { + out << var.first << delim << var.second << endl; + } +} + +template <class T, class T2> +void DbgDisplayMap(const std::map<T, T2> &m, const std::string &delim=" ") { + for(auto var : m) { + std::cerr << var.first << delim << var.second << endl; + } +} + + +template <class T> +void DbgDisplayVectorEndl(const std::vector<T> &v, const std::string &delim=" ") { + DbgDisplayVector(v,delim); + std::cerr << std::endl; +} + +void DisplayStringEndl(std::ostream & out, const std::string text); + +bool CheckIfBegins(const std::string & beggining, const std::string & all); +bool CheckIfEnds (std::string const & ending, std::string const & all); +std::string SpaceFromEscape(const std::string &s); +std::string EscapeFromSpace(const std::string &s); +vector<string> WordsThatMatch(const std::string & sofar, const vector<string> & possib); +char GetLastChar(const std::string & str); +std::string GetLastCharIf(const std::string & str); // TODO unicode? +std::string EscapeString(const std::string &s); + + +template <class T> +std::string DbgVector(const std::vector<T> &v, const std::string &delim="|") { + std::ostringstream oss; + oss << "["; + bool first=true; + for(auto vElement : v) { if (!first) oss<<delim; first=false; oss <<vElement ; } + oss << "]"; + //std::copy( v.begin(), v.end(), std::ostream_iterator<T>(oss, delim.c_str()) ); + return oss.str(); +} + +template <class T> +std::ostream & operator<<(std::ostream & os, const map< T, vector<T> > & obj){ + os << "["; + for(auto const & elem : obj) { + os << " [" << elem.first << "=" << DbgVector(elem.second) << "] "; + } + os << "]"; + return os; +} + +template <class T, class T2> +std::string DbgMap(const map<T, T2> & map) { + std::ostringstream oss; + oss << map; + return oss.str(); +} + +// ==================================================================== +// assert + +// ASRT - assert. Name like ASSERT() was too long, and ASS() was just... no. +// Use it like this: ASRT( x>y ); with the semicolon at end, a clever trick forces this syntax :) +#define ASRT(x) do { if (!(x)) nOT::nUtils::Assert(false, OT_CODE_STAMP, #x); } while(0) + +void Assert(bool result, const std::string &stamp, const std::string &condition); + +// ==================================================================== +// advanced string + +const std::string GetMultiline(string endLine = "~"); +vector<string> SplitString(const string & str); + +bool checkPrefix(const string & str, char prefix = '^'); + +// ==================================================================== +// nUse utils + +enum class eSubjectType {Account, Asset, User, Server, Unknown}; + +string SubjectType2String(const eSubjectType & type); +eSubjectType String2SubjectType(const string & type); + +// ==================================================================== +// operation on files + +/// @brief tools related to filesystem +/// @author rfree (maintainer) +class cFilesystemUtils { // if we do not want to use boost in given project (or we could optionally write boost here later) + public: + static bool CreateDirTree(const std::string & dir, bool only_below=false); + static char GetDirSeparatorSys(); /// < eg '/' or '\' + static char GetDirSeparatorInter(); /// < internal is '/' + static string FileInternalToSystem(const std::string &name); ///< converts from internal file name string to system file name string + static string FileSystemToInternal(const std::string &name); ///< converts from system file name string to internal file name string +}; + + +/// @brief utils to e.g. edit a file from console +/// @author rfree (maintainer) +class cEnvUtils { + int fd; + string mFilename; + + void GetTmpTextFile(); + void CloseFile(); + void OpenEditor(); + const string ReadFromTmpFile(); +public: + const string Compose(); + const string ReadFromFile(const string path); +}; +void hintingToTxt(std::fstream & file, string command, vector<string> &commands); +void generateQuestions (std::fstream & file, string command); +void generateAnswers (std::fstream & file, string command, vector<string> &completions); + +// ==================================================================== + +namespace nOper { // nOT::nUtils::nOper +// cool shortcut operators, like vector + vecotr operator working same as string (appending) +// isolated to namespace because it's unorthodox ide to implement this + +using namespace std; + +// TODO use && and move? +template <class T> +vector<T> operator+(const vector<T> &a, const vector<T> &b) { + vector<T> ret = a; + ret.insert( ret.end() , b.begin(), b.end() ); + return ret; +} + +template <class T> +vector<T> operator+(const T &a, const vector<T> &b) { + vector<T> ret(1,a); + ret.insert( ret.end() , b.begin(), b.end() ); + return ret; +} + +template <class T> +vector<T> operator+(const vector<T> &a, const T &b) { + vector<T> b_vector(1,a); + return a + b_vector; +} + +template <class T> +vector<T>& operator+=(vector<T> &a, const vector<T> &b) { + a.insert( a.end() , b.begin(), b.end() ); + return a; +} + +// map +template <class TK,class TV> +map<TK,TV> operator+(const map<TK,TV> &a, const map<TK,TV> &b) { + map<TK,TV> ret = a; + for (const auto & elem : b) { + ret.insert(elem); + } + return ret; +} + + +} // nOT::nUtils::nOper + +// ==================================================================== + +// ==================================================================== + +// Algorithms + +// ==================================================================== +// ==================================================================== + + +/** +@brief Special type that on creation will be initialized to have value INIT given as template argument. +Might be usefull e.g. to express in the declaration of class what will be the default value of member variable +See also http://www.boost.org/doc/libs/1_56_0/libs/utility/value_init.htm +Probably not needed when using boost in your project. +*/ +template <class T, T INIT> +class value_init { + private: + T data; + public: + value_init(); + + T& operator=(const T& v) { data=v; return *this; } + operator T const &() const { return data; } + operator T&() { return data; } +}; + +template <class T, T INIT> +value_init<T, INIT>::value_init() : data(INIT) { } + +} // namespace nUtils + +} // namespace nOT + + +// global namespace +extern nOT::nUtils::cLogger gCurrentLogger; ///< The current main logger. Usually do not use it directly, instead use macros like _dbg1 etc + +std::string GetObjectName(); ///< Method to return name of current object; To use in debug; Can be shadowed in your classes. (Might be not used currently) + +const extern int _dbg_ignore; ///< the global _dbg_ignore, but local code (blocks, classes etc) you could shadow it in your code blocks, +// to override debug compile-time setting for given block/class, e.g. to disable debug in one of your methods or increase it there. +// Or to make it runtime by providing a class normal member and editing it in runtime + +#define OT_CODE_STAMP ( nOT::nUtils::ToStr("[") + nOT::nUtils::nDetail::DbgShortenCodeFileName(__FILE__) + nOT::nUtils::ToStr("+") + nOT::nUtils::ToStr(__LINE__) + nOT::nUtils::ToStr(" ") + (GetObjectName()) + nOT::nUtils::ToStr("::") + nOT::nUtils::ToStr(__FUNCTION__) + nOT::nUtils::ToStr("]")) + + + + +#endif + diff --git a/contrib/otshell_utils/windows_stream.cpp b/contrib/otshell_utils/windows_stream.cpp new file mode 100644 index 000000000..59d8b12a3 --- /dev/null +++ b/contrib/otshell_utils/windows_stream.cpp @@ -0,0 +1,64 @@ +#if defined(_WIN32)
+#include "windows_stream.h"
+#include <windows.h>
+
+windows_stream::windows_stream(unsigned int pLevel)
+ :
+ mLevel(pLevel)
+{
+}
+
+std::ostream& operator << (std::ostream &stream, windows_stream const& object)
+{
+ HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE);
+
+ if (object.mLevel >= 100)
+ {
+ SetConsoleTextAttribute(h_stdout, FOREGROUND_GREEN | FOREGROUND_RED | FOREGROUND_INTENSITY | BACKGROUND_RED | BACKGROUND_INTENSITY);
+ return stream;
+ }
+ if (object.mLevel >= 90)
+ {
+ SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | FOREGROUND_INTENSITY | BACKGROUND_GREEN | BACKGROUND_RED | BACKGROUND_INTENSITY);
+ return stream;
+ }
+ if (object.mLevel >= 80)
+ {
+ SetConsoleTextAttribute(h_stdout, BACKGROUND_BLUE | BACKGROUND_RED | BACKGROUND_INTENSITY);
+ return stream;
+ }
+ if (object.mLevel >= 75)
+ {
+ SetConsoleTextAttribute(h_stdout, BACKGROUND_RED | BACKGROUND_GREEN | BACKGROUND_INTENSITY);
+ return stream;
+ }
+ if (object.mLevel >= 70)
+ {
+ SetConsoleTextAttribute(h_stdout, FOREGROUND_GREEN | FOREGROUND_INTENSITY);
+ return stream;
+ }
+ if (object.mLevel >= 50)
+ {
+ SetConsoleTextAttribute(h_stdout, FOREGROUND_GREEN | FOREGROUND_BLUE | FOREGROUND_INTENSITY);
+ return stream;
+ }
+ if (object.mLevel >= 40)
+ {
+ SetConsoleTextAttribute(h_stdout, FOREGROUND_GREEN | FOREGROUND_BLUE | FOREGROUND_RED | FOREGROUND_INTENSITY);
+ return stream;
+ }
+ if (object.mLevel >= 30)
+ {
+ SetConsoleTextAttribute(h_stdout, FOREGROUND_BLUE | FOREGROUND_INTENSITY);
+ return stream;
+ }
+ if (object.mLevel >= 20)
+ {
+ SetConsoleTextAttribute(h_stdout, FOREGROUND_BLUE);
+ return stream;
+ }
+
+ return stream;
+}
+
+#endif
diff --git a/contrib/otshell_utils/windows_stream.h b/contrib/otshell_utils/windows_stream.h new file mode 100644 index 000000000..859e7ee50 --- /dev/null +++ b/contrib/otshell_utils/windows_stream.h @@ -0,0 +1,20 @@ +#ifndef WINDOWS_STREAM_H
+#define WINDOWS_STREAM_H
+
+#if defined(_WIN32)
+
+#include <string>
+#include <iostream>
+
+class windows_stream
+{
+public:
+ windows_stream(unsigned int pLevel);
+ friend std::ostream& operator<<(std::ostream &stream, windows_stream const& object);
+private:
+ unsigned int mLevel = 0;
+};
+
+#endif // _WIN32
+
+#endif // WINDOWS_STREAM_H
diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index a7bcfebe0..345aac3f6 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # @@ -35,7 +35,7 @@ # ...except for FreeBSD, because FreeBSD is a special case that doesn't play well with # others. -find_package(MiniUpnpc QUIET) +find_package(Miniupnpc QUIET) # FreeBSD doesn't play well with the local copy, so default to using shared set(USE_SHARED_MINIUPNPC false) @@ -97,3 +97,5 @@ else() die("Found libunbound includes, but could not find libunbound library. Please make sure you have installed libunbound or libunbound-dev or the equivalent") endif() endif() + +add_subdirectory(db_drivers) diff --git a/external/db_drivers/CMakeLists.txt b/external/db_drivers/CMakeLists.txt new file mode 100644 index 000000000..cd7479724 --- /dev/null +++ b/external/db_drivers/CMakeLists.txt @@ -0,0 +1,54 @@ +# Copyright (c) 2014-2015, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# We aren't even going to check the system for an installed LMDB driver, as it is too +# critical a consensus component to rely on dynamically linked libraries +message(STATUS "Using ${ARCH_WIDTH}-bit LMDB from source tree") +add_subdirectory(liblmdb${ARCH_WIDTH}) +set(LMDB_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/liblmdb${ARCH_WIDTH}" CACHE STRING "LMDB Include path") + +set(LMDB_LIBRARY "lmdb" CACHE STRING "LMDB Library name") + +if (NOT STATIC) + find_package(BerkeleyDB) + + if(NOT BERKELEY_DB_LIBRARIES) + die("BerkeleyDB not found. At this time it should be installed in your system for a non-static build.") + else() + message(STATUS "Found BerkeleyDB include (db.h) in ${BERKELEY_DB_INCLUDE_DIR}") + if(BERKELEY_DB_LIBRARIES) + message(STATUS "Found BerkeleyDB shared library") + set(BDB_STATIC false CACHE BOOL "BDB Static flag") + set(BDB_INCLUDE ${BERKELEY_DB_INCLUDE_DIR} CACHE STRING "BDB include path") + set(BDB_LIBRARY ${BERKELEY_DB_LIBRARIES} CACHE STRING "BDB library name") + set(BDB_LIBRARY_DIRS "" CACHE STRING "BDB Library dirs") + else() + die("Found BerkeleyDB includes, but could not find BerkeleyDB library. Please make sure you have installed libdb and libdb-dev or the equivalent") + endif() + endif() +endif() diff --git a/external/db_drivers/liblmdb32/.gitignore b/external/db_drivers/liblmdb32/.gitignore new file mode 100644 index 000000000..f3277afe4 --- /dev/null +++ b/external/db_drivers/liblmdb32/.gitignore @@ -0,0 +1,20 @@ +mtest +mtest[23456] +testdb +mdb_copy +mdb_stat +*.[ao] +*.so +*.exe +*[~#] +*.bak +*.orig +*.rej +*.gcov +*.gcda +*.gcno +core +core.* +valgrind.* +man/ +html/ diff --git a/external/db_drivers/liblmdb32/CHANGES b/external/db_drivers/liblmdb32/CHANGES new file mode 100644 index 000000000..34049ba7a --- /dev/null +++ b/external/db_drivers/liblmdb32/CHANGES @@ -0,0 +1,112 @@ +LMDB 0.9 Change Log + +LMDB 0.9.14 Release (2014/09/20) + Fix to support 64K page size (ITS#7713) + Fix to persist decreased as well as increased mapsizes (ITS#7789) + Fix cursor bug when deleting last node of a DUPSORT key + Fix mdb_env_info to return FIXEDMAP address + Fix ambiguous error code from writing to closed DBI (ITS#7825) + Fix mdb_copy copying past end of file (ITS#7886) + Fix cursor bugs from page_merge/rebalance + Fix to dirty fewer pages in deletes (mdb_page_loose()) + Fix mdb_dbi_open creating subDBs (ITS#7917) + Fix mdb_cursor_get(_DUP) with single value (ITS#7913) + Fix Windows compat issues in mtests (ITS#7879) + Add compacting variant of mdb_copy + Add BigEndian integer key compare code + Add mdb_dump/mdb_load utilities + +LMDB 0.9.13 Release (2014/06/18) + Fix mdb_page_alloc unlimited overflow page search + Documentation + Re-fix MDB_CURRENT doc (ITS#7793) + Fix MDB_GET_MULTIPLE/MDB_NEXT_MULTIPLE doc + +LMDB 0.9.12 Release (2014/06/13) + Fix MDB_GET_BOTH regression (ITS#7875,#7681) + Fix MDB_MULTIPLE writing multiple keys (ITS#7834) + Fix mdb_rebalance (ITS#7829) + Fix mdb_page_split (ITS#7815) + Fix md_entries count (ITS#7861,#7828,#7793) + Fix MDB_CURRENT (ITS#7793) + Fix possible crash on Windows DLL detach + Misc code cleanup + Documentation + mdb_cursor_put: cursor moves on error (ITS#7771) + + +LMDB 0.9.11 Release (2014/01/15) + Add mdb_env_set_assert() (ITS#7775) + Fix: invalidate txn on page allocation errors (ITS#7377) + Fix xcursor tracking in mdb_cursor_del0() (ITS#7771) + Fix corruption from deletes (ITS#7756) + Fix Windows/MSVC build issues + Raise safe limit of max MDB_MAXKEYSIZE + Misc code cleanup + Documentation + Remove spurious note about non-overlapping flags (ITS#7665) + +LMDB 0.9.10 Release (2013/11/12) + Add MDB_NOMEMINIT option + Fix mdb_page_split() again (ITS#7589) + Fix MDB_NORDAHEAD definition (ITS#7734) + Fix mdb_cursor_del() positioning (ITS#7733) + Partial fix for larger page sizes (ITS#7713) + Fix Windows64/MSVC build issues + +LMDB 0.9.9 Release (2013/10/24) + Add mdb_env_get_fd() + Add MDB_NORDAHEAD option + Add MDB_NOLOCK option + Avoid wasting space in mdb_page_split() (ITS#7589) + Fix mdb_page_merge() cursor fixup (ITS#7722) + Fix mdb_cursor_del() on last delete (ITS#7718) + Fix adding WRITEMAP on existing env (ITS#7715) + Fix nested txns (ITS#7515) + Fix mdb_env_copy() O_DIRECT bug (ITS#7682) + Fix mdb_cursor_set(SET_RANGE) return code (ITS#7681) + Fix mdb_rebalance() cursor fixup (ITS#7701) + Misc code cleanup + Documentation + Note that by default, readers need write access + + +LMDB 0.9.8 Release (2013/09/09) + Allow mdb_env_set_mapsize() on an open environment + Fix mdb_dbi_flags() (ITS#7672) + Fix mdb_page_unspill() in nested txns + Fix mdb_cursor_get(CURRENT|NEXT) after a delete + Fix mdb_cursor_get(DUP) to always return key (ITS#7671) + Fix mdb_cursor_del() to always advance to next item (ITS#7670) + Fix mdb_cursor_set(SET_RANGE) for tree with single page (ITS#7681) + Fix mdb_env_copy() retry open if O_DIRECT fails (ITS#7682) + Tweak mdb_page_spill() to be less aggressive + Documentation + Update caveats since mdb_reader_check() added in 0.9.7 + +LMDB 0.9.7 Release (2013/08/17) + Don't leave stale lockfile on failed RDONLY open (ITS#7664) + Fix mdb_page_split() ref beyond cursor depth + Fix read txn data race (ITS#7635) + Fix mdb_rebalance (ITS#7536, #7538) + Fix mdb_drop() (ITS#7561) + Misc DEBUG macro fixes + Add MDB_NOTLS envflag + Add mdb_env_copyfd() + Add mdb_txn_env() (ITS#7660) + Add mdb_dbi_flags() (ITS#7661) + Add mdb_env_get_maxkeysize() + Add mdb_env_reader_list()/mdb_env_reader_check() + Add mdb_page_spill/unspill, remove hard txn size limit + Use shorter names for semaphores (ITS#7615) + Build + Fix install target (ITS#7656) + Documentation + Misc updates for cursors, DB handles, data lifetime + +LMDB 0.9.6 Release (2013/02/25) + Many fixes/enhancements + +LMDB 0.9.5 Release (2012/11/30) + Renamed from libmdb to liblmdb + Many fixes/enhancements diff --git a/external/db_drivers/liblmdb32/CMakeLists.txt b/external/db_drivers/liblmdb32/CMakeLists.txt new file mode 100644 index 000000000..62b800135 --- /dev/null +++ b/external/db_drivers/liblmdb32/CMakeLists.txt @@ -0,0 +1,39 @@ +# Copyright (c) 2014-2015, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set (lmdb_sources +mdb.c +midl.c) + +include_directories("${CMAKE_CURRENT_SOURCE_DIR}") + +add_library(lmdb + ${lmdb_sources}) +target_link_libraries(lmdb + LINK_PRIVATE + ${CMAKE_THREAD_LIBS_INIT}) diff --git a/external/db_drivers/liblmdb32/COPYRIGHT b/external/db_drivers/liblmdb32/COPYRIGHT new file mode 100644 index 000000000..e0eb484c7 --- /dev/null +++ b/external/db_drivers/liblmdb32/COPYRIGHT @@ -0,0 +1,20 @@ +Copyright 2011-2014 Howard Chu, Symas Corp. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted only as authorized by the OpenLDAP +Public License. + +A copy of this license is available in the file LICENSE in the +top-level directory of the distribution or, alternatively, at +<http://www.OpenLDAP.org/license.html>. + +OpenLDAP is a registered trademark of the OpenLDAP Foundation. + +Individual files and/or contributed packages may be copyright by +other parties and/or subject to additional restrictions. + +This work also contains materials derived from public sources. + +Additional information about OpenLDAP can be obtained at +<http://www.openldap.org/>. diff --git a/external/db_drivers/liblmdb32/Doxyfile b/external/db_drivers/liblmdb32/Doxyfile new file mode 100644 index 000000000..92d17b09e --- /dev/null +++ b/external/db_drivers/liblmdb32/Doxyfile @@ -0,0 +1,1631 @@ +# Doxyfile 1.7.1 + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for a project +# +# All text after a hash (#) is considered a comment and will be ignored +# The format is: +# TAG = value [value, ...] +# For lists items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (" ") + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- + +# This tag specifies the encoding used for all characters in the config file +# that follow. The default is UTF-8 which is also the encoding used for all +# text before the first occurrence of this tag. Doxygen uses libiconv (or the +# iconv built into libc) for the transcoding. See +# http://www.gnu.org/software/libiconv for the list of possible encodings. + +DOXYFILE_ENCODING = UTF-8 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded +# by quotes) that should identify the project. + +PROJECT_NAME = LMDB + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. +# This could be handy for archiving the generated documentation or +# if some version control system is used. + +PROJECT_NUMBER = + +# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) +# base path where the generated documentation will be put. +# If a relative path is entered, it will be relative to the location +# where doxygen was started. If left blank the current directory will be used. + +OUTPUT_DIRECTORY = + +# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create +# 4096 sub-directories (in 2 levels) under the output directory of each output +# format and will distribute the generated files over these directories. +# Enabling this option can be useful when feeding doxygen a huge amount of +# source files, where putting all generated files in the same directory would +# otherwise cause performance problems for the file system. + +CREATE_SUBDIRS = NO + +# The OUTPUT_LANGUAGE tag is used to specify the language in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all constant output in the proper language. +# The default language is English, other supported languages are: +# Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional, +# Croatian, Czech, Danish, Dutch, Esperanto, Farsi, Finnish, French, German, +# Greek, Hungarian, Italian, Japanese, Japanese-en (Japanese with English +# messages), Korean, Korean-en, Lithuanian, Norwegian, Macedonian, Persian, +# Polish, Portuguese, Romanian, Russian, Serbian, Serbian-Cyrilic, Slovak, +# Slovene, Spanish, Swedish, Ukrainian, and Vietnamese. + +OUTPUT_LANGUAGE = English + +# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will +# include brief member descriptions after the members that are listed in +# the file and class documentation (similar to JavaDoc). +# Set to NO to disable this. + +BRIEF_MEMBER_DESC = YES + +# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend +# the brief description of a member or function before the detailed description. +# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the +# brief descriptions will be completely suppressed. + +REPEAT_BRIEF = YES + +# This tag implements a quasi-intelligent brief description abbreviator +# that is used to form the text in various listings. Each string +# in this list, if found as the leading text of the brief description, will be +# stripped from the text and the result after processing the whole list, is +# used as the annotated text. Otherwise, the brief description is used as-is. +# If left blank, the following values are used ("$name" is automatically +# replaced with the name of the entity): "The $name class" "The $name widget" +# "The $name file" "is" "provides" "specifies" "contains" +# "represents" "a" "an" "the" + +ABBREVIATE_BRIEF = + +# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then +# Doxygen will generate a detailed section even if there is only a brief +# description. + +ALWAYS_DETAILED_SEC = NO + +# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all +# inherited members of a class in the documentation of that class as if those +# members were ordinary class members. Constructors, destructors and assignment +# operators of the base classes will not be shown. + +INLINE_INHERITED_MEMB = NO + +# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full +# path before files name in the file list and in the header files. If set +# to NO the shortest path that makes the file name unique will be used. + +FULL_PATH_NAMES = YES + +# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag +# can be used to strip a user-defined part of the path. Stripping is +# only done if one of the specified strings matches the left-hand part of +# the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the +# path to strip. + +STRIP_FROM_PATH = + +# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of +# the path mentioned in the documentation of a class, which tells +# the reader which header file to include in order to use a class. +# If left blank only the name of the header file containing the class +# definition is used. Otherwise one should specify the include paths that +# are normally passed to the compiler using the -I flag. + +STRIP_FROM_INC_PATH = + +# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter +# (but less readable) file names. This can be useful is your file systems +# doesn't support long names like on DOS, Mac, or CD-ROM. + +SHORT_NAMES = NO + +# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen +# will interpret the first line (until the first dot) of a JavaDoc-style +# comment as the brief description. If set to NO, the JavaDoc +# comments will behave just like regular Qt-style comments +# (thus requiring an explicit @brief command for a brief description.) + +JAVADOC_AUTOBRIEF = NO + +# If the QT_AUTOBRIEF tag is set to YES then Doxygen will +# interpret the first line (until the first dot) of a Qt-style +# comment as the brief description. If set to NO, the comments +# will behave just like regular Qt-style comments (thus requiring +# an explicit \brief command for a brief description.) + +QT_AUTOBRIEF = NO + +# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen +# treat a multi-line C++ special comment block (i.e. a block of //! or /// +# comments) as a brief description. This used to be the default behaviour. +# The new default is to treat a multi-line C++ comment block as a detailed +# description. Set this tag to YES if you prefer the old behaviour instead. + +MULTILINE_CPP_IS_BRIEF = NO + +# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented +# member inherits the documentation from any documented member that it +# re-implements. + +INHERIT_DOCS = YES + +# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce +# a new page for each member. If set to NO, the documentation of a member will +# be part of the file/class/namespace that contains it. + +SEPARATE_MEMBER_PAGES = NO + +# The TAB_SIZE tag can be used to set the number of spaces in a tab. +# Doxygen uses this value to replace tabs by spaces in code fragments. + +TAB_SIZE = 4 + +# This tag can be used to specify a number of aliases that acts +# as commands in the documentation. An alias has the form "name=value". +# For example adding "sideeffect=\par Side Effects:\n" will allow you to +# put the command \sideeffect (or @sideeffect) in the documentation, which +# will result in a user-defined paragraph with heading "Side Effects:". +# You can put \n's in the value part of an alias to insert newlines. + +ALIASES = + +# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C +# sources only. Doxygen will then generate output that is more tailored for C. +# For instance, some of the names that are used will be different. The list +# of all members will be omitted, etc. + +OPTIMIZE_OUTPUT_FOR_C = YES + +# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java +# sources only. Doxygen will then generate output that is more tailored for +# Java. For instance, namespaces will be presented as packages, qualified +# scopes will look different, etc. + +OPTIMIZE_OUTPUT_JAVA = NO + +# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran +# sources only. Doxygen will then generate output that is more tailored for +# Fortran. + +OPTIMIZE_FOR_FORTRAN = NO + +# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL +# sources. Doxygen will then generate output that is tailored for +# VHDL. + +OPTIMIZE_OUTPUT_VHDL = NO + +# Doxygen selects the parser to use depending on the extension of the files it +# parses. With this tag you can assign which parser to use for a given extension. +# Doxygen has a built-in mapping, but you can override or extend it using this +# tag. The format is ext=language, where ext is a file extension, and language +# is one of the parsers supported by doxygen: IDL, Java, Javascript, CSharp, C, +# C++, D, PHP, Objective-C, Python, Fortran, VHDL, C, C++. For instance to make +# doxygen treat .inc files as Fortran files (default is PHP), and .f files as C +# (default is Fortran), use: inc=Fortran f=C. Note that for custom extensions +# you also need to set FILE_PATTERNS otherwise the files are not read by doxygen. + +EXTENSION_MAPPING = + +# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want +# to include (a tag file for) the STL sources as input, then you should +# set this tag to YES in order to let doxygen match functions declarations and +# definitions whose arguments contain STL classes (e.g. func(std::string); v.s. +# func(std::string) {}). This also make the inheritance and collaboration +# diagrams that involve STL classes more complete and accurate. + +BUILTIN_STL_SUPPORT = NO + +# If you use Microsoft's C++/CLI language, you should set this option to YES to +# enable parsing support. + +CPP_CLI_SUPPORT = NO + +# Set the SIP_SUPPORT tag to YES if your project consists of sip sources only. +# Doxygen will parse them like normal C++ but will assume all classes use public +# instead of private inheritance when no explicit protection keyword is present. + +SIP_SUPPORT = NO + +# For Microsoft's IDL there are propget and propput attributes to indicate getter +# and setter methods for a property. Setting this option to YES (the default) +# will make doxygen to replace the get and set methods by a property in the +# documentation. This will only work if the methods are indeed getting or +# setting a simple type. If this is not the case, or you want to show the +# methods anyway, you should set this option to NO. + +IDL_PROPERTY_SUPPORT = YES + +# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC +# tag is set to YES, then doxygen will reuse the documentation of the first +# member in the group (if any) for the other members of the group. By default +# all members of a group must be documented explicitly. + +DISTRIBUTE_GROUP_DOC = NO + +# Set the SUBGROUPING tag to YES (the default) to allow class member groups of +# the same type (for instance a group of public functions) to be put as a +# subgroup of that type (e.g. under the Public Functions section). Set it to +# NO to prevent subgrouping. Alternatively, this can be done per class using +# the \nosubgrouping command. + +SUBGROUPING = YES + +INLINE_GROUPED_CLASSES = YES +# When TYPEDEF_HIDES_STRUCT is enabled, a typedef of a struct, union, or enum +# is documented as struct, union, or enum with the name of the typedef. So +# typedef struct TypeS {} TypeT, will appear in the documentation as a struct +# with name TypeT. When disabled the typedef will appear as a member of a file, +# namespace, or class. And the struct will be named TypeS. This can typically +# be useful for C code in case the coding convention dictates that all compound +# types are typedef'ed and only the typedef is referenced, never the tag name. + +TYPEDEF_HIDES_STRUCT = YES + +# The SYMBOL_CACHE_SIZE determines the size of the internal cache use to +# determine which symbols to keep in memory and which to flush to disk. +# When the cache is full, less often used symbols will be written to disk. +# For small to medium size projects (<1000 input files) the default value is +# probably good enough. For larger projects a too small cache size can cause +# doxygen to be busy swapping symbols to and from disk most of the time +# causing a significant performance penality. +# If the system has enough physical memory increasing the cache will improve the +# performance by keeping more symbols in memory. Note that the value works on +# a logarithmic scale so increasing the size by one will rougly double the +# memory usage. The cache size is given by this formula: +# 2^(16+SYMBOL_CACHE_SIZE). The valid range is 0..9, the default is 0, +# corresponding to a cache size of 2^16 = 65536 symbols + +SYMBOL_CACHE_SIZE = 0 + +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- + +# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in +# documentation are documented, even if no documentation was available. +# Private class members and static file members will be hidden unless +# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES + +EXTRACT_ALL = NO + +# If the EXTRACT_PRIVATE tag is set to YES all private members of a class +# will be included in the documentation. + +EXTRACT_PRIVATE = NO + +# If the EXTRACT_STATIC tag is set to YES all static members of a file +# will be included in the documentation. + +EXTRACT_STATIC = YES + +# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) +# defined locally in source files will be included in the documentation. +# If set to NO only classes defined in header files are included. + +EXTRACT_LOCAL_CLASSES = YES + +# This flag is only useful for Objective-C code. When set to YES local +# methods, which are defined in the implementation section but not in +# the interface are included in the documentation. +# If set to NO (the default) only methods in the interface are included. + +EXTRACT_LOCAL_METHODS = NO + +# If this flag is set to YES, the members of anonymous namespaces will be +# extracted and appear in the documentation as a namespace called +# 'anonymous_namespace{file}', where file will be replaced with the base +# name of the file that contains the anonymous namespace. By default +# anonymous namespace are hidden. + +EXTRACT_ANON_NSPACES = NO + +# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all +# undocumented members of documented classes, files or namespaces. +# If set to NO (the default) these members will be included in the +# various overviews, but no documentation section is generated. +# This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_MEMBERS = NO + +# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all +# undocumented classes that are normally visible in the class hierarchy. +# If set to NO (the default) these classes will be included in the various +# overviews. This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_CLASSES = NO + +# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all +# friend (class|struct|union) declarations. +# If set to NO (the default) these declarations will be included in the +# documentation. + +HIDE_FRIEND_COMPOUNDS = NO + +# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any +# documentation blocks found inside the body of a function. +# If set to NO (the default) these blocks will be appended to the +# function's detailed documentation block. + +HIDE_IN_BODY_DOCS = NO + +# The INTERNAL_DOCS tag determines if documentation +# that is typed after a \internal command is included. If the tag is set +# to NO (the default) then the documentation will be excluded. +# Set it to YES to include the internal documentation. + +INTERNAL_DOCS = NO + +# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate +# file names in lower-case letters. If set to YES upper-case letters are also +# allowed. This is useful if you have classes or files whose names only differ +# in case and if your file system supports case sensitive file names. Windows +# and Mac users are advised to set this option to NO. + +CASE_SENSE_NAMES = YES + +# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen +# will show members with their full class and namespace scopes in the +# documentation. If set to YES the scope will be hidden. + +HIDE_SCOPE_NAMES = NO + +# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen +# will put a list of the files that are included by a file in the documentation +# of that file. + +SHOW_INCLUDE_FILES = YES + +# If the FORCE_LOCAL_INCLUDES tag is set to YES then Doxygen +# will list include files with double quotes in the documentation +# rather than with sharp brackets. + +FORCE_LOCAL_INCLUDES = NO + +# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] +# is inserted in the documentation for inline members. + +INLINE_INFO = YES + +# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen +# will sort the (detailed) documentation of file and class members +# alphabetically by member name. If set to NO the members will appear in +# declaration order. + +SORT_MEMBER_DOCS = NO + +# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the +# brief documentation of file, namespace and class members alphabetically +# by member name. If set to NO (the default) the members will appear in +# declaration order. + +SORT_BRIEF_DOCS = NO + +# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen +# will sort the (brief and detailed) documentation of class members so that +# constructors and destructors are listed first. If set to NO (the default) +# the constructors will appear in the respective orders defined by +# SORT_MEMBER_DOCS and SORT_BRIEF_DOCS. +# This tag will be ignored for brief docs if SORT_BRIEF_DOCS is set to NO +# and ignored for detailed docs if SORT_MEMBER_DOCS is set to NO. + +SORT_MEMBERS_CTORS_1ST = NO + +# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the +# hierarchy of group names into alphabetical order. If set to NO (the default) +# the group names will appear in their defined order. + +SORT_GROUP_NAMES = NO + +# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be +# sorted by fully-qualified names, including namespaces. If set to +# NO (the default), the class list will be sorted only by class name, +# not including the namespace part. +# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. +# Note: This option applies only to the class list, not to the +# alphabetical list. + +SORT_BY_SCOPE_NAME = NO + +# The GENERATE_TODOLIST tag can be used to enable (YES) or +# disable (NO) the todo list. This list is created by putting \todo +# commands in the documentation. + +GENERATE_TODOLIST = YES + +# The GENERATE_TESTLIST tag can be used to enable (YES) or +# disable (NO) the test list. This list is created by putting \test +# commands in the documentation. + +GENERATE_TESTLIST = YES + +# The GENERATE_BUGLIST tag can be used to enable (YES) or +# disable (NO) the bug list. This list is created by putting \bug +# commands in the documentation. + +GENERATE_BUGLIST = YES + +# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or +# disable (NO) the deprecated list. This list is created by putting +# \deprecated commands in the documentation. + +GENERATE_DEPRECATEDLIST= YES + +# The ENABLED_SECTIONS tag can be used to enable conditional +# documentation sections, marked by \if sectionname ... \endif. + +ENABLED_SECTIONS = + +# The MAX_INITIALIZER_LINES tag determines the maximum number of lines +# the initial value of a variable or define consists of for it to appear in +# the documentation. If the initializer consists of more lines than specified +# here it will be hidden. Use a value of 0 to hide initializers completely. +# The appearance of the initializer of individual variables and defines in the +# documentation can be controlled using \showinitializer or \hideinitializer +# command in the documentation regardless of this setting. + +MAX_INITIALIZER_LINES = 30 + +# Set the SHOW_USED_FILES tag to NO to disable the list of files generated +# at the bottom of the documentation of classes and structs. If set to YES the +# list will mention the files that were used to generate the documentation. + +SHOW_USED_FILES = YES + +# If the sources in your project are distributed over multiple directories +# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy +# in the documentation. The default is NO. + +SHOW_DIRECTORIES = NO + +# Set the SHOW_FILES tag to NO to disable the generation of the Files page. +# This will remove the Files entry from the Quick Index and from the +# Folder Tree View (if specified). The default is YES. + +SHOW_FILES = YES + +# Set the SHOW_NAMESPACES tag to NO to disable the generation of the +# Namespaces page. +# This will remove the Namespaces entry from the Quick Index +# and from the Folder Tree View (if specified). The default is YES. + +SHOW_NAMESPACES = YES + +# The FILE_VERSION_FILTER tag can be used to specify a program or script that +# doxygen should invoke to get the current version for each file (typically from +# the version control system). Doxygen will invoke the program by executing (via +# popen()) the command <command> <input-file>, where <command> is the value of +# the FILE_VERSION_FILTER tag, and <input-file> is the name of an input file +# provided by doxygen. Whatever the program writes to standard output +# is used as the file version. See the manual for examples. + +FILE_VERSION_FILTER = + +# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed +# by doxygen. The layout file controls the global structure of the generated +# output files in an output format independent way. The create the layout file +# that represents doxygen's defaults, run doxygen with the -l option. +# You can optionally specify a file name after the option, if omitted +# DoxygenLayout.xml will be used as the name of the layout file. + +LAYOUT_FILE = + +#--------------------------------------------------------------------------- +# configuration options related to warning and progress messages +#--------------------------------------------------------------------------- + +# The QUIET tag can be used to turn on/off the messages that are generated +# by doxygen. Possible values are YES and NO. If left blank NO is used. + +QUIET = NO + +# The WARNINGS tag can be used to turn on/off the warning messages that are +# generated by doxygen. Possible values are YES and NO. If left blank +# NO is used. + +WARNINGS = YES + +# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings +# for undocumented members. If EXTRACT_ALL is set to YES then this flag will +# automatically be disabled. + +WARN_IF_UNDOCUMENTED = YES + +# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for +# potential errors in the documentation, such as not documenting some +# parameters in a documented function, or documenting parameters that +# don't exist or using markup commands wrongly. + +WARN_IF_DOC_ERROR = YES + +# This WARN_NO_PARAMDOC option can be abled to get warnings for +# functions that are documented, but have no documentation for their parameters +# or return value. If set to NO (the default) doxygen will only warn about +# wrong or incomplete parameter documentation, but not about the absence of +# documentation. + +WARN_NO_PARAMDOC = NO + +# The WARN_FORMAT tag determines the format of the warning messages that +# doxygen can produce. The string should contain the $file, $line, and $text +# tags, which will be replaced by the file and line number from which the +# warning originated and the warning text. Optionally the format may contain +# $version, which will be replaced by the version of the file (if it could +# be obtained via FILE_VERSION_FILTER) + +WARN_FORMAT = "$file:$line: $text" + +# The WARN_LOGFILE tag can be used to specify a file to which warning +# and error messages should be written. If left blank the output is written +# to stderr. + +WARN_LOGFILE = + +#--------------------------------------------------------------------------- +# configuration options related to the input files +#--------------------------------------------------------------------------- + +# The INPUT tag can be used to specify the files and/or directories that contain +# documented source files. You may enter file names like "myfile.cpp" or +# directories like "/usr/src/myproject". Separate the files or directories +# with spaces. + +INPUT = lmdb.h midl.h mdb.c midl.c + +# This tag can be used to specify the character encoding of the source files +# that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is +# also the default input encoding. Doxygen uses libiconv (or the iconv built +# into libc) for the transcoding. See http://www.gnu.org/software/libiconv for +# the list of possible encodings. + +INPUT_ENCODING = UTF-8 + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank the following patterns are tested: +# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx +# *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py *.f90 + +FILE_PATTERNS = + +# The RECURSIVE tag can be used to turn specify whether or not subdirectories +# should be searched for input files as well. Possible values are YES and NO. +# If left blank NO is used. + +RECURSIVE = NO + +# The EXCLUDE tag can be used to specify files and/or directories that should +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. + +EXCLUDE = + +# The EXCLUDE_SYMLINKS tag can be used select whether or not files or +# directories that are symbolic links (a Unix filesystem feature) are excluded +# from the input. + +EXCLUDE_SYMLINKS = NO + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. Note that the wildcards are matched +# against the file with absolute path, so to exclude all test directories +# for example use the pattern */test/* + +EXCLUDE_PATTERNS = + +# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names +# (namespaces, classes, functions, etc.) that should be excluded from the +# output. The symbol name can be a fully qualified name, a word, or if the +# wildcard * is used, a substring. Examples: ANamespace, AClass, +# AClass::ANamespace, ANamespace::*Test + +EXCLUDE_SYMBOLS = + +# The EXAMPLE_PATH tag can be used to specify one or more files or +# directories that contain example code fragments that are included (see +# the \include command). + +EXAMPLE_PATH = + +# If the value of the EXAMPLE_PATH tag contains directories, you can use the +# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank all files are included. + +EXAMPLE_PATTERNS = + +# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be +# searched for input files to be used with the \include or \dontinclude +# commands irrespective of the value of the RECURSIVE tag. +# Possible values are YES and NO. If left blank NO is used. + +EXAMPLE_RECURSIVE = NO + +# The IMAGE_PATH tag can be used to specify one or more files or +# directories that contain image that are included in the documentation (see +# the \image command). + +IMAGE_PATH = + +# The INPUT_FILTER tag can be used to specify a program that doxygen should +# invoke to filter for each input file. Doxygen will invoke the filter program +# by executing (via popen()) the command <filter> <input-file>, where <filter> +# is the value of the INPUT_FILTER tag, and <input-file> is the name of an +# input file. Doxygen will then use the output that the filter program writes +# to standard output. +# If FILTER_PATTERNS is specified, this tag will be +# ignored. + +INPUT_FILTER = + +# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern +# basis. +# Doxygen will compare the file name with each pattern and apply the +# filter if there is a match. +# The filters are a list of the form: +# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further +# info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER +# is applied to all files. + +FILTER_PATTERNS = + +# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using +# INPUT_FILTER) will be used to filter the input files when producing source +# files to browse (i.e. when SOURCE_BROWSER is set to YES). + +FILTER_SOURCE_FILES = NO + +#--------------------------------------------------------------------------- +# configuration options related to source browsing +#--------------------------------------------------------------------------- + +# If the SOURCE_BROWSER tag is set to YES then a list of source files will +# be generated. Documented entities will be cross-referenced with these sources. +# Note: To get rid of all source code in the generated output, make sure also +# VERBATIM_HEADERS is set to NO. + +SOURCE_BROWSER = NO + +# Setting the INLINE_SOURCES tag to YES will include the body +# of functions and classes directly in the documentation. + +INLINE_SOURCES = NO + +# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct +# doxygen to hide any special comment blocks from generated source code +# fragments. Normal C and C++ comments will always remain visible. + +STRIP_CODE_COMMENTS = YES + +# If the REFERENCED_BY_RELATION tag is set to YES +# then for each documented function all documented +# functions referencing it will be listed. + +REFERENCED_BY_RELATION = NO + +# If the REFERENCES_RELATION tag is set to YES +# then for each documented function all documented entities +# called/used by that function will be listed. + +REFERENCES_RELATION = NO + +# If the REFERENCES_LINK_SOURCE tag is set to YES (the default) +# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from +# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will +# link to the source code. +# Otherwise they will link to the documentation. + +REFERENCES_LINK_SOURCE = YES + +# If the USE_HTAGS tag is set to YES then the references to source code +# will point to the HTML generated by the htags(1) tool instead of doxygen +# built-in source browser. The htags tool is part of GNU's global source +# tagging system (see http://www.gnu.org/software/global/global.html). You +# will need version 4.8.6 or higher. + +USE_HTAGS = NO + +# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen +# will generate a verbatim copy of the header file for each class for +# which an include is specified. Set to NO to disable this. + +VERBATIM_HEADERS = YES + +#--------------------------------------------------------------------------- +# configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- + +# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index +# of all compounds will be generated. Enable this if the project +# contains a lot of classes, structs, unions or interfaces. + +ALPHABETICAL_INDEX = YES + +# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then +# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns +# in which this list will be split (can be a number in the range [1..20]) + +COLS_IN_ALPHA_INDEX = 5 + +# In case all classes in a project start with a common prefix, all +# classes will be put under the same header in the alphabetical index. +# The IGNORE_PREFIX tag can be used to specify one or more prefixes that +# should be ignored while generating the index headers. + +IGNORE_PREFIX = + +#--------------------------------------------------------------------------- +# configuration options related to the HTML output +#--------------------------------------------------------------------------- + +# If the GENERATE_HTML tag is set to YES (the default) Doxygen will +# generate HTML output. + +GENERATE_HTML = YES + +# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `html' will be used as the default path. + +HTML_OUTPUT = html + +# The HTML_FILE_EXTENSION tag can be used to specify the file extension for +# each generated HTML page (for example: .htm,.php,.asp). If it is left blank +# doxygen will generate files with .html extension. + +HTML_FILE_EXTENSION = .html + +# The HTML_HEADER tag can be used to specify a personal HTML header for +# each generated HTML page. If it is left blank doxygen will generate a +# standard header. + +HTML_HEADER = + +# The HTML_FOOTER tag can be used to specify a personal HTML footer for +# each generated HTML page. If it is left blank doxygen will generate a +# standard footer. + +HTML_FOOTER = + +# The HTML_STYLESHEET tag can be used to specify a user-defined cascading +# style sheet that is used by each HTML page. It can be used to +# fine-tune the look of the HTML output. If the tag is left blank doxygen +# will generate a default style sheet. Note that doxygen will try to copy +# the style sheet file to the HTML output directory, so don't put your own +# stylesheet in the HTML output directory as well, or it will be erased! + +HTML_STYLESHEET = + +# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. +# Doxygen will adjust the colors in the stylesheet and background images +# according to this color. Hue is specified as an angle on a colorwheel, +# see http://en.wikipedia.org/wiki/Hue for more information. +# For instance the value 0 represents red, 60 is yellow, 120 is green, +# 180 is cyan, 240 is blue, 300 purple, and 360 is red again. +# The allowed range is 0 to 359. + +HTML_COLORSTYLE_HUE = 220 + +# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of +# the colors in the HTML output. For a value of 0 the output will use +# grayscales only. A value of 255 will produce the most vivid colors. + +HTML_COLORSTYLE_SAT = 100 + +# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to +# the luminance component of the colors in the HTML output. Values below +# 100 gradually make the output lighter, whereas values above 100 make +# the output darker. The value divided by 100 is the actual gamma applied, +# so 80 represents a gamma of 0.8, The value 220 represents a gamma of 2.2, +# and 100 does not change the gamma. + +HTML_COLORSTYLE_GAMMA = 80 + +# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML +# page will contain the date and time when the page was generated. Setting +# this to NO can help when comparing the output of multiple runs. + +HTML_TIMESTAMP = YES + +# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, +# files or namespaces will be aligned in HTML using tables. If set to +# NO a bullet list will be used. + +HTML_ALIGN_MEMBERS = YES + +# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML +# documentation will contain sections that can be hidden and shown after the +# page has loaded. For this to work a browser that supports +# JavaScript and DHTML is required (for instance Mozilla 1.0+, Firefox +# Netscape 6.0+, Internet explorer 5.0+, Konqueror, or Safari). + +HTML_DYNAMIC_SECTIONS = NO + +# If the GENERATE_DOCSET tag is set to YES, additional index files +# will be generated that can be used as input for Apple's Xcode 3 +# integrated development environment, introduced with OSX 10.5 (Leopard). +# To create a documentation set, doxygen will generate a Makefile in the +# HTML output directory. Running make will produce the docset in that +# directory and running "make install" will install the docset in +# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find +# it at startup. +# See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html +# for more information. + +GENERATE_DOCSET = NO + +# When GENERATE_DOCSET tag is set to YES, this tag determines the name of the +# feed. A documentation feed provides an umbrella under which multiple +# documentation sets from a single provider (such as a company or product suite) +# can be grouped. + +DOCSET_FEEDNAME = "Doxygen generated docs" + +# When GENERATE_DOCSET tag is set to YES, this tag specifies a string that +# should uniquely identify the documentation set bundle. This should be a +# reverse domain-name style string, e.g. com.mycompany.MyDocSet. Doxygen +# will append .docset to the name. + +DOCSET_BUNDLE_ID = org.doxygen.Project + +# When GENERATE_PUBLISHER_ID tag specifies a string that should uniquely identify +# the documentation publisher. This should be a reverse domain-name style +# string, e.g. com.mycompany.MyDocSet.documentation. + +DOCSET_PUBLISHER_ID = org.doxygen.Publisher + +# The GENERATE_PUBLISHER_NAME tag identifies the documentation publisher. + +DOCSET_PUBLISHER_NAME = Publisher + +# If the GENERATE_HTMLHELP tag is set to YES, additional index files +# will be generated that can be used as input for tools like the +# Microsoft HTML help workshop to generate a compiled HTML help file (.chm) +# of the generated HTML documentation. + +GENERATE_HTMLHELP = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can +# be used to specify the file name of the resulting .chm file. You +# can add a path in front of the file if the result should not be +# written to the html output directory. + +CHM_FILE = + +# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can +# be used to specify the location (absolute path including file name) of +# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run +# the HTML help compiler on the generated index.hhp. + +HHC_LOCATION = + +# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag +# controls if a separate .chi index file is generated (YES) or that +# it should be included in the master .chm file (NO). + +GENERATE_CHI = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the CHM_INDEX_ENCODING +# is used to encode HtmlHelp index (hhk), content (hhc) and project file +# content. + +CHM_INDEX_ENCODING = + +# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag +# controls whether a binary table of contents is generated (YES) or a +# normal table of contents (NO) in the .chm file. + +BINARY_TOC = NO + +# The TOC_EXPAND flag can be set to YES to add extra items for group members +# to the contents of the HTML help documentation and to the tree view. + +TOC_EXPAND = NO + +# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and +# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated +# that can be used as input for Qt's qhelpgenerator to generate a +# Qt Compressed Help (.qch) of the generated HTML documentation. + +GENERATE_QHP = NO + +# If the QHG_LOCATION tag is specified, the QCH_FILE tag can +# be used to specify the file name of the resulting .qch file. +# The path specified is relative to the HTML output folder. + +QCH_FILE = + +# The QHP_NAMESPACE tag specifies the namespace to use when generating +# Qt Help Project output. For more information please see +# http://doc.trolltech.com/qthelpproject.html#namespace + +QHP_NAMESPACE = org.doxygen.Project + +# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating +# Qt Help Project output. For more information please see +# http://doc.trolltech.com/qthelpproject.html#virtual-folders + +QHP_VIRTUAL_FOLDER = doc + +# If QHP_CUST_FILTER_NAME is set, it specifies the name of a custom filter to +# add. For more information please see +# http://doc.trolltech.com/qthelpproject.html#custom-filters + +QHP_CUST_FILTER_NAME = + +# The QHP_CUST_FILT_ATTRS tag specifies the list of the attributes of the +# custom filter to add. For more information please see +# <a href="http://doc.trolltech.com/qthelpproject.html#custom-filters"> +# Qt Help Project / Custom Filters</a>. + +QHP_CUST_FILTER_ATTRS = + +# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this +# project's +# filter section matches. +# <a href="http://doc.trolltech.com/qthelpproject.html#filter-attributes"> +# Qt Help Project / Filter Attributes</a>. + +QHP_SECT_FILTER_ATTRS = + +# If the GENERATE_QHP tag is set to YES, the QHG_LOCATION tag can +# be used to specify the location of Qt's qhelpgenerator. +# If non-empty doxygen will try to run qhelpgenerator on the generated +# .qhp file. + +QHG_LOCATION = + +# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files +# will be generated, which together with the HTML files, form an Eclipse help +# plugin. To install this plugin and make it available under the help contents +# menu in Eclipse, the contents of the directory containing the HTML and XML +# files needs to be copied into the plugins directory of eclipse. The name of +# the directory within the plugins directory should be the same as +# the ECLIPSE_DOC_ID value. After copying Eclipse needs to be restarted before +# the help appears. + +GENERATE_ECLIPSEHELP = NO + +# A unique identifier for the eclipse help plugin. When installing the plugin +# the directory name containing the HTML and XML files should also have +# this name. + +ECLIPSE_DOC_ID = org.doxygen.Project + +# The DISABLE_INDEX tag can be used to turn on/off the condensed index at +# top of each HTML page. The value NO (the default) enables the index and +# the value YES disables it. + +DISABLE_INDEX = NO + +# This tag can be used to set the number of enum values (range [1..20]) +# that doxygen will group on one line in the generated HTML documentation. + +ENUM_VALUES_PER_LINE = 4 + +# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index +# structure should be generated to display hierarchical information. +# If the tag value is set to YES, a side panel will be generated +# containing a tree-like index structure (just like the one that +# is generated for HTML Help). For this to work a browser that supports +# JavaScript, DHTML, CSS and frames is required (i.e. any modern browser). +# Windows users are probably better off using the HTML help feature. + +GENERATE_TREEVIEW = NO + +# By enabling USE_INLINE_TREES, doxygen will generate the Groups, Directories, +# and Class Hierarchy pages using a tree view instead of an ordered list. + +USE_INLINE_TREES = NO + +# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be +# used to set the initial width (in pixels) of the frame in which the tree +# is shown. + +TREEVIEW_WIDTH = 250 + +# When the EXT_LINKS_IN_WINDOW option is set to YES doxygen will open +# links to external symbols imported via tag files in a separate window. + +EXT_LINKS_IN_WINDOW = NO + +# Use this tag to change the font size of Latex formulas included +# as images in the HTML documentation. The default is 10. Note that +# when you change the font size after a successful doxygen run you need +# to manually remove any form_*.png images from the HTML output directory +# to force them to be regenerated. + +FORMULA_FONTSIZE = 10 + +# Use the FORMULA_TRANPARENT tag to determine whether or not the images +# generated for formulas are transparent PNGs. Transparent PNGs are +# not supported properly for IE 6.0, but are supported on all modern browsers. +# Note that when changing this option you need to delete any form_*.png files +# in the HTML output before the changes have effect. + +FORMULA_TRANSPARENT = YES + +# When the SEARCHENGINE tag is enabled doxygen will generate a search box +# for the HTML output. The underlying search engine uses javascript +# and DHTML and should work on any modern browser. Note that when using +# HTML help (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets +# (GENERATE_DOCSET) there is already a search function so this one should +# typically be disabled. For large projects the javascript based search engine +# can be slow, then enabling SERVER_BASED_SEARCH may provide a better solution. + +SEARCHENGINE = YES + +# When the SERVER_BASED_SEARCH tag is enabled the search engine will be +# implemented using a PHP enabled web server instead of at the web client +# using Javascript. Doxygen will generate the search PHP script and index +# file to put on the web server. The advantage of the server +# based approach is that it scales better to large projects and allows +# full text search. The disadvances is that it is more difficult to setup +# and does not have live searching capabilities. + +SERVER_BASED_SEARCH = NO + +#--------------------------------------------------------------------------- +# configuration options related to the LaTeX output +#--------------------------------------------------------------------------- + +# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will +# generate Latex output. + +GENERATE_LATEX = NO + +# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `latex' will be used as the default path. + +LATEX_OUTPUT = latex + +# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be +# invoked. If left blank `latex' will be used as the default command name. +# Note that when enabling USE_PDFLATEX this option is only used for +# generating bitmaps for formulas in the HTML output, but not in the +# Makefile that is written to the output directory. + +LATEX_CMD_NAME = latex + +# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to +# generate index for LaTeX. If left blank `makeindex' will be used as the +# default command name. + +MAKEINDEX_CMD_NAME = makeindex + +# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact +# LaTeX documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_LATEX = NO + +# The PAPER_TYPE tag can be used to set the paper type that is used +# by the printer. Possible values are: a4, a4wide, letter, legal and +# executive. If left blank a4wide will be used. + +PAPER_TYPE = a4wide + +# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX +# packages that should be included in the LaTeX output. + +EXTRA_PACKAGES = + +# The LATEX_HEADER tag can be used to specify a personal LaTeX header for +# the generated latex document. The header should contain everything until +# the first chapter. If it is left blank doxygen will generate a +# standard header. Notice: only use this tag if you know what you are doing! + +LATEX_HEADER = + +# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated +# is prepared for conversion to pdf (using ps2pdf). The pdf file will +# contain links (just like the HTML output) instead of page references +# This makes the output suitable for online browsing using a pdf viewer. + +PDF_HYPERLINKS = YES + +# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of +# plain latex in the generated Makefile. Set this option to YES to get a +# higher quality PDF documentation. + +USE_PDFLATEX = YES + +# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. +# command to the generated LaTeX files. This will instruct LaTeX to keep +# running if errors occur, instead of asking the user for help. +# This option is also used when generating formulas in HTML. + +LATEX_BATCHMODE = NO + +# If LATEX_HIDE_INDICES is set to YES then doxygen will not +# include the index chapters (such as File Index, Compound Index, etc.) +# in the output. + +LATEX_HIDE_INDICES = NO + +# If LATEX_SOURCE_CODE is set to YES then doxygen will include +# source code with syntax highlighting in the LaTeX output. +# Note that which sources are shown also depends on other settings +# such as SOURCE_BROWSER. + +LATEX_SOURCE_CODE = NO + +#--------------------------------------------------------------------------- +# configuration options related to the RTF output +#--------------------------------------------------------------------------- + +# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output +# The RTF output is optimized for Word 97 and may not look very pretty with +# other RTF readers or editors. + +GENERATE_RTF = NO + +# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `rtf' will be used as the default path. + +RTF_OUTPUT = rtf + +# If the COMPACT_RTF tag is set to YES Doxygen generates more compact +# RTF documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_RTF = NO + +# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated +# will contain hyperlink fields. The RTF file will +# contain links (just like the HTML output) instead of page references. +# This makes the output suitable for online browsing using WORD or other +# programs which support those fields. +# Note: wordpad (write) and others do not support links. + +RTF_HYPERLINKS = NO + +# Load stylesheet definitions from file. Syntax is similar to doxygen's +# config file, i.e. a series of assignments. You only have to provide +# replacements, missing definitions are set to their default value. + +RTF_STYLESHEET_FILE = + +# Set optional variables used in the generation of an rtf document. +# Syntax is similar to doxygen's config file. + +RTF_EXTENSIONS_FILE = + +#--------------------------------------------------------------------------- +# configuration options related to the man page output +#--------------------------------------------------------------------------- + +# If the GENERATE_MAN tag is set to YES (the default) Doxygen will +# generate man pages + +GENERATE_MAN = YES + +# The MAN_OUTPUT tag is used to specify where the man pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `man' will be used as the default path. + +MAN_OUTPUT = man + +# The MAN_EXTENSION tag determines the extension that is added to +# the generated man pages (default is the subroutine's section .3) + +MAN_EXTENSION = .3 + +# If the MAN_LINKS tag is set to YES and Doxygen generates man output, +# then it will generate one additional man file for each entity +# documented in the real man page(s). These additional files +# only source the real man page, but without them the man command +# would be unable to find the correct page. The default is NO. + +MAN_LINKS = NO + +#--------------------------------------------------------------------------- +# configuration options related to the XML output +#--------------------------------------------------------------------------- + +# If the GENERATE_XML tag is set to YES Doxygen will +# generate an XML file that captures the structure of +# the code including all documentation. + +GENERATE_XML = NO + +# The XML_OUTPUT tag is used to specify where the XML pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `xml' will be used as the default path. + +XML_OUTPUT = xml + +# The XML_SCHEMA tag can be used to specify an XML schema, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_SCHEMA = + +# The XML_DTD tag can be used to specify an XML DTD, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_DTD = + +# If the XML_PROGRAMLISTING tag is set to YES Doxygen will +# dump the program listings (including syntax highlighting +# and cross-referencing information) to the XML output. Note that +# enabling this will significantly increase the size of the XML output. + +XML_PROGRAMLISTING = YES + +#--------------------------------------------------------------------------- +# configuration options for the AutoGen Definitions output +#--------------------------------------------------------------------------- + +# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will +# generate an AutoGen Definitions (see autogen.sf.net) file +# that captures the structure of the code including all +# documentation. Note that this feature is still experimental +# and incomplete at the moment. + +GENERATE_AUTOGEN_DEF = NO + +#--------------------------------------------------------------------------- +# configuration options related to the Perl module output +#--------------------------------------------------------------------------- + +# If the GENERATE_PERLMOD tag is set to YES Doxygen will +# generate a Perl module file that captures the structure of +# the code including all documentation. Note that this +# feature is still experimental and incomplete at the +# moment. + +GENERATE_PERLMOD = NO + +# If the PERLMOD_LATEX tag is set to YES Doxygen will generate +# the necessary Makefile rules, Perl scripts and LaTeX code to be able +# to generate PDF and DVI output from the Perl module output. + +PERLMOD_LATEX = NO + +# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be +# nicely formatted so it can be parsed by a human reader. +# This is useful +# if you want to understand what is going on. +# On the other hand, if this +# tag is set to NO the size of the Perl module output will be much smaller +# and Perl will parse it just the same. + +PERLMOD_PRETTY = YES + +# The names of the make variables in the generated doxyrules.make file +# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. +# This is useful so different doxyrules.make files included by the same +# Makefile don't overwrite each other's variables. + +PERLMOD_MAKEVAR_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the preprocessor +#--------------------------------------------------------------------------- + +# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will +# evaluate all C-preprocessor directives found in the sources and include +# files. + +ENABLE_PREPROCESSING = YES + +# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro +# names in the source code. If set to NO (the default) only conditional +# compilation will be performed. Macro expansion can be done in a controlled +# way by setting EXPAND_ONLY_PREDEF to YES. + +MACRO_EXPANSION = NO + +# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES +# then the macro expansion is limited to the macros specified with the +# PREDEFINED and EXPAND_AS_DEFINED tags. + +EXPAND_ONLY_PREDEF = NO + +# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files +# in the INCLUDE_PATH (see below) will be search if a #include is found. + +SEARCH_INCLUDES = YES + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by +# the preprocessor. + +INCLUDE_PATH = + +# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard +# patterns (like *.h and *.hpp) to filter out the header-files in the +# directories. If left blank, the patterns specified with FILE_PATTERNS will +# be used. + +INCLUDE_FILE_PATTERNS = + +# The PREDEFINED tag can be used to specify one or more macro names that +# are defined before the preprocessor is started (similar to the -D option of +# gcc). The argument of the tag is a list of macros of the form: name +# or name=definition (no spaces). If the definition and the = are +# omitted =1 is assumed. To prevent a macro definition from being +# undefined via #undef or recursively expanded use the := operator +# instead of the = operator. + +PREDEFINED = DEBUG=2 __GNUC__=1 + +# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then +# this tag can be used to specify a list of macro names that should be expanded. +# The macro definition that is found in the sources will be used. +# Use the PREDEFINED tag if you want to use a different macro definition. + +EXPAND_AS_DEFINED = + +# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then +# doxygen's preprocessor will remove all function-like macros that are alone +# on a line, have an all uppercase name, and do not end with a semicolon. Such +# function macros are typically used for boiler-plate code, and will confuse +# the parser if not removed. + +SKIP_FUNCTION_MACROS = YES + +#--------------------------------------------------------------------------- +# Configuration::additions related to external references +#--------------------------------------------------------------------------- + +# The TAGFILES option can be used to specify one or more tagfiles. +# Optionally an initial location of the external documentation +# can be added for each tagfile. The format of a tag file without +# this location is as follows: +# +# TAGFILES = file1 file2 ... +# Adding location for the tag files is done as follows: +# +# TAGFILES = file1=loc1 "file2 = loc2" ... +# where "loc1" and "loc2" can be relative or absolute paths or +# URLs. If a location is present for each tag, the installdox tool +# does not have to be run to correct the links. +# Note that each tag file must have a unique name +# (where the name does NOT include the path) +# If a tag file is not located in the directory in which doxygen +# is run, you must also specify the path to the tagfile here. + +TAGFILES = tooltag=./man1 + +# When a file name is specified after GENERATE_TAGFILE, doxygen will create +# a tag file that is based on the input files it reads. + +GENERATE_TAGFILE = + +# If the ALLEXTERNALS tag is set to YES all external classes will be listed +# in the class index. If set to NO only the inherited external classes +# will be listed. + +ALLEXTERNALS = NO + +# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed +# in the modules index. If set to NO, only the current project's groups will +# be listed. + +EXTERNAL_GROUPS = YES + +# The PERL_PATH should be the absolute path and name of the perl script +# interpreter (i.e. the result of `which perl'). + +PERL_PATH = /usr/bin/perl + +#--------------------------------------------------------------------------- +# Configuration options related to the dot tool +#--------------------------------------------------------------------------- + +# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will +# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base +# or super classes. Setting the tag to NO turns the diagrams off. Note that +# this option is superseded by the HAVE_DOT option below. This is only a +# fallback. It is recommended to install and use dot, since it yields more +# powerful graphs. + +CLASS_DIAGRAMS = YES + +# You can define message sequence charts within doxygen comments using the \msc +# command. Doxygen will then run the mscgen tool (see +# http://www.mcternan.me.uk/mscgen/) to produce the chart and insert it in the +# documentation. The MSCGEN_PATH tag allows you to specify the directory where +# the mscgen tool resides. If left empty the tool is assumed to be found in the +# default search path. + +MSCGEN_PATH = + +# If set to YES, the inheritance and collaboration graphs will hide +# inheritance and usage relations if the target is undocumented +# or is not a class. + +HIDE_UNDOC_RELATIONS = YES + +# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is +# available from the path. This tool is part of Graphviz, a graph visualization +# toolkit from AT&T and Lucent Bell Labs. The other options in this section +# have no effect if this option is set to NO (the default) + +HAVE_DOT = NO + +# The DOT_NUM_THREADS specifies the number of dot invocations doxygen is +# allowed to run in parallel. When set to 0 (the default) doxygen will +# base this on the number of processors available in the system. You can set it +# explicitly to a value larger than 0 to get control over the balance +# between CPU load and processing speed. + +DOT_NUM_THREADS = 0 + +# By default doxygen will write a font called FreeSans.ttf to the output +# directory and reference it in all dot files that doxygen generates. This +# font does not include all possible unicode characters however, so when you need +# these (or just want a differently looking font) you can specify the font name +# using DOT_FONTNAME. You need need to make sure dot is able to find the font, +# which can be done by putting it in a standard location or by setting the +# DOTFONTPATH environment variable or by setting DOT_FONTPATH to the directory +# containing the font. + +DOT_FONTNAME = FreeSans.ttf + +# The DOT_FONTSIZE tag can be used to set the size of the font of dot graphs. +# The default size is 10pt. + +DOT_FONTSIZE = 10 + +# By default doxygen will tell dot to use the output directory to look for the +# FreeSans.ttf font (which doxygen will put there itself). If you specify a +# different font using DOT_FONTNAME you can set the path where dot +# can find it using this tag. + +DOT_FONTPATH = + +# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect inheritance relations. Setting this tag to YES will force the +# the CLASS_DIAGRAMS tag to NO. + +CLASS_GRAPH = YES + +# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect implementation dependencies (inheritance, containment, and +# class references variables) of the class with other documented classes. + +COLLABORATION_GRAPH = YES + +# If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for groups, showing the direct groups dependencies + +GROUP_GRAPHS = YES + +# If the UML_LOOK tag is set to YES doxygen will generate inheritance and +# collaboration diagrams in a style similar to the OMG's Unified Modeling +# Language. + +UML_LOOK = NO + +# If set to YES, the inheritance and collaboration graphs will show the +# relations between templates and their instances. + +TEMPLATE_RELATIONS = NO + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT +# tags are set to YES then doxygen will generate a graph for each documented +# file showing the direct and indirect include dependencies of the file with +# other documented files. + +INCLUDE_GRAPH = YES + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and +# HAVE_DOT tags are set to YES then doxygen will generate a graph for each +# documented header file showing the documented files that directly or +# indirectly include this file. + +INCLUDED_BY_GRAPH = YES + +# If the CALL_GRAPH and HAVE_DOT options are set to YES then +# doxygen will generate a call dependency graph for every global function +# or class method. Note that enabling this option will significantly increase +# the time of a run. So in most cases it will be better to enable call graphs +# for selected functions only using the \callgraph command. + +CALL_GRAPH = NO + +# If the CALLER_GRAPH and HAVE_DOT tags are set to YES then +# doxygen will generate a caller dependency graph for every global function +# or class method. Note that enabling this option will significantly increase +# the time of a run. So in most cases it will be better to enable caller +# graphs for selected functions only using the \callergraph command. + +CALLER_GRAPH = NO + +# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen +# will graphical hierarchy of all classes instead of a textual one. + +GRAPHICAL_HIERARCHY = YES + +# If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES +# then doxygen will show the dependencies a directory has on other directories +# in a graphical way. The dependency relations are determined by the #include +# relations between the files in the directories. + +DIRECTORY_GRAPH = YES + +# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images +# generated by dot. Possible values are png, jpg, or gif +# If left blank png will be used. + +DOT_IMAGE_FORMAT = png + +# The tag DOT_PATH can be used to specify the path where the dot tool can be +# found. If left blank, it is assumed the dot tool can be found in the path. + +DOT_PATH = + +# The DOTFILE_DIRS tag can be used to specify one or more directories that +# contain dot files that are included in the documentation (see the +# \dotfile command). + +DOTFILE_DIRS = + +# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of +# nodes that will be shown in the graph. If the number of nodes in a graph +# becomes larger than this value, doxygen will truncate the graph, which is +# visualized by representing a node as a red box. Note that doxygen if the +# number of direct children of the root node in a graph is already larger than +# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note +# that the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH. + +DOT_GRAPH_MAX_NODES = 50 + +# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the +# graphs generated by dot. A depth value of 3 means that only nodes reachable +# from the root by following a path via at most 3 edges will be shown. Nodes +# that lay further from the root node will be omitted. Note that setting this +# option to 1 or 2 may greatly reduce the computation time needed for large +# code bases. Also note that the size of a graph can be further restricted by +# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction. + +MAX_DOT_GRAPH_DEPTH = 0 + +# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent +# background. This is disabled by default, because dot on Windows does not +# seem to support this out of the box. Warning: Depending on the platform used, +# enabling this option may lead to badly anti-aliased labels on the edges of +# a graph (i.e. they become hard to read). + +DOT_TRANSPARENT = NO + +# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output +# files in one run (i.e. multiple -o and -T options on the command line). This +# makes dot run faster, but since only newer versions of dot (>1.8.10) +# support this, this feature is disabled by default. + +DOT_MULTI_TARGETS = YES + +# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will +# generate a legend page explaining the meaning of the various boxes and +# arrows in the dot generated graphs. + +GENERATE_LEGEND = YES + +# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will +# remove the intermediate dot files that are used to generate +# the various graphs. + +DOT_CLEANUP = YES diff --git a/external/db_drivers/liblmdb32/LICENSE b/external/db_drivers/liblmdb32/LICENSE new file mode 100644 index 000000000..05ad7571e --- /dev/null +++ b/external/db_drivers/liblmdb32/LICENSE @@ -0,0 +1,47 @@ +The OpenLDAP Public License + Version 2.8, 17 August 2003 + +Redistribution and use of this software and associated documentation +("Software"), with or without modification, are permitted provided +that the following conditions are met: + +1. Redistributions in source form must retain copyright statements + and notices, + +2. Redistributions in binary form must reproduce applicable copyright + statements and notices, this list of conditions, and the following + disclaimer in the documentation and/or other materials provided + with the distribution, and + +3. Redistributions must contain a verbatim copy of this document. + +The OpenLDAP Foundation may revise this license from time to time. +Each revision is distinguished by a version number. You may use +this Software under terms of this license revision or under the +terms of any subsequent revision of the license. + +THIS SOFTWARE IS PROVIDED BY THE OPENLDAP FOUNDATION AND ITS +CONTRIBUTORS ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT +SHALL THE OPENLDAP FOUNDATION, ITS CONTRIBUTORS, OR THE AUTHOR(S) +OR OWNER(S) OF THE SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +The names of the authors and copyright holders must not be used in +advertising or otherwise to promote the sale, use or other dealing +in this Software without specific, written prior permission. Title +to copyright in this Software shall at all times remain with copyright +holders. + +OpenLDAP is a registered trademark of the OpenLDAP Foundation. + +Copyright 1999-2003 The OpenLDAP Foundation, Redwood City, +California, USA. All Rights Reserved. Permission to copy and +distribute verbatim copies of this document is granted. diff --git a/external/db_drivers/liblmdb32/Makefile b/external/db_drivers/liblmdb32/Makefile new file mode 100644 index 000000000..095e65587 --- /dev/null +++ b/external/db_drivers/liblmdb32/Makefile @@ -0,0 +1,98 @@ +# Makefile for liblmdb (Lightning memory-mapped database library). + +######################################################################## +# Configuration. The compiler options must enable threaded compilation. +# +# Preprocessor macros (for CPPFLAGS) of interest... +# Note that the defaults should already be correct for most +# platforms; you should not need to change any of these. +# Read their descriptions in mdb.c if you do: +# +# - MDB_USE_POSIX_SEM +# - MDB_DSYNC +# - MDB_FDATASYNC +# - MDB_USE_PWRITEV +# +# There may be other macros in mdb.c of interest. You should +# read mdb.c before changing any of them. +# +CC = gcc +W = -W -Wall -Wno-unused-parameter -Wbad-function-cast -Wuninitialized +THREADS = -pthread +OPT = -O2 -g +CFLAGS = $(THREADS) $(OPT) $(W) $(XCFLAGS) +LDLIBS = +SOLIBS = +prefix = /usr/local +XCFLAGS = -DVL32 + +######################################################################## + +IHDRS = lmdb.h +ILIBS = liblmdb.a liblmdb.so +IPROGS = mdb_stat mdb_copy mdb_dump mdb_load +IDOCS = mdb_stat.1 mdb_copy.1 mdb_dump.1 mdb_load.1 +PROGS = $(IPROGS) mtest mtest2 mtest3 mtest4 mtest5 +all: $(ILIBS) $(PROGS) + +install: $(ILIBS) $(IPROGS) $(IHDRS) + for f in $(IPROGS); do cp $$f $(DESTDIR)$(prefix)/bin; done + for f in $(ILIBS); do cp $$f $(DESTDIR)$(prefix)/lib; done + for f in $(IHDRS); do cp $$f $(DESTDIR)$(prefix)/include; done + for f in $(IDOCS); do cp $$f $(DESTDIR)$(prefix)/man/man1; done + +clean: + rm -rf $(PROGS) *.[ao] *.so *~ testdb + +test: all + mkdir testdb + ./mtest && ./mdb_stat testdb + +liblmdb.a: mdb.o midl.o + ar rs $@ mdb.o midl.o + +liblmdb.so: mdb.o midl.o +# $(CC) $(LDFLAGS) -pthread -shared -Wl,-Bsymbolic -o $@ mdb.o midl.o $(SOLIBS) + $(CC) $(LDFLAGS) -pthread -shared -o $@ mdb.o midl.o $(SOLIBS) + +mdb_stat: mdb_stat.o liblmdb.a +mdb_copy: mdb_copy.o liblmdb.a +mdb_dump: mdb_dump.o liblmdb.a +mdb_load: mdb_load.o liblmdb.a +mtest: mtest.o liblmdb.a +mtest2: mtest2.o liblmdb.a +mtest3: mtest3.o liblmdb.a +mtest4: mtest4.o liblmdb.a +mtest5: mtest5.o liblmdb.a +mtest6: mtest6.o liblmdb.a + +mdb.o: mdb.c lmdb.h midl.h + $(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -c mdb.c + +midl.o: midl.c midl.h + $(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -c midl.c + +%: %.o + $(CC) $(CFLAGS) $(LDFLAGS) $^ $(LDLIBS) -o $@ + +%.o: %.c lmdb.h + $(CC) $(CFLAGS) $(CPPFLAGS) -c $< + +COV_FLAGS=-fprofile-arcs -ftest-coverage +COV_OBJS=xmdb.o xmidl.o + +coverage: xmtest + for i in mtest*.c [0-9]*.c; do j=`basename \$$i .c`; $(MAKE) $$j.o; \ + gcc -o x$$j $$j.o $(COV_OBJS) -pthread $(COV_FLAGS); \ + rm -rf testdb; mkdir testdb; ./x$$j; done + gcov xmdb.c + gcov xmidl.c + +xmtest: mtest.o xmdb.o xmidl.o + gcc -o xmtest mtest.o xmdb.o xmidl.o -pthread $(COV_FLAGS) + +xmdb.o: mdb.c lmdb.h midl.h + $(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -O0 $(COV_FLAGS) -c mdb.c -o $@ + +xmidl.o: midl.c midl.h + $(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -O0 $(COV_FLAGS) -c midl.c -o $@ diff --git a/external/db_drivers/liblmdb32/lmdb.h b/external/db_drivers/liblmdb32/lmdb.h new file mode 100644 index 000000000..bdbb0b909 --- /dev/null +++ b/external/db_drivers/liblmdb32/lmdb.h @@ -0,0 +1,1553 @@ +/** @file lmdb.h + * @brief Lightning memory-mapped database library + * + * @mainpage Lightning Memory-Mapped Database Manager (LMDB) + * + * @section intro_sec Introduction + * LMDB is a Btree-based database management library modeled loosely on the + * BerkeleyDB API, but much simplified. The entire database is exposed + * in a memory map, and all data fetches return data directly + * from the mapped memory, so no malloc's or memcpy's occur during + * data fetches. As such, the library is extremely simple because it + * requires no page caching layer of its own, and it is extremely high + * performance and memory-efficient. It is also fully transactional with + * full ACID semantics, and when the memory map is read-only, the + * database integrity cannot be corrupted by stray pointer writes from + * application code. + * + * The library is fully thread-aware and supports concurrent read/write + * access from multiple processes and threads. Data pages use a copy-on- + * write strategy so no active data pages are ever overwritten, which + * also provides resistance to corruption and eliminates the need of any + * special recovery procedures after a system crash. Writes are fully + * serialized; only one write transaction may be active at a time, which + * guarantees that writers can never deadlock. The database structure is + * multi-versioned so readers run with no locks; writers cannot block + * readers, and readers don't block writers. + * + * Unlike other well-known database mechanisms which use either write-ahead + * transaction logs or append-only data writes, LMDB requires no maintenance + * during operation. Both write-ahead loggers and append-only databases + * require periodic checkpointing and/or compaction of their log or database + * files otherwise they grow without bound. LMDB tracks free pages within + * the database and re-uses them for new write operations, so the database + * size does not grow without bound in normal use. + * + * The memory map can be used as a read-only or read-write map. It is + * read-only by default as this provides total immunity to corruption. + * Using read-write mode offers much higher write performance, but adds + * the possibility for stray application writes thru pointers to silently + * corrupt the database. Of course if your application code is known to + * be bug-free (...) then this is not an issue. + * + * @section caveats_sec Caveats + * Troubleshooting the lock file, plus semaphores on BSD systems: + * + * - A broken lockfile can cause sync issues. + * Stale reader transactions left behind by an aborted program + * cause further writes to grow the database quickly, and + * stale locks can block further operation. + * + * Fix: Check for stale readers periodically, using the + * #mdb_reader_check function or the \ref mdb_stat_1 "mdb_stat" tool. Or just + * make all programs using the database close it; the lockfile + * is always reset on first open of the environment. + * + * - On BSD systems or others configured with MDB_USE_POSIX_SEM, + * startup can fail due to semaphores owned by another userid. + * + * Fix: Open and close the database as the user which owns the + * semaphores (likely last user) or as root, while no other + * process is using the database. + * + * Restrictions/caveats (in addition to those listed for some functions): + * + * - Only the database owner should normally use the database on + * BSD systems or when otherwise configured with MDB_USE_POSIX_SEM. + * Multiple users can cause startup to fail later, as noted above. + * + * - There is normally no pure read-only mode, since readers need write + * access to locks and lock file. Exceptions: On read-only filesystems + * or with the #MDB_NOLOCK flag described under #mdb_env_open(). + * + * - By default, in versions before 0.9.10, unused portions of the data + * file might receive garbage data from memory freed by other code. + * (This does not happen when using the #MDB_WRITEMAP flag.) As of + * 0.9.10 the default behavior is to initialize such memory before + * writing to the data file. Since there may be a slight performance + * cost due to this initialization, applications may disable it using + * the #MDB_NOMEMINIT flag. Applications handling sensitive data + * which must not be written should not use this flag. This flag is + * irrelevant when using #MDB_WRITEMAP. + * + * - A thread can only use one transaction at a time, plus any child + * transactions. Each transaction belongs to one thread. See below. + * The #MDB_NOTLS flag changes this for read-only transactions. + * + * - Use an MDB_env* in the process which opened it, without fork()ing. + * + * - Do not have open an LMDB database twice in the same process at + * the same time. Not even from a plain open() call - close()ing it + * breaks flock() advisory locking. + * + * - Avoid long-lived transactions. Read transactions prevent + * reuse of pages freed by newer write transactions, thus the + * database can grow quickly. Write transactions prevent + * other write transactions, since writes are serialized. + * + * - Avoid suspending a process with active transactions. These + * would then be "long-lived" as above. Also read transactions + * suspended when writers commit could sometimes see wrong data. + * + * ...when several processes can use a database concurrently: + * + * - Avoid aborting a process with an active transaction. + * The transaction becomes "long-lived" as above until a check + * for stale readers is performed or the lockfile is reset, + * since the process may not remove it from the lockfile. + * + * - If you do that anyway, do a periodic check for stale readers. Or + * close the environment once in a while, so the lockfile can get reset. + * + * - Do not use LMDB databases on remote filesystems, even between + * processes on the same host. This breaks flock() on some OSes, + * possibly memory map sync, and certainly sync between programs + * on different hosts. + * + * - Opening a database can fail if another process is opening or + * closing it at exactly the same time. + * + * @author Howard Chu, Symas Corporation. + * + * @copyright Copyright 2011-2014 Howard Chu, Symas Corp. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + * + * @par Derived From: + * This code is derived from btree.c written by Martin Hedenfalk. + * + * Copyright (c) 2009, 2010 Martin Hedenfalk <martin@bzero.se> + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#ifndef _LMDB_H_ +#define _LMDB_H_ + +#include <sys/types.h> + +#ifdef __cplusplus +extern "C" { +#endif + +/** Unix permissions for creating files, or dummy definition for Windows */ +#ifdef _MSC_VER +typedef int mdb_mode_t; +#else +typedef mode_t mdb_mode_t; +#endif + +/** An abstraction for a file handle. + * On POSIX systems file handles are small integers. On Windows + * they're opaque pointers. + */ +#ifdef _WIN32 +typedef void *mdb_filehandle_t; +#else +typedef int mdb_filehandle_t; +#endif + +/** @defgroup mdb LMDB API + * @{ + * @brief OpenLDAP Lightning Memory-Mapped Database Manager + */ +/** @defgroup Version Version Macros + * @{ + */ +/** Library major version */ +#define MDB_VERSION_MAJOR 0 +/** Library minor version */ +#define MDB_VERSION_MINOR 9 +/** Library patch version */ +#define MDB_VERSION_PATCH 14 + +/** Combine args a,b,c into a single integer for easy version comparisons */ +#define MDB_VERINT(a,b,c) (((a) << 24) | ((b) << 16) | (c)) + +/** The full library version as a single integer */ +#define MDB_VERSION_FULL \ + MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH) + +/** The release date of this library version */ +#define MDB_VERSION_DATE "September 20, 2014" + +/** A stringifier for the version info */ +#define MDB_VERSTR(a,b,c,d) "LMDB " #a "." #b "." #c ": (" d ")" + +/** A helper for the stringifier macro */ +#define MDB_VERFOO(a,b,c,d) MDB_VERSTR(a,b,c,d) + +/** The full library version as a C string */ +#define MDB_VERSION_STRING \ + MDB_VERFOO(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH,MDB_VERSION_DATE) +/** @} */ + +/** @brief Opaque structure for a database environment. + * + * A DB environment supports multiple databases, all residing in the same + * shared-memory map. + */ +typedef struct MDB_env MDB_env; + +/** @brief Opaque structure for a transaction handle. + * + * All database operations require a transaction handle. Transactions may be + * read-only or read-write. + */ +typedef struct MDB_txn MDB_txn; + +/** @brief A handle for an individual database in the DB environment. */ +typedef unsigned int MDB_dbi; + +/** @brief Opaque structure for navigating through a database */ +typedef struct MDB_cursor MDB_cursor; + +/** @brief Generic structure used for passing keys and data in and out + * of the database. + * + * Values returned from the database are valid only until a subsequent + * update operation, or the end of the transaction. Do not modify or + * free them, they commonly point into the database itself. + * + * Key sizes must be between 1 and #mdb_env_get_maxkeysize() inclusive. + * The same applies to data sizes in databases with the #MDB_DUPSORT flag. + * Other data items can in theory be from 0 to 0xffffffff bytes long. + */ +typedef struct MDB_val { + size_t mv_size; /**< size of the data item */ + void *mv_data; /**< address of the data item */ +} MDB_val; + +/** @brief A callback function used to compare two keys in a database */ +typedef int (MDB_cmp_func)(const MDB_val *a, const MDB_val *b); + +/** @brief A callback function used to relocate a position-dependent data item + * in a fixed-address database. + * + * The \b newptr gives the item's desired address in + * the memory map, and \b oldptr gives its previous address. The item's actual + * data resides at the address in \b item. This callback is expected to walk + * through the fields of the record in \b item and modify any + * values based at the \b oldptr address to be relative to the \b newptr address. + * @param[in,out] item The item that is to be relocated. + * @param[in] oldptr The previous address. + * @param[in] newptr The new address to relocate to. + * @param[in] relctx An application-provided context, set by #mdb_set_relctx(). + * @todo This feature is currently unimplemented. + */ +typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *relctx); + +/** @defgroup mdb_env Environment Flags + * @{ + */ + /** mmap at a fixed address (experimental) */ +#define MDB_FIXEDMAP 0x01 + /** no environment directory */ +#define MDB_NOSUBDIR 0x4000 + /** don't fsync after commit */ +#define MDB_NOSYNC 0x10000 + /** read only */ +#define MDB_RDONLY 0x20000 + /** don't fsync metapage after commit */ +#define MDB_NOMETASYNC 0x40000 + /** use writable mmap */ +#define MDB_WRITEMAP 0x80000 + /** use asynchronous msync when #MDB_WRITEMAP is used */ +#define MDB_MAPASYNC 0x100000 + /** tie reader locktable slots to #MDB_txn objects instead of to threads */ +#define MDB_NOTLS 0x200000 + /** don't do any locking, caller must manage their own locks */ +#define MDB_NOLOCK 0x400000 + /** don't do readahead (no effect on Windows) */ +#define MDB_NORDAHEAD 0x800000 + /** don't initialize malloc'd memory before writing to datafile */ +#define MDB_NOMEMINIT 0x1000000 +/** @} */ + +/** @defgroup mdb_dbi_open Database Flags + * @{ + */ + /** use reverse string keys */ +#define MDB_REVERSEKEY 0x02 + /** use sorted duplicates */ +#define MDB_DUPSORT 0x04 + /** numeric keys in native byte order. + * The keys must all be of the same size. */ +#define MDB_INTEGERKEY 0x08 + /** with #MDB_DUPSORT, sorted dup items have fixed size */ +#define MDB_DUPFIXED 0x10 + /** with #MDB_DUPSORT, dups are numeric in native byte order */ +#define MDB_INTEGERDUP 0x20 + /** with #MDB_DUPSORT, use reverse string dups */ +#define MDB_REVERSEDUP 0x40 + /** create DB if not already existing */ +#define MDB_CREATE 0x40000 +/** @} */ + +/** @defgroup mdb_put Write Flags + * @{ + */ +/** For put: Don't write if the key already exists. */ +#define MDB_NOOVERWRITE 0x10 +/** Only for #MDB_DUPSORT<br> + * For put: don't write if the key and data pair already exist.<br> + * For mdb_cursor_del: remove all duplicate data items. + */ +#define MDB_NODUPDATA 0x20 +/** For mdb_cursor_put: overwrite the current key/data pair */ +#define MDB_CURRENT 0x40 +/** For put: Just reserve space for data, don't copy it. Return a + * pointer to the reserved space. + */ +#define MDB_RESERVE 0x10000 +/** Data is being appended, don't split full pages. */ +#define MDB_APPEND 0x20000 +/** Duplicate data is being appended, don't split full pages. */ +#define MDB_APPENDDUP 0x40000 +/** Store multiple data items in one call. Only for #MDB_DUPFIXED. */ +#define MDB_MULTIPLE 0x80000 +/* @} */ + +/** @defgroup mdb_copy Copy Flags + * @{ + */ +/** Compacting copy: Omit free space from copy, and renumber all + * pages sequentially. + */ +#define MDB_CP_COMPACT 0x01 +/* @} */ + +/** @brief Cursor Get operations. + * + * This is the set of all operations for retrieving data + * using a cursor. + */ +typedef enum MDB_cursor_op { + MDB_FIRST, /**< Position at first key/data item */ + MDB_FIRST_DUP, /**< Position at first data item of current key. + Only for #MDB_DUPSORT */ + MDB_GET_BOTH, /**< Position at key/data pair. Only for #MDB_DUPSORT */ + MDB_GET_BOTH_RANGE, /**< position at key, nearest data. Only for #MDB_DUPSORT */ + MDB_GET_CURRENT, /**< Return key/data at current cursor position */ + MDB_GET_MULTIPLE, /**< Return key and up to a page of duplicate data items + from current cursor position. Move cursor to prepare + for #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED */ + MDB_LAST, /**< Position at last key/data item */ + MDB_LAST_DUP, /**< Position at last data item of current key. + Only for #MDB_DUPSORT */ + MDB_NEXT, /**< Position at next data item */ + MDB_NEXT_DUP, /**< Position at next data item of current key. + Only for #MDB_DUPSORT */ + MDB_NEXT_MULTIPLE, /**< Return key and up to a page of duplicate data items + from next cursor position. Move cursor to prepare + for #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED */ + MDB_NEXT_NODUP, /**< Position at first data item of next key */ + MDB_PREV, /**< Position at previous data item */ + MDB_PREV_DUP, /**< Position at previous data item of current key. + Only for #MDB_DUPSORT */ + MDB_PREV_NODUP, /**< Position at last data item of previous key */ + MDB_SET, /**< Position at specified key */ + MDB_SET_KEY, /**< Position at specified key, return key + data */ + MDB_SET_RANGE /**< Position at first key greater than or equal to specified key. */ +} MDB_cursor_op; + +/** @defgroup errors Return Codes + * + * BerkeleyDB uses -30800 to -30999, we'll go under them + * @{ + */ + /** Successful result */ +#define MDB_SUCCESS 0 + /** key/data pair already exists */ +#define MDB_KEYEXIST (-30799) + /** key/data pair not found (EOF) */ +#define MDB_NOTFOUND (-30798) + /** Requested page not found - this usually indicates corruption */ +#define MDB_PAGE_NOTFOUND (-30797) + /** Located page was wrong type */ +#define MDB_CORRUPTED (-30796) + /** Update of meta page failed, probably I/O error */ +#define MDB_PANIC (-30795) + /** Environment version mismatch */ +#define MDB_VERSION_MISMATCH (-30794) + /** File is not a valid LMDB file */ +#define MDB_INVALID (-30793) + /** Environment mapsize reached */ +#define MDB_MAP_FULL (-30792) + /** Environment maxdbs reached */ +#define MDB_DBS_FULL (-30791) + /** Environment maxreaders reached */ +#define MDB_READERS_FULL (-30790) + /** Too many TLS keys in use - Windows only */ +#define MDB_TLS_FULL (-30789) + /** Txn has too many dirty pages */ +#define MDB_TXN_FULL (-30788) + /** Cursor stack too deep - internal error */ +#define MDB_CURSOR_FULL (-30787) + /** Page has not enough space - internal error */ +#define MDB_PAGE_FULL (-30786) + /** Database contents grew beyond environment mapsize */ +#define MDB_MAP_RESIZED (-30785) + /** MDB_INCOMPATIBLE: Operation and DB incompatible, or DB flags changed */ +#define MDB_INCOMPATIBLE (-30784) + /** Invalid reuse of reader locktable slot */ +#define MDB_BAD_RSLOT (-30783) + /** Transaction cannot recover - it must be aborted */ +#define MDB_BAD_TXN (-30782) + /** Unsupported size of key/DB name/data, or wrong DUPFIXED size */ +#define MDB_BAD_VALSIZE (-30781) + /** The specified DBI was changed unexpectedly */ +#define MDB_BAD_DBI (-30780) + /** The last defined error code */ +#define MDB_LAST_ERRCODE MDB_BAD_DBI +/** @} */ + +/** @brief Statistics for a database in the environment */ +typedef struct MDB_stat { + unsigned int ms_psize; /**< Size of a database page. + This is currently the same for all databases. */ + unsigned int ms_depth; /**< Depth (height) of the B-tree */ + size_t ms_branch_pages; /**< Number of internal (non-leaf) pages */ + size_t ms_leaf_pages; /**< Number of leaf pages */ + size_t ms_overflow_pages; /**< Number of overflow pages */ + size_t ms_entries; /**< Number of data items */ +} MDB_stat; + +/** @brief Information about the environment */ +typedef struct MDB_envinfo { + void *me_mapaddr; /**< Address of map, if fixed */ + size_t me_mapsize; /**< Size of the data memory map */ + size_t me_last_pgno; /**< ID of the last used page */ + size_t me_last_txnid; /**< ID of the last committed transaction */ + unsigned int me_maxreaders; /**< max reader slots in the environment */ + unsigned int me_numreaders; /**< max reader slots used in the environment */ +} MDB_envinfo; + + /** @brief Return the LMDB library version information. + * + * @param[out] major if non-NULL, the library major version number is copied here + * @param[out] minor if non-NULL, the library minor version number is copied here + * @param[out] patch if non-NULL, the library patch version number is copied here + * @retval "version string" The library version as a string + */ +char *mdb_version(int *major, int *minor, int *patch); + + /** @brief Return a string describing a given error code. + * + * This function is a superset of the ANSI C X3.159-1989 (ANSI C) strerror(3) + * function. If the error code is greater than or equal to 0, then the string + * returned by the system function strerror(3) is returned. If the error code + * is less than 0, an error string corresponding to the LMDB library error is + * returned. See @ref errors for a list of LMDB-specific error codes. + * @param[in] err The error code + * @retval "error message" The description of the error + */ +char *mdb_strerror(int err); + + /** @brief Create an LMDB environment handle. + * + * This function allocates memory for a #MDB_env structure. To release + * the allocated memory and discard the handle, call #mdb_env_close(). + * Before the handle may be used, it must be opened using #mdb_env_open(). + * Various other options may also need to be set before opening the handle, + * e.g. #mdb_env_set_mapsize(), #mdb_env_set_maxreaders(), #mdb_env_set_maxdbs(), + * depending on usage requirements. + * @param[out] env The address where the new handle will be stored + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_create(MDB_env **env); + + /** @brief Open an environment handle. + * + * If this function fails, #mdb_env_close() must be called to discard the #MDB_env handle. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] path The directory in which the database files reside. This + * directory must already exist and be writable. + * @param[in] flags Special options for this environment. This parameter + * must be set to 0 or by bitwise OR'ing together one or more of the + * values described here. + * Flags set by mdb_env_set_flags() are also used. + * <ul> + * <li>#MDB_FIXEDMAP + * use a fixed address for the mmap region. This flag must be specified + * when creating the environment, and is stored persistently in the environment. + * If successful, the memory map will always reside at the same virtual address + * and pointers used to reference data items in the database will be constant + * across multiple invocations. This option may not always work, depending on + * how the operating system has allocated memory to shared libraries and other uses. + * The feature is highly experimental. + * <li>#MDB_NOSUBDIR + * By default, LMDB creates its environment in a directory whose + * pathname is given in \b path, and creates its data and lock files + * under that directory. With this option, \b path is used as-is for + * the database main data file. The database lock file is the \b path + * with "-lock" appended. + * <li>#MDB_RDONLY + * Open the environment in read-only mode. No write operations will be + * allowed. LMDB will still modify the lock file - except on read-only + * filesystems, where LMDB does not use locks. + * <li>#MDB_WRITEMAP + * Use a writeable memory map unless MDB_RDONLY is set. This is faster + * and uses fewer mallocs, but loses protection from application bugs + * like wild pointer writes and other bad updates into the database. + * Incompatible with nested transactions. + * Processes with and without MDB_WRITEMAP on the same environment do + * not cooperate well. + * <li>#MDB_NOMETASYNC + * Flush system buffers to disk only once per transaction, omit the + * metadata flush. Defer that until the system flushes files to disk, + * or next non-MDB_RDONLY commit or #mdb_env_sync(). This optimization + * maintains database integrity, but a system crash may undo the last + * committed transaction. I.e. it preserves the ACI (atomicity, + * consistency, isolation) but not D (durability) database property. + * This flag may be changed at any time using #mdb_env_set_flags(). + * <li>#MDB_NOSYNC + * Don't flush system buffers to disk when committing a transaction. + * This optimization means a system crash can corrupt the database or + * lose the last transactions if buffers are not yet flushed to disk. + * The risk is governed by how often the system flushes dirty buffers + * to disk and how often #mdb_env_sync() is called. However, if the + * filesystem preserves write order and the #MDB_WRITEMAP flag is not + * used, transactions exhibit ACI (atomicity, consistency, isolation) + * properties and only lose D (durability). I.e. database integrity + * is maintained, but a system crash may undo the final transactions. + * Note that (#MDB_NOSYNC | #MDB_WRITEMAP) leaves the system with no + * hint for when to write transactions to disk, unless #mdb_env_sync() + * is called. (#MDB_MAPASYNC | #MDB_WRITEMAP) may be preferable. + * This flag may be changed at any time using #mdb_env_set_flags(). + * <li>#MDB_MAPASYNC + * When using #MDB_WRITEMAP, use asynchronous flushes to disk. + * As with #MDB_NOSYNC, a system crash can then corrupt the + * database or lose the last transactions. Calling #mdb_env_sync() + * ensures on-disk database integrity until next commit. + * This flag may be changed at any time using #mdb_env_set_flags(). + * <li>#MDB_NOTLS + * Don't use Thread-Local Storage. Tie reader locktable slots to + * #MDB_txn objects instead of to threads. I.e. #mdb_txn_reset() keeps + * the slot reseved for the #MDB_txn object. A thread may use parallel + * read-only transactions. A read-only transaction may span threads if + * the user synchronizes its use. Applications that multiplex many + * user threads over individual OS threads need this option. Such an + * application must also serialize the write transactions in an OS + * thread, since LMDB's write locking is unaware of the user threads. + * <li>#MDB_NOLOCK + * Don't do any locking. If concurrent access is anticipated, the + * caller must manage all concurrency itself. For proper operation + * the caller must enforce single-writer semantics, and must ensure + * that no readers are using old transactions while a writer is + * active. The simplest approach is to use an exclusive lock so that + * no readers may be active at all when a writer begins. + * <li>#MDB_NORDAHEAD + * Turn off readahead. Most operating systems perform readahead on + * read requests by default. This option turns it off if the OS + * supports it. Turning it off may help random read performance + * when the DB is larger than RAM and system RAM is full. + * The option is not implemented on Windows. + * <li>#MDB_NOMEMINIT + * Don't initialize malloc'd memory before writing to unused spaces + * in the data file. By default, memory for pages written to the data + * file is obtained using malloc. While these pages may be reused in + * subsequent transactions, freshly malloc'd pages will be initialized + * to zeroes before use. This avoids persisting leftover data from other + * code (that used the heap and subsequently freed the memory) into the + * data file. Note that many other system libraries may allocate + * and free memory from the heap for arbitrary uses. E.g., stdio may + * use the heap for file I/O buffers. This initialization step has a + * modest performance cost so some applications may want to disable + * it using this flag. This option can be a problem for applications + * which handle sensitive data like passwords, and it makes memory + * checkers like Valgrind noisy. This flag is not needed with #MDB_WRITEMAP, + * which writes directly to the mmap instead of using malloc for pages. The + * initialization is also skipped if #MDB_RESERVE is used; the + * caller is expected to overwrite all of the memory that was + * reserved in that case. + * This flag may be changed at any time using #mdb_env_set_flags(). + * </ul> + * @param[in] mode The UNIX permissions to set on created files. This parameter + * is ignored on Windows. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>#MDB_VERSION_MISMATCH - the version of the LMDB library doesn't match the + * version that created the database environment. + * <li>#MDB_INVALID - the environment file headers are corrupted. + * <li>ENOENT - the directory specified by the path parameter doesn't exist. + * <li>EACCES - the user didn't have permission to access the environment files. + * <li>EAGAIN - the environment was locked by another process. + * </ul> + */ +int mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode); + + /** @brief Copy an LMDB environment to the specified path. + * + * This function may be used to make a backup of an existing environment. + * No lockfile is created, since it gets recreated at need. + * @note This call can trigger significant file size growth if run in + * parallel with write transactions, because it employs a read-only + * transaction. See long-lived transactions under @ref caveats_sec. + * @param[in] env An environment handle returned by #mdb_env_create(). It + * must have already been opened successfully. + * @param[in] path The directory in which the copy will reside. This + * directory must already exist and be writable but must otherwise be + * empty. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_copy(MDB_env *env, const char *path); + + /** @brief Copy an LMDB environment to the specified file descriptor. + * + * This function may be used to make a backup of an existing environment. + * No lockfile is created, since it gets recreated at need. + * @note This call can trigger significant file size growth if run in + * parallel with write transactions, because it employs a read-only + * transaction. See long-lived transactions under @ref caveats_sec. + * @param[in] env An environment handle returned by #mdb_env_create(). It + * must have already been opened successfully. + * @param[in] fd The filedescriptor to write the copy to. It must + * have already been opened for Write access. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_copyfd(MDB_env *env, mdb_filehandle_t fd); + + /** @brief Copy an LMDB environment to the specified path, with options. + * + * This function may be used to make a backup of an existing environment. + * No lockfile is created, since it gets recreated at need. + * @note This call can trigger significant file size growth if run in + * parallel with write transactions, because it employs a read-only + * transaction. See long-lived transactions under @ref caveats_sec. + * @param[in] env An environment handle returned by #mdb_env_create(). It + * must have already been opened successfully. + * @param[in] path The directory in which the copy will reside. This + * directory must already exist and be writable but must otherwise be + * empty. + * @param[in] flags Special options for this operation. This parameter + * must be set to 0 or by bitwise OR'ing together one or more of the + * values described here. + * <ul> + * <li>#MDB_CP_COMPACT - Perform compaction while copying: omit free + * pages and sequentially renumber all pages in output. This option + * consumes more CPU and runs more slowly than the default. + * </ul> + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags); + + /** @brief Copy an LMDB environment to the specified file descriptor, + * with options. + * + * This function may be used to make a backup of an existing environment. + * No lockfile is created, since it gets recreated at need. See + * #mdb_env_copy2() for further details. + * @note This call can trigger significant file size growth if run in + * parallel with write transactions, because it employs a read-only + * transaction. See long-lived transactions under @ref caveats_sec. + * @param[in] env An environment handle returned by #mdb_env_create(). It + * must have already been opened successfully. + * @param[in] fd The filedescriptor to write the copy to. It must + * have already been opened for Write access. + * @param[in] flags Special options for this operation. + * See #mdb_env_copy2() for options. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_copyfd2(MDB_env *env, mdb_filehandle_t fd, unsigned int flags); + + /** @brief Return statistics about the LMDB environment. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] stat The address of an #MDB_stat structure + * where the statistics will be copied + */ +int mdb_env_stat(MDB_env *env, MDB_stat *stat); + + /** @brief Return information about the LMDB environment. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] stat The address of an #MDB_envinfo structure + * where the information will be copied + */ +int mdb_env_info(MDB_env *env, MDB_envinfo *stat); + + /** @brief Flush the data buffers to disk. + * + * Data is always written to disk when #mdb_txn_commit() is called, + * but the operating system may keep it buffered. LMDB always flushes + * the OS buffers upon commit as well, unless the environment was + * opened with #MDB_NOSYNC or in part #MDB_NOMETASYNC. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] force If non-zero, force a synchronous flush. Otherwise + * if the environment has the #MDB_NOSYNC flag set the flushes + * will be omitted, and with #MDB_MAPASYNC they will be asynchronous. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified. + * <li>EIO - an error occurred during synchronization. + * </ul> + */ +int mdb_env_sync(MDB_env *env, int force); + + /** @brief Close the environment and release the memory map. + * + * Only a single thread may call this function. All transactions, databases, + * and cursors must already be closed before calling this function. Attempts to + * use any such handles after calling this function will cause a SIGSEGV. + * The environment handle will be freed and must not be used again after this call. + * @param[in] env An environment handle returned by #mdb_env_create() + */ +void mdb_env_close(MDB_env *env); + + /** @brief Set environment flags. + * + * This may be used to set some flags in addition to those from + * #mdb_env_open(), or to unset these flags. If several threads + * change the flags at the same time, the result is undefined. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] flags The flags to change, bitwise OR'ed together + * @param[in] onoff A non-zero value sets the flags, zero clears them. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_env_set_flags(MDB_env *env, unsigned int flags, int onoff); + + /** @brief Get environment flags. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] flags The address of an integer to store the flags + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_env_get_flags(MDB_env *env, unsigned int *flags); + + /** @brief Return the path that was used in #mdb_env_open(). + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] path Address of a string pointer to contain the path. This + * is the actual string in the environment, not a copy. It should not be + * altered in any way. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_env_get_path(MDB_env *env, const char **path); + + /** @brief Return the filedescriptor for the given environment. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] fd Address of a mdb_filehandle_t to contain the descriptor. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_env_get_fd(MDB_env *env, mdb_filehandle_t *fd); + + /** @brief Set the size of the memory map to use for this environment. + * + * The size should be a multiple of the OS page size. The default is + * 10485760 bytes. The size of the memory map is also the maximum size + * of the database. The value should be chosen as large as possible, + * to accommodate future growth of the database. + * This function should be called after #mdb_env_create() and before #mdb_env_open(). + * It may be called at later times if no transactions are active in + * this process. Note that the library does not check for this condition, + * the caller must ensure it explicitly. + * + * The new size takes effect immediately for the current process but + * will not be persisted to any others until a write transaction has been + * committed by the current process. Also, only mapsize increases are + * persisted into the environment. + * + * If the mapsize is increased by another process, and data has grown + * beyond the range of the current mapsize, #mdb_txn_begin() will + * return #MDB_MAP_RESIZED. This function may be called with a size + * of zero to adopt the new size. + * + * Any attempt to set a size smaller than the space already consumed + * by the environment will be silently changed to the current size of the used space. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] size The size in bytes + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified, or the environment has + * an active write transaction. + * </ul> + */ +int mdb_env_set_mapsize(MDB_env *env, size_t size); + + /** @brief Set the maximum number of threads/reader slots for the environment. + * + * This defines the number of slots in the lock table that is used to track readers in the + * the environment. The default is 126. + * Starting a read-only transaction normally ties a lock table slot to the + * current thread until the environment closes or the thread exits. If + * MDB_NOTLS is in use, #mdb_txn_begin() instead ties the slot to the + * MDB_txn object until it or the #MDB_env object is destroyed. + * This function may only be called after #mdb_env_create() and before #mdb_env_open(). + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] readers The maximum number of reader lock table slots + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified, or the environment is already open. + * </ul> + */ +int mdb_env_set_maxreaders(MDB_env *env, unsigned int readers); + + /** @brief Get the maximum number of threads/reader slots for the environment. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] readers Address of an integer to store the number of readers + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_env_get_maxreaders(MDB_env *env, unsigned int *readers); + + /** @brief Set the maximum number of named databases for the environment. + * + * This function is only needed if multiple databases will be used in the + * environment. Simpler applications that use the environment as a single + * unnamed database can ignore this option. + * This function may only be called after #mdb_env_create() and before #mdb_env_open(). + * + * Currently a moderate number of slots are cheap but a huge number gets + * expensive: 7-120 words per transaction, and every #mdb_dbi_open() + * does a linear search of the opened slots. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] dbs The maximum number of databases + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified, or the environment is already open. + * </ul> + */ +int mdb_env_set_maxdbs(MDB_env *env, MDB_dbi dbs); + + /** @brief Get the maximum size of keys and #MDB_DUPSORT data we can write. + * + * Depends on the compile-time constant #MDB_MAXKEYSIZE. Default 511. + * See @ref MDB_val. + * @param[in] env An environment handle returned by #mdb_env_create() + * @return The maximum size of a key we can write + */ +int mdb_env_get_maxkeysize(MDB_env *env); + + /** @brief Set application information associated with the #MDB_env. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] ctx An arbitrary pointer for whatever the application needs. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_set_userctx(MDB_env *env, void *ctx); + + /** @brief Get the application information associated with the #MDB_env. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @return The pointer set by #mdb_env_set_userctx(). + */ +void *mdb_env_get_userctx(MDB_env *env); + + /** @brief A callback function for most LMDB assert() failures, + * called before printing the message and aborting. + * + * @param[in] env An environment handle returned by #mdb_env_create(). + * @param[in] msg The assertion message, not including newline. + */ +typedef void MDB_assert_func(MDB_env *env, const char *msg); + + /** Set or reset the assert() callback of the environment. + * Disabled if liblmdb is buillt with NDEBUG. + * @note This hack should become obsolete as lmdb's error handling matures. + * @param[in] env An environment handle returned by #mdb_env_create(). + * @param[in] func An #MDB_assert_func function, or 0. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_set_assert(MDB_env *env, MDB_assert_func *func); + + /** @brief Create a transaction for use with the environment. + * + * The transaction handle may be discarded using #mdb_txn_abort() or #mdb_txn_commit(). + * @note A transaction and its cursors must only be used by a single + * thread, and a thread may only have a single transaction at a time. + * If #MDB_NOTLS is in use, this does not apply to read-only transactions. + * @note Cursors may not span transactions. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] parent If this parameter is non-NULL, the new transaction + * will be a nested transaction, with the transaction indicated by \b parent + * as its parent. Transactions may be nested to any level. A parent + * transaction and its cursors may not issue any other operations than + * mdb_txn_commit and mdb_txn_abort while it has active child transactions. + * @param[in] flags Special options for this transaction. This parameter + * must be set to 0 or by bitwise OR'ing together one or more of the + * values described here. + * <ul> + * <li>#MDB_RDONLY + * This transaction will not perform any write operations. + * </ul> + * @param[out] txn Address where the new #MDB_txn handle will be stored + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>#MDB_PANIC - a fatal error occurred earlier and the environment + * must be shut down. + * <li>#MDB_MAP_RESIZED - another process wrote data beyond this MDB_env's + * mapsize and this environment's map must be resized as well. + * See #mdb_env_set_mapsize(). + * <li>#MDB_READERS_FULL - a read-only transaction was requested and + * the reader lock table is full. See #mdb_env_set_maxreaders(). + * <li>ENOMEM - out of memory. + * </ul> + */ +int mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **txn); + + /** @brief Returns the transaction's #MDB_env + * + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + */ +MDB_env *mdb_txn_env(MDB_txn *txn); + + /** @brief Commit all the operations of a transaction into the database. + * + * The transaction handle is freed. It and its cursors must not be used + * again after this call, except with #mdb_cursor_renew(). + * @note Earlier documentation incorrectly said all cursors would be freed. + * Only write-transactions free cursors. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified. + * <li>ENOSPC - no more disk space. + * <li>EIO - a low-level I/O error occurred while writing. + * <li>ENOMEM - out of memory. + * </ul> + */ +int mdb_txn_commit(MDB_txn *txn); + + /** @brief Abandon all the operations of the transaction instead of saving them. + * + * The transaction handle is freed. It and its cursors must not be used + * again after this call, except with #mdb_cursor_renew(). + * @note Earlier documentation incorrectly said all cursors would be freed. + * Only write-transactions free cursors. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + */ +void mdb_txn_abort(MDB_txn *txn); + + /** @brief Reset a read-only transaction. + * + * Abort the transaction like #mdb_txn_abort(), but keep the transaction + * handle. #mdb_txn_renew() may reuse the handle. This saves allocation + * overhead if the process will start a new read-only transaction soon, + * and also locking overhead if #MDB_NOTLS is in use. The reader table + * lock is released, but the table slot stays tied to its thread or + * #MDB_txn. Use mdb_txn_abort() to discard a reset handle, and to free + * its lock table slot if MDB_NOTLS is in use. + * Cursors opened within the transaction must not be used + * again after this call, except with #mdb_cursor_renew(). + * Reader locks generally don't interfere with writers, but they keep old + * versions of database pages allocated. Thus they prevent the old pages + * from being reused when writers commit new data, and so under heavy load + * the database size may grow much more rapidly than otherwise. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + */ +void mdb_txn_reset(MDB_txn *txn); + + /** @brief Renew a read-only transaction. + * + * This acquires a new reader lock for a transaction handle that had been + * released by #mdb_txn_reset(). It must be called before a reset transaction + * may be used again. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>#MDB_PANIC - a fatal error occurred earlier and the environment + * must be shut down. + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_txn_renew(MDB_txn *txn); + +/** Compat with version <= 0.9.4, avoid clash with libmdb from MDB Tools project */ +#define mdb_open(txn,name,flags,dbi) mdb_dbi_open(txn,name,flags,dbi) +/** Compat with version <= 0.9.4, avoid clash with libmdb from MDB Tools project */ +#define mdb_close(env,dbi) mdb_dbi_close(env,dbi) + + /** @brief Open a database in the environment. + * + * A database handle denotes the name and parameters of a database, + * independently of whether such a database exists. + * The database handle may be discarded by calling #mdb_dbi_close(). + * The old database handle is returned if the database was already open. + * The handle may only be closed once. + * The database handle will be private to the current transaction until + * the transaction is successfully committed. If the transaction is + * aborted the handle will be closed automatically. + * After a successful commit the + * handle will reside in the shared environment, and may be used + * by other transactions. This function must not be called from + * multiple concurrent transactions. A transaction that uses this function + * must finish (either commit or abort) before any other transaction may + * use this function. + * + * To use named databases (with name != NULL), #mdb_env_set_maxdbs() + * must be called before opening the environment. Database names + * are kept as keys in the unnamed database. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] name The name of the database to open. If only a single + * database is needed in the environment, this value may be NULL. + * @param[in] flags Special options for this database. This parameter + * must be set to 0 or by bitwise OR'ing together one or more of the + * values described here. + * <ul> + * <li>#MDB_REVERSEKEY + * Keys are strings to be compared in reverse order, from the end + * of the strings to the beginning. By default, Keys are treated as strings and + * compared from beginning to end. + * <li>#MDB_DUPSORT + * Duplicate keys may be used in the database. (Or, from another perspective, + * keys may have multiple data items, stored in sorted order.) By default + * keys must be unique and may have only a single data item. + * <li>#MDB_INTEGERKEY + * Keys are binary integers in native byte order. Setting this option + * requires all keys to be the same size, typically sizeof(int) + * or sizeof(size_t). + * <li>#MDB_DUPFIXED + * This flag may only be used in combination with #MDB_DUPSORT. This option + * tells the library that the data items for this database are all the same + * size, which allows further optimizations in storage and retrieval. When + * all data items are the same size, the #MDB_GET_MULTIPLE and #MDB_NEXT_MULTIPLE + * cursor operations may be used to retrieve multiple items at once. + * <li>#MDB_INTEGERDUP + * This option specifies that duplicate data items are also integers, and + * should be sorted as such. + * <li>#MDB_REVERSEDUP + * This option specifies that duplicate data items should be compared as + * strings in reverse order. + * <li>#MDB_CREATE + * Create the named database if it doesn't exist. This option is not + * allowed in a read-only transaction or a read-only environment. + * </ul> + * @param[out] dbi Address where the new #MDB_dbi handle will be stored + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>#MDB_NOTFOUND - the specified database doesn't exist in the environment + * and #MDB_CREATE was not specified. + * <li>#MDB_DBS_FULL - too many databases have been opened. See #mdb_env_set_maxdbs(). + * </ul> + */ +int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *dbi); + + /** @brief Retrieve statistics for a database. + * + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[out] stat The address of an #MDB_stat structure + * where the statistics will be copied + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *stat); + + /** @brief Retrieve the DB flags for a database handle. + * + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[out] flags Address where the flags will be returned. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_dbi_flags(MDB_txn *txn, MDB_dbi dbi, unsigned int *flags); + + /** @brief Close a database handle. Normally unnecessary. Use with care: + * + * This call is not mutex protected. Handles should only be closed by + * a single thread, and only if no other threads are going to reference + * the database handle or one of its cursors any further. Do not close + * a handle if an existing transaction has modified its database. + * Doing so can cause misbehavior from database corruption to errors + * like MDB_BAD_VALSIZE (since the DB name is gone). + * + * Closing a database handle is not necessary, but lets #mdb_dbi_open() + * reuse the handle value. Usually it's better to set a bigger + * #mdb_env_set_maxdbs(), unless that value would be large. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + */ +void mdb_dbi_close(MDB_env *env, MDB_dbi dbi); + + /** @brief Empty or delete+close a database. + * + * See #mdb_dbi_close() for restrictions about closing the DB handle. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] del 0 to empty the DB, 1 to delete it from the + * environment and close the DB handle. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del); + + /** @brief Set a custom key comparison function for a database. + * + * The comparison function is called whenever it is necessary to compare a + * key specified by the application with a key currently stored in the database. + * If no comparison function is specified, and no special key flags were specified + * with #mdb_dbi_open(), the keys are compared lexically, with shorter keys collating + * before longer keys. + * @warning This function must be called before any data access functions are used, + * otherwise data corruption may occur. The same comparison function must be used by every + * program accessing the database, every time the database is used. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] cmp A #MDB_cmp_func function + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_set_compare(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp); + + /** @brief Set a custom data comparison function for a #MDB_DUPSORT database. + * + * This comparison function is called whenever it is necessary to compare a data + * item specified by the application with a data item currently stored in the database. + * This function only takes effect if the database was opened with the #MDB_DUPSORT + * flag. + * If no comparison function is specified, and no special key flags were specified + * with #mdb_dbi_open(), the data items are compared lexically, with shorter items collating + * before longer items. + * @warning This function must be called before any data access functions are used, + * otherwise data corruption may occur. The same comparison function must be used by every + * program accessing the database, every time the database is used. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] cmp A #MDB_cmp_func function + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_set_dupsort(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp); + + /** @brief Set a relocation function for a #MDB_FIXEDMAP database. + * + * @todo The relocation function is called whenever it is necessary to move the data + * of an item to a different position in the database (e.g. through tree + * balancing operations, shifts as a result of adds or deletes, etc.). It is + * intended to allow address/position-dependent data items to be stored in + * a database in an environment opened with the #MDB_FIXEDMAP option. + * Currently the relocation feature is unimplemented and setting + * this function has no effect. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] rel A #MDB_rel_func function + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_set_relfunc(MDB_txn *txn, MDB_dbi dbi, MDB_rel_func *rel); + + /** @brief Set a context pointer for a #MDB_FIXEDMAP database's relocation function. + * + * See #mdb_set_relfunc and #MDB_rel_func for more details. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] ctx An arbitrary pointer for whatever the application needs. + * It will be passed to the callback function set by #mdb_set_relfunc + * as its \b relctx parameter whenever the callback is invoked. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_set_relctx(MDB_txn *txn, MDB_dbi dbi, void *ctx); + + /** @brief Get items from a database. + * + * This function retrieves key/data pairs from the database. The address + * and length of the data associated with the specified \b key are returned + * in the structure to which \b data refers. + * If the database supports duplicate keys (#MDB_DUPSORT) then the + * first data item for the key will be returned. Retrieval of other + * items requires the use of #mdb_cursor_get(). + * + * @note The memory pointed to by the returned values is owned by the + * database. The caller need not dispose of the memory, and may not + * modify it in any way. For values returned in a read-only transaction + * any modification attempts will cause a SIGSEGV. + * @note Values returned from the database are valid only until a + * subsequent update operation, or the end of the transaction. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] key The key to search for in the database + * @param[out] data The data corresponding to the key + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>#MDB_NOTFOUND - the key was not in the database. + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_get(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data); + + /** @brief Store items into a database. + * + * This function stores key/data pairs in the database. The default behavior + * is to enter the new key/data pair, replacing any previously existing key + * if duplicates are disallowed, or adding a duplicate data item if + * duplicates are allowed (#MDB_DUPSORT). + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] key The key to store in the database + * @param[in,out] data The data to store + * @param[in] flags Special options for this operation. This parameter + * must be set to 0 or by bitwise OR'ing together one or more of the + * values described here. + * <ul> + * <li>#MDB_NODUPDATA - enter the new key/data pair only if it does not + * already appear in the database. This flag may only be specified + * if the database was opened with #MDB_DUPSORT. The function will + * return #MDB_KEYEXIST if the key/data pair already appears in the + * database. + * <li>#MDB_NOOVERWRITE - enter the new key/data pair only if the key + * does not already appear in the database. The function will return + * #MDB_KEYEXIST if the key already appears in the database, even if + * the database supports duplicates (#MDB_DUPSORT). The \b data + * parameter will be set to point to the existing item. + * <li>#MDB_RESERVE - reserve space for data of the given size, but + * don't copy the given data. Instead, return a pointer to the + * reserved space, which the caller can fill in later - before + * the next update operation or the transaction ends. This saves + * an extra memcpy if the data is being generated later. + * LMDB does nothing else with this memory, the caller is expected + * to modify all of the space requested. + * <li>#MDB_APPEND - append the given key/data pair to the end of the + * database. No key comparisons are performed. This option allows + * fast bulk loading when keys are already known to be in the + * correct order. Loading unsorted keys with this flag will cause + * data corruption. + * <li>#MDB_APPENDDUP - as above, but for sorted dup data. + * </ul> + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>#MDB_MAP_FULL - the database is full, see #mdb_env_set_mapsize(). + * <li>#MDB_TXN_FULL - the transaction has too many dirty pages. + * <li>EACCES - an attempt was made to write in a read-only transaction. + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_put(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data, + unsigned int flags); + + /** @brief Delete items from a database. + * + * This function removes key/data pairs from the database. + * If the database does not support sorted duplicate data items + * (#MDB_DUPSORT) the data parameter is ignored. + * If the database supports sorted duplicates and the data parameter + * is NULL, all of the duplicate data items for the key will be + * deleted. Otherwise, if the data parameter is non-NULL + * only the matching data item will be deleted. + * This function will return #MDB_NOTFOUND if the specified key/data + * pair is not in the database. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] key The key to delete from the database + * @param[in] data The data to delete + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EACCES - an attempt was made to write in a read-only transaction. + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_del(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data); + + /** @brief Create a cursor handle. + * + * A cursor is associated with a specific transaction and database. + * A cursor cannot be used when its database handle is closed. Nor + * when its transaction has ended, except with #mdb_cursor_renew(). + * It can be discarded with #mdb_cursor_close(). + * A cursor in a write-transaction can be closed before its transaction + * ends, and will otherwise be closed when its transaction ends. + * A cursor in a read-only transaction must be closed explicitly, before + * or after its transaction ends. It can be reused with + * #mdb_cursor_renew() before finally closing it. + * @note Earlier documentation said that cursors in every transaction + * were closed when the transaction committed or aborted. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[out] cursor Address where the new #MDB_cursor handle will be stored + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **cursor); + + /** @brief Close a cursor handle. + * + * The cursor handle will be freed and must not be used again after this call. + * Its transaction must still be live if it is a write-transaction. + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + */ +void mdb_cursor_close(MDB_cursor *cursor); + + /** @brief Renew a cursor handle. + * + * A cursor is associated with a specific transaction and database. + * Cursors that are only used in read-only + * transactions may be re-used, to avoid unnecessary malloc/free overhead. + * The cursor may be associated with a new read-only transaction, and + * referencing the same database handle as it was created with. + * This may be done whether the previous transaction is live or dead. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_cursor_renew(MDB_txn *txn, MDB_cursor *cursor); + + /** @brief Return the cursor's transaction handle. + * + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + */ +MDB_txn *mdb_cursor_txn(MDB_cursor *cursor); + + /** @brief Return the cursor's database handle. + * + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + */ +MDB_dbi mdb_cursor_dbi(MDB_cursor *cursor); + + /** @brief Retrieve by cursor. + * + * This function retrieves key/data pairs from the database. The address and length + * of the key are returned in the object to which \b key refers (except for the + * case of the #MDB_SET option, in which the \b key object is unchanged), and + * the address and length of the data are returned in the object to which \b data + * refers. + * See #mdb_get() for restrictions on using the output values. + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + * @param[in,out] key The key for a retrieved item + * @param[in,out] data The data of a retrieved item + * @param[in] op A cursor operation #MDB_cursor_op + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>#MDB_NOTFOUND - no matching key found. + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_cursor_get(MDB_cursor *cursor, MDB_val *key, MDB_val *data, + MDB_cursor_op op); + + /** @brief Store by cursor. + * + * This function stores key/data pairs into the database. + * The cursor is positioned at the new item, or on failure usually near it. + * @note Earlier documentation incorrectly said errors would leave the + * state of the cursor unchanged. + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + * @param[in] key The key operated on. + * @param[in] data The data operated on. + * @param[in] flags Options for this operation. This parameter + * must be set to 0 or one of the values described here. + * <ul> + * <li>#MDB_CURRENT - replace the item at the current cursor position. + * The \b key parameter must still be provided, and must match it. + * If using sorted duplicates (#MDB_DUPSORT) the data item must still + * sort into the same place. This is intended to be used when the + * new data is the same size as the old. Otherwise it will simply + * perform a delete of the old record followed by an insert. + * <li>#MDB_NODUPDATA - enter the new key/data pair only if it does not + * already appear in the database. This flag may only be specified + * if the database was opened with #MDB_DUPSORT. The function will + * return #MDB_KEYEXIST if the key/data pair already appears in the + * database. + * <li>#MDB_NOOVERWRITE - enter the new key/data pair only if the key + * does not already appear in the database. The function will return + * #MDB_KEYEXIST if the key already appears in the database, even if + * the database supports duplicates (#MDB_DUPSORT). + * <li>#MDB_RESERVE - reserve space for data of the given size, but + * don't copy the given data. Instead, return a pointer to the + * reserved space, which the caller can fill in later. This saves + * an extra memcpy if the data is being generated later. + * <li>#MDB_APPEND - append the given key/data pair to the end of the + * database. No key comparisons are performed. This option allows + * fast bulk loading when keys are already known to be in the + * correct order. Loading unsorted keys with this flag will cause + * data corruption. + * <li>#MDB_APPENDDUP - as above, but for sorted dup data. + * <li>#MDB_MULTIPLE - store multiple contiguous data elements in a + * single request. This flag may only be specified if the database + * was opened with #MDB_DUPFIXED. The \b data argument must be an + * array of two MDB_vals. The mv_size of the first MDB_val must be + * the size of a single data element. The mv_data of the first MDB_val + * must point to the beginning of the array of contiguous data elements. + * The mv_size of the second MDB_val must be the count of the number + * of data elements to store. On return this field will be set to + * the count of the number of elements actually written. The mv_data + * of the second MDB_val is unused. + * </ul> + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>#MDB_MAP_FULL - the database is full, see #mdb_env_set_mapsize(). + * <li>#MDB_TXN_FULL - the transaction has too many dirty pages. + * <li>EACCES - an attempt was made to modify a read-only database. + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_cursor_put(MDB_cursor *cursor, MDB_val *key, MDB_val *data, + unsigned int flags); + + /** @brief Delete current key/data pair + * + * This function deletes the key/data pair to which the cursor refers. + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + * @param[in] flags Options for this operation. This parameter + * must be set to 0 or one of the values described here. + * <ul> + * <li>#MDB_NODUPDATA - delete all of the data items for the current key. + * This flag may only be specified if the database was opened with #MDB_DUPSORT. + * </ul> + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EACCES - an attempt was made to modify a read-only database. + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_cursor_del(MDB_cursor *cursor, unsigned int flags); + + /** @brief Return count of duplicates for current key. + * + * This call is only valid on databases that support sorted duplicate + * data items #MDB_DUPSORT. + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + * @param[out] countp Address where the count will be stored + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - cursor is not initialized, or an invalid parameter was specified. + * </ul> + */ +int mdb_cursor_count(MDB_cursor *cursor, size_t *countp); + + /** @brief Compare two data items according to a particular database. + * + * This returns a comparison as if the two data items were keys in the + * specified database. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] a The first item to compare + * @param[in] b The second item to compare + * @return < 0 if a < b, 0 if a == b, > 0 if a > b + */ +int mdb_cmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b); + + /** @brief Compare two data items according to a particular database. + * + * This returns a comparison as if the two items were data items of + * the specified database. The database must have the #MDB_DUPSORT flag. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] a The first item to compare + * @param[in] b The second item to compare + * @return < 0 if a < b, 0 if a == b, > 0 if a > b + */ +int mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b); + + /** @brief A callback function used to print a message from the library. + * + * @param[in] msg The string to be printed. + * @param[in] ctx An arbitrary context pointer for the callback. + * @return < 0 on failure, >= 0 on success. + */ +typedef int (MDB_msg_func)(const char *msg, void *ctx); + + /** @brief Dump the entries in the reader lock table. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] func A #MDB_msg_func function + * @param[in] ctx Anything the message function needs + * @return < 0 on failure, >= 0 on success. + */ +int mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx); + + /** @brief Check for stale entries in the reader lock table. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] dead Number of stale slots that were cleared + * @return 0 on success, non-zero on failure. + */ +int mdb_reader_check(MDB_env *env, int *dead); +/** @} */ + +#ifdef __cplusplus +} +#endif +/** @page tools LMDB Command Line Tools + The following describes the command line tools that are available for LMDB. + \li \ref mdb_copy_1 + \li \ref mdb_dump_1 + \li \ref mdb_load_1 + \li \ref mdb_stat_1 +*/ + +#endif /* _LMDB_H_ */ diff --git a/external/db_drivers/liblmdb32/mdb.c b/external/db_drivers/liblmdb32/mdb.c new file mode 100644 index 000000000..63fece3d7 --- /dev/null +++ b/external/db_drivers/liblmdb32/mdb.c @@ -0,0 +1,9572 @@ +/** @file mdb.c + * @brief Lightning memory-mapped database library + * + * A Btree-based database management library modeled loosely on the + * BerkeleyDB API, but much simplified. + */ +/* + * Copyright 2011-2014 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + * + * This code is derived from btree.c written by Martin Hedenfalk. + * + * Copyright (c) 2009, 2010 Martin Hedenfalk <martin@bzero.se> + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE 1 +#endif +#ifdef _WIN32 +#include <malloc.h> +#include <windows.h> +/** getpid() returns int; MinGW defines pid_t but MinGW64 typedefs it + * as int64 which is wrong. MSVC doesn't define it at all, so just + * don't use it. + */ +#define MDB_PID_T int +#define MDB_THR_T DWORD +#include <sys/types.h> +#include <sys/stat.h> +#ifdef __GNUC__ +# include <sys/param.h> +#else +# define LITTLE_ENDIAN 1234 +# define BIG_ENDIAN 4321 +# define BYTE_ORDER LITTLE_ENDIAN +# ifndef SSIZE_MAX +# define SSIZE_MAX INT_MAX +# endif +#endif +#else +#include <sys/types.h> +#include <sys/stat.h> +#define MDB_PID_T pid_t +#define MDB_THR_T pthread_t +#include <sys/param.h> +#include <sys/uio.h> +#include <sys/mman.h> +#ifdef HAVE_SYS_FILE_H +#include <sys/file.h> +#endif +#include <fcntl.h> +#endif + +#if defined(__mips) && defined(__linux) +/* MIPS has cache coherency issues, requires explicit cache control */ +#include <asm/cachectl.h> +extern int cacheflush(char *addr, int nbytes, int cache); +#define CACHEFLUSH(addr, bytes, cache) cacheflush(addr, bytes, cache) +#else +#define CACHEFLUSH(addr, bytes, cache) +#endif + + +#include <errno.h> +#include <limits.h> +#include <stddef.h> +#include <inttypes.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <unistd.h> + +#if defined(__sun) || defined(ANDROID) +/* Most platforms have posix_memalign, older may only have memalign */ +#define HAVE_MEMALIGN 1 +#include <malloc.h> +#endif + +#if !(defined(BYTE_ORDER) || defined(__BYTE_ORDER)) +#include <netinet/in.h> +#include <resolv.h> /* defines BYTE_ORDER on HPUX and Solaris */ +#endif + +#if defined(__APPLE__) || defined (BSD) +# define MDB_USE_POSIX_SEM 1 +# define MDB_FDATASYNC fsync +#elif defined(ANDROID) +# define MDB_FDATASYNC fsync +#endif + +#ifndef _WIN32 +#include <pthread.h> +#ifdef MDB_USE_POSIX_SEM +# define MDB_USE_HASH 1 +#include <semaphore.h> +#endif +#endif + +#ifdef USE_VALGRIND +#include <valgrind/memcheck.h> +#define VGMEMP_CREATE(h,r,z) VALGRIND_CREATE_MEMPOOL(h,r,z) +#define VGMEMP_ALLOC(h,a,s) VALGRIND_MEMPOOL_ALLOC(h,a,s) +#define VGMEMP_FREE(h,a) VALGRIND_MEMPOOL_FREE(h,a) +#define VGMEMP_DESTROY(h) VALGRIND_DESTROY_MEMPOOL(h) +#define VGMEMP_DEFINED(a,s) VALGRIND_MAKE_MEM_DEFINED(a,s) +#else +#define VGMEMP_CREATE(h,r,z) +#define VGMEMP_ALLOC(h,a,s) +#define VGMEMP_FREE(h,a) +#define VGMEMP_DESTROY(h) +#define VGMEMP_DEFINED(a,s) +#endif + +#ifndef BYTE_ORDER +# if (defined(_LITTLE_ENDIAN) || defined(_BIG_ENDIAN)) && !(defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN)) +/* Solaris just defines one or the other */ +# define LITTLE_ENDIAN 1234 +# define BIG_ENDIAN 4321 +# ifdef _LITTLE_ENDIAN +# define BYTE_ORDER LITTLE_ENDIAN +# else +# define BYTE_ORDER BIG_ENDIAN +# endif +# else +# define BYTE_ORDER __BYTE_ORDER +# endif +#endif + +#ifndef LITTLE_ENDIAN +#define LITTLE_ENDIAN __LITTLE_ENDIAN +#endif +#ifndef BIG_ENDIAN +#define BIG_ENDIAN __BIG_ENDIAN +#endif + +#if defined(__i386) || defined(__x86_64) || defined(_M_IX86) +#define MISALIGNED_OK 1 +#endif + +#include "lmdb.h" +#include "midl.h" + +#if (BYTE_ORDER == LITTLE_ENDIAN) == (BYTE_ORDER == BIG_ENDIAN) +# error "Unknown or unsupported endianness (BYTE_ORDER)" +#elif (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF +# error "Two's complement, reasonably sized integer types, please" +#endif + +#ifdef __GNUC__ +/** Put infrequently used env functions in separate section */ +# ifdef __APPLE__ +# define ESECT __attribute__ ((section("__TEXT,text_env"))) +# else +# define ESECT __attribute__ ((section("text_env"))) +# endif +#else +#define ESECT +#endif + +/** @defgroup internal LMDB Internals + * @{ + */ +/** @defgroup compat Compatibility Macros + * A bunch of macros to minimize the amount of platform-specific ifdefs + * needed throughout the rest of the code. When the features this library + * needs are similar enough to POSIX to be hidden in a one-or-two line + * replacement, this macro approach is used. + * @{ + */ + + /** Features under development */ +#ifndef MDB_DEVEL +#define MDB_DEVEL 0 +#endif + + /** Wrapper around __func__, which is a C99 feature */ +#if __STDC_VERSION__ >= 199901L +# define mdb_func_ __func__ +#elif __GNUC__ >= 2 || _MSC_VER >= 1300 +# define mdb_func_ __FUNCTION__ +#else +/* If a debug message says <mdb_unknown>(), update the #if statements above */ +# define mdb_func_ "<mdb_unknown>" +#endif + +#ifdef _WIN32 +#define MDB_USE_HASH 1 +#define MDB_PIDLOCK 0 +#define THREAD_RET DWORD +#define pthread_t HANDLE +#define pthread_mutex_t HANDLE +#define pthread_cond_t HANDLE +#define pthread_key_t DWORD +#define pthread_self() GetCurrentThreadId() +#define pthread_key_create(x,y) \ + ((*(x) = TlsAlloc()) == TLS_OUT_OF_INDEXES ? ErrCode() : 0) +#define pthread_key_delete(x) TlsFree(x) +#define pthread_getspecific(x) TlsGetValue(x) +#define pthread_setspecific(x,y) (TlsSetValue(x,y) ? 0 : ErrCode()) +#define pthread_mutex_unlock(x) ReleaseMutex(*x) +#define pthread_mutex_lock(x) WaitForSingleObject(*x, INFINITE) +#define pthread_cond_signal(x) SetEvent(*x) +#define pthread_cond_wait(cond,mutex) do{SignalObjectAndWait(*mutex, *cond, INFINITE, FALSE); WaitForSingleObject(*mutex, INFINITE);}while(0) +#define THREAD_CREATE(thr,start,arg) thr=CreateThread(NULL,0,start,arg,0,NULL) +#define THREAD_FINISH(thr) WaitForSingleObject(thr, INFINITE) +#define LOCK_MUTEX_R(env) pthread_mutex_lock(&(env)->me_rmutex) +#define UNLOCK_MUTEX_R(env) pthread_mutex_unlock(&(env)->me_rmutex) +#define LOCK_MUTEX_W(env) pthread_mutex_lock(&(env)->me_wmutex) +#define UNLOCK_MUTEX_W(env) pthread_mutex_unlock(&(env)->me_wmutex) +#define getpid() GetCurrentProcessId() +#define MDB_FDATASYNC(fd) (!FlushFileBuffers(fd)) +#define MDB_MSYNC(addr,len,flags) (!FlushViewOfFile(addr,len)) +#define ErrCode() GetLastError() +#define GET_PAGESIZE(x) {SYSTEM_INFO si; GetSystemInfo(&si); (x) = si.dwPageSize;} +#define close(fd) (CloseHandle(fd) ? 0 : -1) +#define munmap(ptr,len) UnmapViewOfFile(ptr) +#ifdef PROCESS_QUERY_LIMITED_INFORMATION +#define MDB_PROCESS_QUERY_LIMITED_INFORMATION PROCESS_QUERY_LIMITED_INFORMATION +#else +#define MDB_PROCESS_QUERY_LIMITED_INFORMATION 0x1000 +#endif +#define Z "I" +#else +#define THREAD_RET void * +#define THREAD_CREATE(thr,start,arg) pthread_create(&thr,NULL,start,arg) +#define THREAD_FINISH(thr) pthread_join(thr,NULL) +#define Z "z" /**< printf format modifier for size_t */ + + /** For MDB_LOCK_FORMAT: True if readers take a pid lock in the lockfile */ +#define MDB_PIDLOCK 1 + +#ifdef MDB_USE_POSIX_SEM + +#define LOCK_MUTEX_R(env) mdb_sem_wait((env)->me_rmutex) +#define UNLOCK_MUTEX_R(env) sem_post((env)->me_rmutex) +#define LOCK_MUTEX_W(env) mdb_sem_wait((env)->me_wmutex) +#define UNLOCK_MUTEX_W(env) sem_post((env)->me_wmutex) + +static int +mdb_sem_wait(sem_t *sem) +{ + int rc; + while ((rc = sem_wait(sem)) && (rc = errno) == EINTR) ; + return rc; +} + +#else + /** Lock the reader mutex. + */ +#define LOCK_MUTEX_R(env) pthread_mutex_lock(&(env)->me_txns->mti_mutex) + /** Unlock the reader mutex. + */ +#define UNLOCK_MUTEX_R(env) pthread_mutex_unlock(&(env)->me_txns->mti_mutex) + + /** Lock the writer mutex. + * Only a single write transaction is allowed at a time. Other writers + * will block waiting for this mutex. + */ +#define LOCK_MUTEX_W(env) pthread_mutex_lock(&(env)->me_txns->mti_wmutex) + /** Unlock the writer mutex. + */ +#define UNLOCK_MUTEX_W(env) pthread_mutex_unlock(&(env)->me_txns->mti_wmutex) +#endif /* MDB_USE_POSIX_SEM */ + + /** Get the error code for the last failed system function. + */ +#define ErrCode() errno + + /** An abstraction for a file handle. + * On POSIX systems file handles are small integers. On Windows + * they're opaque pointers. + */ +#define HANDLE int + + /** A value for an invalid file handle. + * Mainly used to initialize file variables and signify that they are + * unused. + */ +#define INVALID_HANDLE_VALUE (-1) + + /** Get the size of a memory page for the system. + * This is the basic size that the platform's memory manager uses, and is + * fundamental to the use of memory-mapped files. + */ +#define GET_PAGESIZE(x) ((x) = sysconf(_SC_PAGE_SIZE)) +#endif + +#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) +#define MNAME_LEN 32 +#else +#define MNAME_LEN (sizeof(pthread_mutex_t)) +#endif + +/** @} */ + +#ifndef _WIN32 +/** A flag for opening a file and requesting synchronous data writes. + * This is only used when writing a meta page. It's not strictly needed; + * we could just do a normal write and then immediately perform a flush. + * But if this flag is available it saves us an extra system call. + * + * @note If O_DSYNC is undefined but exists in /usr/include, + * preferably set some compiler flag to get the definition. + * Otherwise compile with the less efficient -DMDB_DSYNC=O_SYNC. + */ +#ifndef MDB_DSYNC +# define MDB_DSYNC O_DSYNC +#endif +#endif + +/** Function for flushing the data of a file. Define this to fsync + * if fdatasync() is not supported. + */ +#ifndef MDB_FDATASYNC +# define MDB_FDATASYNC fdatasync +#endif + +#ifndef MDB_MSYNC +# define MDB_MSYNC(addr,len,flags) msync(addr,len,flags) +#endif + +#ifndef MS_SYNC +#define MS_SYNC 1 +#endif + +#ifndef MS_ASYNC +#define MS_ASYNC 0 +#endif + + /** A page number in the database. + * Note that 64 bit page numbers are overkill, since pages themselves + * already represent 12-13 bits of addressable memory, and the OS will + * always limit applications to a maximum of 63 bits of address space. + * + * @note In the #MDB_node structure, we only store 48 bits of this value, + * which thus limits us to only 60 bits of addressable data. + */ +typedef MDB_ID pgno_t; + + /** A transaction ID. + * See struct MDB_txn.mt_txnid for details. + */ +typedef MDB_ID txnid_t; + +/** @defgroup debug Debug Macros + * @{ + */ +#ifndef MDB_DEBUG + /** Enable debug output. Needs variable argument macros (a C99 feature). + * Set this to 1 for copious tracing. Set to 2 to add dumps of all IDLs + * read from and written to the database (used for free space management). + */ +#define MDB_DEBUG 0 +#endif + +#if MDB_DEBUG +static int mdb_debug; +static txnid_t mdb_debug_start; + + /** Print a debug message with printf formatting. + * Requires double parenthesis around 2 or more args. + */ +# define DPRINTF(args) ((void) ((mdb_debug) && DPRINTF0 args)) +# define DPRINTF0(fmt, ...) \ + fprintf(stderr, "%s:%d " fmt "\n", mdb_func_, __LINE__, __VA_ARGS__) +#else +# define DPRINTF(args) ((void) 0) +#endif + /** Print a debug string. + * The string is printed literally, with no format processing. + */ +#define DPUTS(arg) DPRINTF(("%s", arg)) + /** Debuging output value of a cursor DBI: Negative in a sub-cursor. */ +#define DDBI(mc) \ + (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi) +/** @} */ + + /** @brief The maximum size of a database page. + * + * It is 32k or 64k, since value-PAGEBASE must fit in + * #MDB_page.%mp_upper. + * + * LMDB will use database pages < OS pages if needed. + * That causes more I/O in write transactions: The OS must + * know (read) the whole page before writing a partial page. + * + * Note that we don't currently support Huge pages. On Linux, + * regular data files cannot use Huge pages, and in general + * Huge pages aren't actually pageable. We rely on the OS + * demand-pager to read our data and page it out when memory + * pressure from other processes is high. So until OSs have + * actual paging support for Huge pages, they're not viable. + */ +#define MAX_PAGESIZE (PAGEBASE ? 0x10000 : 0x8000) + + /** The minimum number of keys required in a database page. + * Setting this to a larger value will place a smaller bound on the + * maximum size of a data item. Data items larger than this size will + * be pushed into overflow pages instead of being stored directly in + * the B-tree node. This value used to default to 4. With a page size + * of 4096 bytes that meant that any item larger than 1024 bytes would + * go into an overflow page. That also meant that on average 2-3KB of + * each overflow page was wasted space. The value cannot be lower than + * 2 because then there would no longer be a tree structure. With this + * value, items larger than 2KB will go into overflow pages, and on + * average only 1KB will be wasted. + */ +#define MDB_MINKEYS 2 + + /** A stamp that identifies a file as an LMDB file. + * There's nothing special about this value other than that it is easily + * recognizable, and it will reflect any byte order mismatches. + */ +#define MDB_MAGIC 0xBEEFC0DE + + /** The version number for a database's datafile format. */ +#define MDB_DATA_VERSION ((MDB_DEVEL) ? 999 : 1) + /** The version number for a database's lockfile format. */ +#define MDB_LOCK_VERSION 1 + + /** @brief The max size of a key we can write, or 0 for dynamic max. + * + * Define this as 0 to compute the max from the page size. 511 + * is default for backwards compat: liblmdb <= 0.9.10 can break + * when modifying a DB with keys/dupsort data bigger than its max. + * #MDB_DEVEL sets the default to 0. + * + * Data items in an #MDB_DUPSORT database are also limited to + * this size, since they're actually keys of a sub-DB. Keys and + * #MDB_DUPSORT data items must fit on a node in a regular page. + */ +#ifndef MDB_MAXKEYSIZE +#define MDB_MAXKEYSIZE ((MDB_DEVEL) ? 0 : 511) +#endif + + /** The maximum size of a key we can write to the environment. */ +#if MDB_MAXKEYSIZE +#define ENV_MAXKEY(env) (MDB_MAXKEYSIZE) +#else +#define ENV_MAXKEY(env) ((env)->me_maxkey) +#endif + + /** @brief The maximum size of a data item. + * + * We only store a 32 bit value for node sizes. + */ +#define MAXDATASIZE 0xffffffffUL + +#if MDB_DEBUG + /** Key size which fits in a #DKBUF. + * @ingroup debug + */ +#define DKBUF_MAXKEYSIZE ((MDB_MAXKEYSIZE) > 0 ? (MDB_MAXKEYSIZE) : 511) + /** A key buffer. + * @ingroup debug + * This is used for printing a hex dump of a key's contents. + */ +#define DKBUF char kbuf[DKBUF_MAXKEYSIZE*2+1] + /** Display a key in hex. + * @ingroup debug + * Invoke a function to display a key in hex. + */ +#define DKEY(x) mdb_dkey(x, kbuf) +#else +#define DKBUF +#define DKEY(x) 0 +#endif + + /** An invalid page number. + * Mainly used to denote an empty tree. + */ +#define P_INVALID (~(pgno_t)0) + + /** Test if the flags \b f are set in a flag word \b w. */ +#define F_ISSET(w, f) (((w) & (f)) == (f)) + + /** Round \b n up to an even number. */ +#define EVEN(n) (((n) + 1U) & -2) /* sign-extending -2 to match n+1U */ + + /** Used for offsets within a single page. + * Since memory pages are typically 4 or 8KB in size, 12-13 bits, + * this is plenty. + */ +typedef uint16_t indx_t; + + /** Default size of memory map. + * This is certainly too small for any actual applications. Apps should always set + * the size explicitly using #mdb_env_set_mapsize(). + */ +#define DEFAULT_MAPSIZE 1048576 + +/** @defgroup readers Reader Lock Table + * Readers don't acquire any locks for their data access. Instead, they + * simply record their transaction ID in the reader table. The reader + * mutex is needed just to find an empty slot in the reader table. The + * slot's address is saved in thread-specific data so that subsequent read + * transactions started by the same thread need no further locking to proceed. + * + * If #MDB_NOTLS is set, the slot address is not saved in thread-specific data. + * + * No reader table is used if the database is on a read-only filesystem, or + * if #MDB_NOLOCK is set. + * + * Since the database uses multi-version concurrency control, readers don't + * actually need any locking. This table is used to keep track of which + * readers are using data from which old transactions, so that we'll know + * when a particular old transaction is no longer in use. Old transactions + * that have discarded any data pages can then have those pages reclaimed + * for use by a later write transaction. + * + * The lock table is constructed such that reader slots are aligned with the + * processor's cache line size. Any slot is only ever used by one thread. + * This alignment guarantees that there will be no contention or cache + * thrashing as threads update their own slot info, and also eliminates + * any need for locking when accessing a slot. + * + * A writer thread will scan every slot in the table to determine the oldest + * outstanding reader transaction. Any freed pages older than this will be + * reclaimed by the writer. The writer doesn't use any locks when scanning + * this table. This means that there's no guarantee that the writer will + * see the most up-to-date reader info, but that's not required for correct + * operation - all we need is to know the upper bound on the oldest reader, + * we don't care at all about the newest reader. So the only consequence of + * reading stale information here is that old pages might hang around a + * while longer before being reclaimed. That's actually good anyway, because + * the longer we delay reclaiming old pages, the more likely it is that a + * string of contiguous pages can be found after coalescing old pages from + * many old transactions together. + * @{ + */ + /** Number of slots in the reader table. + * This value was chosen somewhat arbitrarily. 126 readers plus a + * couple mutexes fit exactly into 8KB on my development machine. + * Applications should set the table size using #mdb_env_set_maxreaders(). + */ +#define DEFAULT_READERS 126 + + /** The size of a CPU cache line in bytes. We want our lock structures + * aligned to this size to avoid false cache line sharing in the + * lock table. + * This value works for most CPUs. For Itanium this should be 128. + */ +#ifndef CACHELINE +#define CACHELINE 64 +#endif + + /** The information we store in a single slot of the reader table. + * In addition to a transaction ID, we also record the process and + * thread ID that owns a slot, so that we can detect stale information, + * e.g. threads or processes that went away without cleaning up. + * @note We currently don't check for stale records. We simply re-init + * the table when we know that we're the only process opening the + * lock file. + */ +typedef struct MDB_rxbody { + /** Current Transaction ID when this transaction began, or (txnid_t)-1. + * Multiple readers that start at the same time will probably have the + * same ID here. Again, it's not important to exclude them from + * anything; all we need to know is which version of the DB they + * started from so we can avoid overwriting any data used in that + * particular version. + */ + txnid_t mrb_txnid; + /** The process ID of the process owning this reader txn. */ + MDB_PID_T mrb_pid; + /** The thread ID of the thread owning this txn. */ + MDB_THR_T mrb_tid; +} MDB_rxbody; + + /** The actual reader record, with cacheline padding. */ +typedef struct MDB_reader { + union { + MDB_rxbody mrx; + /** shorthand for mrb_txnid */ +#define mr_txnid mru.mrx.mrb_txnid +#define mr_pid mru.mrx.mrb_pid +#define mr_tid mru.mrx.mrb_tid + /** cache line alignment */ + char pad[(sizeof(MDB_rxbody)+CACHELINE-1) & ~(CACHELINE-1)]; + } mru; +} MDB_reader; + + /** The header for the reader table. + * The table resides in a memory-mapped file. (This is a different file + * than is used for the main database.) + * + * For POSIX the actual mutexes reside in the shared memory of this + * mapped file. On Windows, mutexes are named objects allocated by the + * kernel; we store the mutex names in this mapped file so that other + * processes can grab them. This same approach is also used on + * MacOSX/Darwin (using named semaphores) since MacOSX doesn't support + * process-shared POSIX mutexes. For these cases where a named object + * is used, the object name is derived from a 64 bit FNV hash of the + * environment pathname. As such, naming collisions are extremely + * unlikely. If a collision occurs, the results are unpredictable. + */ +typedef struct MDB_txbody { + /** Stamp identifying this as an LMDB file. It must be set + * to #MDB_MAGIC. */ + uint32_t mtb_magic; + /** Format of this lock file. Must be set to #MDB_LOCK_FORMAT. */ + uint32_t mtb_format; +#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) + char mtb_rmname[MNAME_LEN]; +#else + /** Mutex protecting access to this table. + * This is the reader lock that #LOCK_MUTEX_R acquires. + */ + pthread_mutex_t mtb_mutex; +#endif + /** The ID of the last transaction committed to the database. + * This is recorded here only for convenience; the value can always + * be determined by reading the main database meta pages. + */ + txnid_t mtb_txnid; + /** The number of slots that have been used in the reader table. + * This always records the maximum count, it is not decremented + * when readers release their slots. + */ + unsigned mtb_numreaders; +} MDB_txbody; + + /** The actual reader table definition. */ +typedef struct MDB_txninfo { + union { + MDB_txbody mtb; +#define mti_magic mt1.mtb.mtb_magic +#define mti_format mt1.mtb.mtb_format +#define mti_mutex mt1.mtb.mtb_mutex +#define mti_rmname mt1.mtb.mtb_rmname +#define mti_txnid mt1.mtb.mtb_txnid +#define mti_numreaders mt1.mtb.mtb_numreaders + char pad[(sizeof(MDB_txbody)+CACHELINE-1) & ~(CACHELINE-1)]; + } mt1; + union { +#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) + char mt2_wmname[MNAME_LEN]; +#define mti_wmname mt2.mt2_wmname +#else + pthread_mutex_t mt2_wmutex; +#define mti_wmutex mt2.mt2_wmutex +#endif + char pad[(MNAME_LEN+CACHELINE-1) & ~(CACHELINE-1)]; + } mt2; + MDB_reader mti_readers[1]; +} MDB_txninfo; + + /** Lockfile format signature: version, features and field layout */ +#define MDB_LOCK_FORMAT \ + ((uint32_t) \ + ((MDB_LOCK_VERSION) \ + /* Flags which describe functionality */ \ + + (((MDB_PIDLOCK) != 0) << 16))) +/** @} */ + +/** Common header for all page types. + * Overflow records occupy a number of contiguous pages with no + * headers on any page after the first. + */ +typedef struct MDB_page { +#define mp_pgno mp_p.p_pgno +#define mp_next mp_p.p_next + union { + pgno_t p_pgno; /**< page number */ + struct MDB_page *p_next; /**< for in-memory list of freed pages */ + } mp_p; + uint16_t mp_pad; +/** @defgroup mdb_page Page Flags + * @ingroup internal + * Flags for the page headers. + * @{ + */ +#define P_BRANCH 0x01 /**< branch page */ +#define P_LEAF 0x02 /**< leaf page */ +#define P_OVERFLOW 0x04 /**< overflow page */ +#define P_META 0x08 /**< meta page */ +#define P_DIRTY 0x10 /**< dirty page, also set for #P_SUBP pages */ +#define P_LEAF2 0x20 /**< for #MDB_DUPFIXED records */ +#define P_SUBP 0x40 /**< for #MDB_DUPSORT sub-pages */ +#define P_LOOSE 0x4000 /**< page was dirtied then freed, can be reused */ +#define P_KEEP 0x8000 /**< leave this page alone during spill */ +/** @} */ + uint16_t mp_flags; /**< @ref mdb_page */ +#define mp_lower mp_pb.pb.pb_lower +#define mp_upper mp_pb.pb.pb_upper +#define mp_pages mp_pb.pb_pages + union { + struct { + indx_t pb_lower; /**< lower bound of free space */ + indx_t pb_upper; /**< upper bound of free space */ + } pb; + uint32_t pb_pages; /**< number of overflow pages */ + } mp_pb; + indx_t mp_ptrs[1]; /**< dynamic size */ +} MDB_page; + + /** Size of the page header, excluding dynamic data at the end */ +#define PAGEHDRSZ ((unsigned) offsetof(MDB_page, mp_ptrs)) + + /** Address of first usable data byte in a page, after the header */ +#define METADATA(p) ((void *)((char *)(p) + PAGEHDRSZ)) + + /** ITS#7713, change PAGEBASE to handle 65536 byte pages */ +#define PAGEBASE ((MDB_DEVEL) ? PAGEHDRSZ : 0) + + /** Number of nodes on a page */ +#define NUMKEYS(p) (((p)->mp_lower - (PAGEHDRSZ-PAGEBASE)) >> 1) + + /** The amount of space remaining in the page */ +#define SIZELEFT(p) (indx_t)((p)->mp_upper - (p)->mp_lower) + + /** The percentage of space used in the page, in tenths of a percent. */ +#define PAGEFILL(env, p) (1000L * ((env)->me_psize - PAGEHDRSZ - SIZELEFT(p)) / \ + ((env)->me_psize - PAGEHDRSZ)) + /** The minimum page fill factor, in tenths of a percent. + * Pages emptier than this are candidates for merging. + */ +#define FILL_THRESHOLD 250 + + /** Test if a page is a leaf page */ +#define IS_LEAF(p) F_ISSET((p)->mp_flags, P_LEAF) + /** Test if a page is a LEAF2 page */ +#define IS_LEAF2(p) F_ISSET((p)->mp_flags, P_LEAF2) + /** Test if a page is a branch page */ +#define IS_BRANCH(p) F_ISSET((p)->mp_flags, P_BRANCH) + /** Test if a page is an overflow page */ +#define IS_OVERFLOW(p) F_ISSET((p)->mp_flags, P_OVERFLOW) + /** Test if a page is a sub page */ +#define IS_SUBP(p) F_ISSET((p)->mp_flags, P_SUBP) + + /** The number of overflow pages needed to store the given size. */ +#define OVPAGES(size, psize) ((PAGEHDRSZ-1 + (size)) / (psize) + 1) + + /** Link in #MDB_txn.%mt_loose_pgs list */ +#define NEXT_LOOSE_PAGE(p) (*(MDB_page **)((p) + 2)) + + /** Header for a single key/data pair within a page. + * Used in pages of type #P_BRANCH and #P_LEAF without #P_LEAF2. + * We guarantee 2-byte alignment for 'MDB_node's. + */ +typedef struct MDB_node { + /** lo and hi are used for data size on leaf nodes and for + * child pgno on branch nodes. On 64 bit platforms, flags + * is also used for pgno. (Branch nodes have no flags). + * They are in host byte order in case that lets some + * accesses be optimized into a 32-bit word access. + */ +#if BYTE_ORDER == LITTLE_ENDIAN + unsigned short mn_lo, mn_hi; /**< part of data size or pgno */ +#else + unsigned short mn_hi, mn_lo; +#endif +/** @defgroup mdb_node Node Flags + * @ingroup internal + * Flags for node headers. + * @{ + */ +#define F_BIGDATA 0x01 /**< data put on overflow page */ +#define F_SUBDATA 0x02 /**< data is a sub-database */ +#define F_DUPDATA 0x04 /**< data has duplicates */ + +/** valid flags for #mdb_node_add() */ +#define NODE_ADD_FLAGS (F_DUPDATA|F_SUBDATA|MDB_RESERVE|MDB_APPEND) + +/** @} */ + unsigned short mn_flags; /**< @ref mdb_node */ + unsigned short mn_ksize; /**< key size */ + char mn_data[1]; /**< key and data are appended here */ +} MDB_node; + + /** Size of the node header, excluding dynamic data at the end */ +#define NODESIZE offsetof(MDB_node, mn_data) + + /** Bit position of top word in page number, for shifting mn_flags */ +#define PGNO_TOPWORD ((pgno_t)-1 > 0xffffffffu ? 32 : 0) + + /** Size of a node in a branch page with a given key. + * This is just the node header plus the key, there is no data. + */ +#define INDXSIZE(k) (NODESIZE + ((k) == NULL ? 0 : (k)->mv_size)) + + /** Size of a node in a leaf page with a given key and data. + * This is node header plus key plus data size. + */ +#define LEAFSIZE(k, d) (NODESIZE + (k)->mv_size + (d)->mv_size) + + /** Address of node \b i in page \b p */ +#define NODEPTR(p, i) ((MDB_node *)((char *)(p) + (p)->mp_ptrs[i] + PAGEBASE)) + + /** Address of the key for the node */ +#define NODEKEY(node) (void *)((node)->mn_data) + + /** Address of the data for a node */ +#define NODEDATA(node) (void *)((char *)(node)->mn_data + (node)->mn_ksize) + + /** Get the page number pointed to by a branch node */ +#define NODEPGNO(node) \ + ((node)->mn_lo | ((pgno_t) (node)->mn_hi << 16) | \ + (PGNO_TOPWORD ? ((pgno_t) (node)->mn_flags << PGNO_TOPWORD) : 0)) + /** Set the page number in a branch node */ +#define SETPGNO(node,pgno) do { \ + (node)->mn_lo = (pgno) & 0xffff; (node)->mn_hi = (pgno) >> 16; \ + if (PGNO_TOPWORD) (node)->mn_flags = (pgno) >> PGNO_TOPWORD; } while(0) + + /** Get the size of the data in a leaf node */ +#define NODEDSZ(node) ((node)->mn_lo | ((unsigned)(node)->mn_hi << 16)) + /** Set the size of the data for a leaf node */ +#define SETDSZ(node,size) do { \ + (node)->mn_lo = (size) & 0xffff; (node)->mn_hi = (size) >> 16;} while(0) + /** The size of a key in a node */ +#define NODEKSZ(node) ((node)->mn_ksize) + + /** Copy a page number from src to dst */ +#ifdef MISALIGNED_OK +#define COPY_PGNO(dst,src) dst = src +#else +#if SIZE_MAX > 4294967295UL +#define COPY_PGNO(dst,src) do { \ + unsigned short *s, *d; \ + s = (unsigned short *)&(src); \ + d = (unsigned short *)&(dst); \ + *d++ = *s++; \ + *d++ = *s++; \ + *d++ = *s++; \ + *d = *s; \ +} while (0) +#else +#define COPY_PGNO(dst,src) do { \ + unsigned short *s, *d; \ + s = (unsigned short *)&(src); \ + d = (unsigned short *)&(dst); \ + *d++ = *s++; \ + *d = *s; \ +} while (0) +#endif +#endif + /** The address of a key in a LEAF2 page. + * LEAF2 pages are used for #MDB_DUPFIXED sorted-duplicate sub-DBs. + * There are no node headers, keys are stored contiguously. + */ +#define LEAF2KEY(p, i, ks) ((char *)(p) + PAGEHDRSZ + ((i)*(ks))) + + /** Set the \b node's key into \b keyptr, if requested. */ +#define MDB_GET_KEY(node, keyptr) { if ((keyptr) != NULL) { \ + (keyptr)->mv_size = NODEKSZ(node); (keyptr)->mv_data = NODEKEY(node); } } + + /** Set the \b node's key into \b key. */ +#define MDB_GET_KEY2(node, key) { key.mv_size = NODEKSZ(node); key.mv_data = NODEKEY(node); } + + /** Information about a single database in the environment. */ +typedef struct MDB_db { + uint32_t md_pad; /**< also ksize for LEAF2 pages */ + uint16_t md_flags; /**< @ref mdb_dbi_open */ + uint16_t md_depth; /**< depth of this tree */ + pgno_t md_branch_pages; /**< number of internal pages */ + pgno_t md_leaf_pages; /**< number of leaf pages */ + pgno_t md_overflow_pages; /**< number of overflow pages */ + size_t md_entries; /**< number of data items */ + pgno_t md_root; /**< the root page of this tree */ +} MDB_db; + + /** mdb_dbi_open flags */ +#define MDB_VALID 0x8000 /**< DB handle is valid, for me_dbflags */ +#define PERSISTENT_FLAGS (0xffff & ~(MDB_VALID)) +#define VALID_FLAGS (MDB_REVERSEKEY|MDB_DUPSORT|MDB_INTEGERKEY|MDB_DUPFIXED|\ + MDB_INTEGERDUP|MDB_REVERSEDUP|MDB_CREATE) + + /** Handle for the DB used to track free pages. */ +#define FREE_DBI 0 + /** Handle for the default DB. */ +#define MAIN_DBI 1 + + /** Meta page content. + * A meta page is the start point for accessing a database snapshot. + * Pages 0-1 are meta pages. Transaction N writes meta page #(N % 2). + */ +typedef struct MDB_meta { + /** Stamp identifying this as an LMDB file. It must be set + * to #MDB_MAGIC. */ + uint32_t mm_magic; + /** Version number of this lock file. Must be set to #MDB_DATA_VERSION. */ + uint32_t mm_version; + void *mm_address; /**< address for fixed mapping */ + size_t mm_mapsize; /**< size of mmap region */ + MDB_db mm_dbs[2]; /**< first is free space, 2nd is main db */ + /** The size of pages used in this DB */ +#define mm_psize mm_dbs[0].md_pad + /** Any persistent environment flags. @ref mdb_env */ +#define mm_flags mm_dbs[0].md_flags + pgno_t mm_last_pg; /**< last used page in file */ + txnid_t mm_txnid; /**< txnid that committed this page */ +} MDB_meta; + + /** Buffer for a stack-allocated meta page. + * The members define size and alignment, and silence type + * aliasing warnings. They are not used directly; that could + * mean incorrectly using several union members in parallel. + */ +typedef union MDB_metabuf { + MDB_page mb_page; + struct { + char mm_pad[PAGEHDRSZ]; + MDB_meta mm_meta; + } mb_metabuf; +} MDB_metabuf; + + /** Auxiliary DB info. + * The information here is mostly static/read-only. There is + * only a single copy of this record in the environment. + */ +typedef struct MDB_dbx { + MDB_val md_name; /**< name of the database */ + MDB_cmp_func *md_cmp; /**< function for comparing keys */ + MDB_cmp_func *md_dcmp; /**< function for comparing data items */ + MDB_rel_func *md_rel; /**< user relocate function */ + void *md_relctx; /**< user-provided context for md_rel */ +} MDB_dbx; + + /** A database transaction. + * Every operation requires a transaction handle. + */ +struct MDB_txn { + MDB_txn *mt_parent; /**< parent of a nested txn */ + MDB_txn *mt_child; /**< nested txn under this txn */ + pgno_t mt_next_pgno; /**< next unallocated page */ + /** The ID of this transaction. IDs are integers incrementing from 1. + * Only committed write transactions increment the ID. If a transaction + * aborts, the ID may be re-used by the next writer. + */ + txnid_t mt_txnid; + MDB_env *mt_env; /**< the DB environment */ + /** The list of pages that became unused during this transaction. + */ + MDB_IDL mt_free_pgs; + /** The list of loose pages that became unused and may be reused + * in this transaction, linked through #NEXT_LOOSE_PAGE(page). + */ + MDB_page *mt_loose_pgs; + /* #Number of loose pages (#mt_loose_pgs) */ + int mt_loose_count; + /** The sorted list of dirty pages we temporarily wrote to disk + * because the dirty list was full. page numbers in here are + * shifted left by 1, deleted slots have the LSB set. + */ + MDB_IDL mt_spill_pgs; + union { + /** For write txns: Modified pages. Sorted when not MDB_WRITEMAP. */ + MDB_ID2L dirty_list; + /** For read txns: This thread/txn's reader table slot, or NULL. */ + MDB_reader *reader; + } mt_u; + /** Array of records for each DB known in the environment. */ + MDB_dbx *mt_dbxs; + /** Array of MDB_db records for each known DB */ + MDB_db *mt_dbs; + /** Array of sequence numbers for each DB handle */ + unsigned int *mt_dbiseqs; +/** @defgroup mt_dbflag Transaction DB Flags + * @ingroup internal + * @{ + */ +#define DB_DIRTY 0x01 /**< DB was modified or is DUPSORT data */ +#define DB_STALE 0x02 /**< Named-DB record is older than txnID */ +#define DB_NEW 0x04 /**< Named-DB handle opened in this txn */ +#define DB_VALID 0x08 /**< DB handle is valid, see also #MDB_VALID */ +/** @} */ + /** In write txns, array of cursors for each DB */ + MDB_cursor **mt_cursors; + /** Array of flags for each DB */ + unsigned char *mt_dbflags; +#ifdef VL32 + /** List of read-only pages */ + MDB_ID2L mt_rpages; +#endif + /** Number of DB records in use. This number only ever increments; + * we don't decrement it when individual DB handles are closed. + */ + MDB_dbi mt_numdbs; + +/** @defgroup mdb_txn Transaction Flags + * @ingroup internal + * @{ + */ +#define MDB_TXN_RDONLY 0x01 /**< read-only transaction */ +#define MDB_TXN_ERROR 0x02 /**< txn is unusable after an error */ +#define MDB_TXN_DIRTY 0x04 /**< must write, even if dirty list is empty */ +#define MDB_TXN_SPILLS 0x08 /**< txn or a parent has spilled pages */ +/** @} */ + unsigned int mt_flags; /**< @ref mdb_txn */ + /** #dirty_list room: Array size - \#dirty pages visible to this txn. + * Includes ancestor txns' dirty pages not hidden by other txns' + * dirty/spilled pages. Thus commit(nested txn) has room to merge + * dirty_list into mt_parent after freeing hidden mt_parent pages. + */ + unsigned int mt_dirty_room; +}; + +/** Enough space for 2^32 nodes with minimum of 2 keys per node. I.e., plenty. + * At 4 keys per node, enough for 2^64 nodes, so there's probably no need to + * raise this on a 64 bit machine. + */ +#define CURSOR_STACK 32 + +struct MDB_xcursor; + + /** Cursors are used for all DB operations. + * A cursor holds a path of (page pointer, key index) from the DB + * root to a position in the DB, plus other state. #MDB_DUPSORT + * cursors include an xcursor to the current data item. Write txns + * track their cursors and keep them up to date when data moves. + * Exception: An xcursor's pointer to a #P_SUBP page can be stale. + * (A node with #F_DUPDATA but no #F_SUBDATA contains a subpage). + */ +struct MDB_cursor { + /** Next cursor on this DB in this txn */ + MDB_cursor *mc_next; + /** Backup of the original cursor if this cursor is a shadow */ + MDB_cursor *mc_backup; + /** Context used for databases with #MDB_DUPSORT, otherwise NULL */ + struct MDB_xcursor *mc_xcursor; + /** The transaction that owns this cursor */ + MDB_txn *mc_txn; + /** The database handle this cursor operates on */ + MDB_dbi mc_dbi; + /** The database record for this cursor */ + MDB_db *mc_db; + /** The database auxiliary record for this cursor */ + MDB_dbx *mc_dbx; + /** The @ref mt_dbflag for this database */ + unsigned char *mc_dbflag; + unsigned short mc_snum; /**< number of pushed pages */ + unsigned short mc_top; /**< index of top page, normally mc_snum-1 */ +/** @defgroup mdb_cursor Cursor Flags + * @ingroup internal + * Cursor state flags. + * @{ + */ +#define C_INITIALIZED 0x01 /**< cursor has been initialized and is valid */ +#define C_EOF 0x02 /**< No more data */ +#define C_SUB 0x04 /**< Cursor is a sub-cursor */ +#define C_DEL 0x08 /**< last op was a cursor_del */ +#define C_SPLITTING 0x20 /**< Cursor is in page_split */ +#define C_UNTRACK 0x40 /**< Un-track cursor when closing */ +/** @} */ + unsigned int mc_flags; /**< @ref mdb_cursor */ + MDB_page *mc_pg[CURSOR_STACK]; /**< stack of pushed pages */ + indx_t mc_ki[CURSOR_STACK]; /**< stack of page indices */ +}; + + /** Context for sorted-dup records. + * We could have gone to a fully recursive design, with arbitrarily + * deep nesting of sub-databases. But for now we only handle these + * levels - main DB, optional sub-DB, sorted-duplicate DB. + */ +typedef struct MDB_xcursor { + /** A sub-cursor for traversing the Dup DB */ + MDB_cursor mx_cursor; + /** The database record for this Dup DB */ + MDB_db mx_db; + /** The auxiliary DB record for this Dup DB */ + MDB_dbx mx_dbx; + /** The @ref mt_dbflag for this Dup DB */ + unsigned char mx_dbflag; +} MDB_xcursor; + + /** State of FreeDB old pages, stored in the MDB_env */ +typedef struct MDB_pgstate { + pgno_t *mf_pghead; /**< Reclaimed freeDB pages, or NULL before use */ + txnid_t mf_pglast; /**< ID of last used record, or 0 if !mf_pghead */ +} MDB_pgstate; + + /** The database environment. */ +struct MDB_env { + HANDLE me_fd; /**< The main data file */ + HANDLE me_lfd; /**< The lock file */ + HANDLE me_mfd; /**< just for writing the meta pages */ +#if defined(VL32) && defined(_WIN32) + HANDLE me_fmh; /**< File Mapping handle */ +#endif + /** Failed to update the meta page. Probably an I/O error. */ +#define MDB_FATAL_ERROR 0x80000000U + /** Some fields are initialized. */ +#define MDB_ENV_ACTIVE 0x20000000U + /** me_txkey is set */ +#define MDB_ENV_TXKEY 0x10000000U + uint32_t me_flags; /**< @ref mdb_env */ + unsigned int me_psize; /**< DB page size, inited from me_os_psize */ + unsigned int me_os_psize; /**< OS page size, from #GET_PAGESIZE */ + unsigned int me_maxreaders; /**< size of the reader table */ + unsigned int me_numreaders; /**< max numreaders set by this env */ + MDB_dbi me_numdbs; /**< number of DBs opened */ + MDB_dbi me_maxdbs; /**< size of the DB table */ + MDB_PID_T me_pid; /**< process ID of this env */ + char *me_path; /**< path to the DB files */ + char *me_map; /**< the memory map of the data file */ + MDB_txninfo *me_txns; /**< the memory map of the lock file or NULL */ + MDB_meta *me_metas[2]; /**< pointers to the two meta pages */ + void *me_pbuf; /**< scratch area for DUPSORT put() */ + MDB_txn *me_txn; /**< current write transaction */ + MDB_txn *me_txn0; /**< prealloc'd write transaction */ + size_t me_mapsize; /**< size of the data memory map */ + off_t me_size; /**< current file size */ + pgno_t me_maxpg; /**< me_mapsize / me_psize */ + MDB_dbx *me_dbxs; /**< array of static DB info */ + uint16_t *me_dbflags; /**< array of flags from MDB_db.md_flags */ + unsigned int *me_dbiseqs; /**< array of dbi sequence numbers */ + pthread_key_t me_txkey; /**< thread-key for readers */ + txnid_t me_pgoldest; /**< ID of oldest reader last time we looked */ + MDB_pgstate me_pgstate; /**< state of old pages from freeDB */ +# define me_pglast me_pgstate.mf_pglast +# define me_pghead me_pgstate.mf_pghead + MDB_page *me_dpages; /**< list of malloc'd blocks for re-use */ + /** IDL of pages that became unused in a write txn */ + MDB_IDL me_free_pgs; + /** ID2L of pages written during a write txn. Length MDB_IDL_UM_SIZE. */ + MDB_ID2L me_dirty_list; + /** Max number of freelist items that can fit in a single overflow page */ + int me_maxfree_1pg; + /** Max size of a node on a page */ + unsigned int me_nodemax; +#if !(MDB_MAXKEYSIZE) + unsigned int me_maxkey; /**< max size of a key */ +#endif + int me_live_reader; /**< have liveness lock in reader table */ +#ifdef _WIN32 + int me_pidquery; /**< Used in OpenProcess */ + HANDLE me_rmutex; /* Windows mutexes don't reside in shared mem */ + HANDLE me_wmutex; +#elif defined(MDB_USE_POSIX_SEM) + sem_t *me_rmutex; /* Shared mutexes are not supported */ + sem_t *me_wmutex; +#endif + void *me_userctx; /**< User-settable context */ + MDB_assert_func *me_assert_func; /**< Callback for assertion failures */ +}; + + /** Nested transaction */ +typedef struct MDB_ntxn { + MDB_txn mnt_txn; /**< the transaction */ + MDB_pgstate mnt_pgstate; /**< parent transaction's saved freestate */ +} MDB_ntxn; + + /** max number of pages to commit in one writev() call */ +#define MDB_COMMIT_PAGES 64 +#if defined(IOV_MAX) && IOV_MAX < MDB_COMMIT_PAGES +#undef MDB_COMMIT_PAGES +#define MDB_COMMIT_PAGES IOV_MAX +#endif + + /** max bytes to write in one call */ +#define MAX_WRITE (0x80000000U >> (sizeof(ssize_t) == 4)) + + /** Check \b txn and \b dbi arguments to a function */ +#define TXN_DBI_EXIST(txn, dbi) \ + ((txn) && (dbi) < (txn)->mt_numdbs && ((txn)->mt_dbflags[dbi] & DB_VALID)) + + /** Check for misused \b dbi handles */ +#define TXN_DBI_CHANGED(txn, dbi) \ + ((txn)->mt_dbiseqs[dbi] != (txn)->mt_env->me_dbiseqs[dbi]) + +static int mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp); +static int mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp); +static int mdb_page_touch(MDB_cursor *mc); + +static int mdb_page_get(MDB_txn *txn, pgno_t pgno, MDB_page **mp, int *lvl); +static int mdb_page_search_root(MDB_cursor *mc, + MDB_val *key, int modify); +#define MDB_PS_MODIFY 1 +#define MDB_PS_ROOTONLY 2 +#define MDB_PS_FIRST 4 +#define MDB_PS_LAST 8 +static int mdb_page_search(MDB_cursor *mc, + MDB_val *key, int flags); +static int mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst); + +#define MDB_SPLIT_REPLACE MDB_APPENDDUP /**< newkey is not new */ +static int mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, + pgno_t newpgno, unsigned int nflags); + +static int mdb_env_read_header(MDB_env *env, MDB_meta *meta); +static int mdb_env_pick_meta(const MDB_env *env); +static int mdb_env_write_meta(MDB_txn *txn); +#if !(defined(_WIN32) || defined(MDB_USE_POSIX_SEM)) /* Drop unused excl arg */ +# define mdb_env_close0(env, excl) mdb_env_close1(env) +#endif +static void mdb_env_close0(MDB_env *env, int excl); + +static MDB_node *mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp); +static int mdb_node_add(MDB_cursor *mc, indx_t indx, + MDB_val *key, MDB_val *data, pgno_t pgno, unsigned int flags); +static void mdb_node_del(MDB_cursor *mc, int ksize); +static void mdb_node_shrink(MDB_page *mp, indx_t indx); +static int mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst); +static int mdb_node_read(MDB_txn *txn, MDB_node *leaf, MDB_val *data); +static size_t mdb_leaf_size(MDB_env *env, MDB_val *key, MDB_val *data); +static size_t mdb_branch_size(MDB_env *env, MDB_val *key); + +static int mdb_rebalance(MDB_cursor *mc); +static int mdb_update_key(MDB_cursor *mc, MDB_val *key); + +static void mdb_cursor_pop(MDB_cursor *mc); +static int mdb_cursor_push(MDB_cursor *mc, MDB_page *mp); + +static int mdb_cursor_del0(MDB_cursor *mc); +static int mdb_del0(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data, unsigned flags); +static int mdb_cursor_sibling(MDB_cursor *mc, int move_right); +static int mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op); +static int mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op); +static int mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op, + int *exactp); +static int mdb_cursor_first(MDB_cursor *mc, MDB_val *key, MDB_val *data); +static int mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data); + +static void mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx); +static void mdb_xcursor_init0(MDB_cursor *mc); +static void mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node); + +static int mdb_drop0(MDB_cursor *mc, int subs); +static void mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi); + +/** @cond */ +static MDB_cmp_func mdb_cmp_memn, mdb_cmp_memnr, mdb_cmp_int, mdb_cmp_cint, mdb_cmp_long; +/** @endcond */ + +#ifdef _WIN32 +static SECURITY_DESCRIPTOR mdb_null_sd; +static SECURITY_ATTRIBUTES mdb_all_sa; +static int mdb_sec_inited; +#endif + +/** Return the library version info. */ +char * +mdb_version(int *major, int *minor, int *patch) +{ + if (major) *major = MDB_VERSION_MAJOR; + if (minor) *minor = MDB_VERSION_MINOR; + if (patch) *patch = MDB_VERSION_PATCH; + return MDB_VERSION_STRING; +} + +/** Table of descriptions for LMDB @ref errors */ +static char *const mdb_errstr[] = { + "MDB_KEYEXIST: Key/data pair already exists", + "MDB_NOTFOUND: No matching key/data pair found", + "MDB_PAGE_NOTFOUND: Requested page not found", + "MDB_CORRUPTED: Located page was wrong type", + "MDB_PANIC: Update of meta page failed", + "MDB_VERSION_MISMATCH: Database environment version mismatch", + "MDB_INVALID: File is not an LMDB file", + "MDB_MAP_FULL: Environment mapsize limit reached", + "MDB_DBS_FULL: Environment maxdbs limit reached", + "MDB_READERS_FULL: Environment maxreaders limit reached", + "MDB_TLS_FULL: Thread-local storage keys full - too many environments open", + "MDB_TXN_FULL: Transaction has too many dirty pages - transaction too big", + "MDB_CURSOR_FULL: Internal error - cursor stack limit reached", + "MDB_PAGE_FULL: Internal error - page has no more space", + "MDB_MAP_RESIZED: Database contents grew beyond environment mapsize", + "MDB_INCOMPATIBLE: Operation and DB incompatible, or DB flags changed", + "MDB_BAD_RSLOT: Invalid reuse of reader locktable slot", + "MDB_BAD_TXN: Transaction cannot recover - it must be aborted", + "MDB_BAD_VALSIZE: Unsupported size of key/DB name/data, or wrong DUPFIXED size", + "MDB_BAD_DBI: The specified DBI handle was closed/changed unexpectedly", +}; + +char * +mdb_strerror(int err) +{ +#ifdef _WIN32 + /** HACK: pad 4KB on stack over the buf. Return system msgs in buf. + * This works as long as no function between the call to mdb_strerror + * and the actual use of the message uses more than 4K of stack. + */ + char pad[4096]; + char buf[1024], *ptr = buf; +#endif + int i; + if (!err) + return ("Successful return: 0"); + + if (err >= MDB_KEYEXIST && err <= MDB_LAST_ERRCODE) { + i = err - MDB_KEYEXIST; + return mdb_errstr[i]; + } + +#ifdef _WIN32 + /* These are the C-runtime error codes we use. The comment indicates + * their numeric value, and the Win32 error they would correspond to + * if the error actually came from a Win32 API. A major mess, we should + * have used LMDB-specific error codes for everything. + */ + switch(err) { + case ENOENT: /* 2, FILE_NOT_FOUND */ + case EIO: /* 5, ACCESS_DENIED */ + case ENOMEM: /* 12, INVALID_ACCESS */ + case EACCES: /* 13, INVALID_DATA */ + case EBUSY: /* 16, CURRENT_DIRECTORY */ + case EINVAL: /* 22, BAD_COMMAND */ + case ENOSPC: /* 28, OUT_OF_PAPER */ + return strerror(err); + default: + ; + } + buf[0] = 0; + FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, err, 0, ptr, sizeof(buf), pad); + return ptr; +#else + return strerror(err); +#endif +} + +/** assert(3) variant in cursor context */ +#define mdb_cassert(mc, expr) mdb_assert0((mc)->mc_txn->mt_env, expr, #expr) +/** assert(3) variant in transaction context */ +#define mdb_tassert(mc, expr) mdb_assert0((txn)->mt_env, expr, #expr) +/** assert(3) variant in environment context */ +#define mdb_eassert(env, expr) mdb_assert0(env, expr, #expr) + +#ifndef NDEBUG +# define mdb_assert0(env, expr, expr_txt) ((expr) ? (void)0 : \ + mdb_assert_fail(env, expr_txt, mdb_func_, __FILE__, __LINE__)) + +static void +mdb_assert_fail(MDB_env *env, const char *expr_txt, + const char *func, const char *file, int line) +{ + char buf[400]; + sprintf(buf, "%.100s:%d: Assertion '%.200s' failed in %.40s()", + file, line, expr_txt, func); + if (env->me_assert_func) + env->me_assert_func(env, buf); + fprintf(stderr, "%s\n", buf); + abort(); +} +#else +# define mdb_assert0(env, expr, expr_txt) ((void) 0) +#endif /* NDEBUG */ + +#if MDB_DEBUG +/** Return the page number of \b mp which may be sub-page, for debug output */ +static pgno_t +mdb_dbg_pgno(MDB_page *mp) +{ + pgno_t ret; + COPY_PGNO(ret, mp->mp_pgno); + return ret; +} + +/** Display a key in hexadecimal and return the address of the result. + * @param[in] key the key to display + * @param[in] buf the buffer to write into. Should always be #DKBUF. + * @return The key in hexadecimal form. + */ +char * +mdb_dkey(MDB_val *key, char *buf) +{ + char *ptr = buf; + unsigned char *c = key->mv_data; + unsigned int i; + + if (!key) + return ""; + + if (key->mv_size > DKBUF_MAXKEYSIZE) + return "MDB_MAXKEYSIZE"; + /* may want to make this a dynamic check: if the key is mostly + * printable characters, print it as-is instead of converting to hex. + */ +#if 1 + buf[0] = '\0'; + for (i=0; i<key->mv_size; i++) + ptr += sprintf(ptr, "%02x", *c++); +#else + sprintf(buf, "%.*s", key->mv_size, key->mv_data); +#endif + return buf; +} + +static const char * +mdb_leafnode_type(MDB_node *n) +{ + static char *const tp[2][2] = {{"", ": DB"}, {": sub-page", ": sub-DB"}}; + return F_ISSET(n->mn_flags, F_BIGDATA) ? ": overflow page" : + tp[F_ISSET(n->mn_flags, F_DUPDATA)][F_ISSET(n->mn_flags, F_SUBDATA)]; +} + +/** Display all the keys in the page. */ +void +mdb_page_list(MDB_page *mp) +{ + pgno_t pgno = mdb_dbg_pgno(mp); + const char *type, *state = (mp->mp_flags & P_DIRTY) ? ", dirty" : ""; + MDB_node *node; + unsigned int i, nkeys, nsize, total = 0; + MDB_val key; + DKBUF; + + switch (mp->mp_flags & (P_BRANCH|P_LEAF|P_LEAF2|P_META|P_OVERFLOW|P_SUBP)) { + case P_BRANCH: type = "Branch page"; break; + case P_LEAF: type = "Leaf page"; break; + case P_LEAF|P_SUBP: type = "Sub-page"; break; + case P_LEAF|P_LEAF2: type = "LEAF2 page"; break; + case P_LEAF|P_LEAF2|P_SUBP: type = "LEAF2 sub-page"; break; + case P_OVERFLOW: + fprintf(stderr, "Overflow page %"Z"u pages %u%s\n", + pgno, mp->mp_pages, state); + return; + case P_META: + fprintf(stderr, "Meta-page %"Z"u txnid %"Z"u\n", + pgno, ((MDB_meta *)METADATA(mp))->mm_txnid); + return; + default: + fprintf(stderr, "Bad page %"Z"u flags 0x%u\n", pgno, mp->mp_flags); + return; + } + + nkeys = NUMKEYS(mp); + fprintf(stderr, "%s %"Z"u numkeys %d%s\n", type, pgno, nkeys, state); + + for (i=0; i<nkeys; i++) { + if (IS_LEAF2(mp)) { /* LEAF2 pages have no mp_ptrs[] or node headers */ + key.mv_size = nsize = mp->mp_pad; + key.mv_data = LEAF2KEY(mp, i, nsize); + total += nsize; + fprintf(stderr, "key %d: nsize %d, %s\n", i, nsize, DKEY(&key)); + continue; + } + node = NODEPTR(mp, i); + key.mv_size = node->mn_ksize; + key.mv_data = node->mn_data; + nsize = NODESIZE + key.mv_size; + if (IS_BRANCH(mp)) { + fprintf(stderr, "key %d: page %"Z"u, %s\n", i, NODEPGNO(node), + DKEY(&key)); + total += nsize; + } else { + if (F_ISSET(node->mn_flags, F_BIGDATA)) + nsize += sizeof(pgno_t); + else + nsize += NODEDSZ(node); + total += nsize; + nsize += sizeof(indx_t); + fprintf(stderr, "key %d: nsize %d, %s%s\n", + i, nsize, DKEY(&key), mdb_leafnode_type(node)); + } + total = EVEN(total); + } + fprintf(stderr, "Total: header %d + contents %d + unused %d\n", + IS_LEAF2(mp) ? PAGEHDRSZ : PAGEBASE + mp->mp_lower, total, SIZELEFT(mp)); +} + +void +mdb_cursor_chk(MDB_cursor *mc) +{ + unsigned int i; + MDB_node *node; + MDB_page *mp; + + if (!mc->mc_snum && !(mc->mc_flags & C_INITIALIZED)) return; + for (i=0; i<mc->mc_top; i++) { + mp = mc->mc_pg[i]; + node = NODEPTR(mp, mc->mc_ki[i]); + if (NODEPGNO(node) != mc->mc_pg[i+1]->mp_pgno) + printf("oops!\n"); + } + if (mc->mc_ki[i] >= NUMKEYS(mc->mc_pg[i])) + printf("ack!\n"); +} +#endif + +#if (MDB_DEBUG) > 2 +/** Count all the pages in each DB and in the freelist + * and make sure it matches the actual number of pages + * being used. + * All named DBs must be open for a correct count. + */ +static void mdb_audit(MDB_txn *txn) +{ + MDB_cursor mc; + MDB_val key, data; + MDB_ID freecount, count; + MDB_dbi i; + int rc; + + freecount = 0; + mdb_cursor_init(&mc, txn, FREE_DBI, NULL); + while ((rc = mdb_cursor_get(&mc, &key, &data, MDB_NEXT)) == 0) + freecount += *(MDB_ID *)data.mv_data; + mdb_tassert(txn, rc == MDB_NOTFOUND); + + count = 0; + for (i = 0; i<txn->mt_numdbs; i++) { + MDB_xcursor mx; + if (!(txn->mt_dbflags[i] & DB_VALID)) + continue; + mdb_cursor_init(&mc, txn, i, &mx); + if (txn->mt_dbs[i].md_root == P_INVALID) + continue; + count += txn->mt_dbs[i].md_branch_pages + + txn->mt_dbs[i].md_leaf_pages + + txn->mt_dbs[i].md_overflow_pages; + if (txn->mt_dbs[i].md_flags & MDB_DUPSORT) { + rc = mdb_page_search(&mc, NULL, MDB_PS_FIRST); + for (; rc == MDB_SUCCESS; rc = mdb_cursor_sibling(&mc, 1)) { + unsigned j; + MDB_page *mp; + mp = mc.mc_pg[mc.mc_top]; + for (j=0; j<NUMKEYS(mp); j++) { + MDB_node *leaf = NODEPTR(mp, j); + if (leaf->mn_flags & F_SUBDATA) { + MDB_db db; + memcpy(&db, NODEDATA(leaf), sizeof(db)); + count += db.md_branch_pages + db.md_leaf_pages + + db.md_overflow_pages; + } + } + } + mdb_tassert(txn, rc == MDB_NOTFOUND); + } + } + if (freecount + count + 2 /* metapages */ != txn->mt_next_pgno) { + fprintf(stderr, "audit: %lu freecount: %lu count: %lu total: %lu next_pgno: %lu\n", + txn->mt_txnid, freecount, count+2, freecount+count+2, txn->mt_next_pgno); + } +} +#endif + +int +mdb_cmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b) +{ + return txn->mt_dbxs[dbi].md_cmp(a, b); +} + +int +mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b) +{ + return txn->mt_dbxs[dbi].md_dcmp(a, b); +} + +/** Allocate memory for a page. + * Re-use old malloc'd pages first for singletons, otherwise just malloc. + */ +static MDB_page * +mdb_page_malloc(MDB_txn *txn, unsigned num) +{ + MDB_env *env = txn->mt_env; + MDB_page *ret = env->me_dpages; + size_t psize = env->me_psize, sz = psize, off; + /* For ! #MDB_NOMEMINIT, psize counts how much to init. + * For a single page alloc, we init everything after the page header. + * For multi-page, we init the final page; if the caller needed that + * many pages they will be filling in at least up to the last page. + */ + if (num == 1) { + if (ret) { + VGMEMP_ALLOC(env, ret, sz); + VGMEMP_DEFINED(ret, sizeof(ret->mp_next)); + env->me_dpages = ret->mp_next; + return ret; + } + psize -= off = PAGEHDRSZ; + } else { + sz *= num; + off = sz - psize; + } + if ((ret = malloc(sz)) != NULL) { + VGMEMP_ALLOC(env, ret, sz); + if (!(env->me_flags & MDB_NOMEMINIT)) { + memset((char *)ret + off, 0, psize); + ret->mp_pad = 0; + } + } else { + txn->mt_flags |= MDB_TXN_ERROR; + } + return ret; +} +/** Free a single page. + * Saves single pages to a list, for future reuse. + * (This is not used for multi-page overflow pages.) + */ +static void +mdb_page_free(MDB_env *env, MDB_page *mp) +{ + mp->mp_next = env->me_dpages; + VGMEMP_FREE(env, mp); + env->me_dpages = mp; +} + +/** Free a dirty page */ +static void +mdb_dpage_free(MDB_env *env, MDB_page *dp) +{ + if (!IS_OVERFLOW(dp) || dp->mp_pages == 1) { + mdb_page_free(env, dp); + } else { + /* large pages just get freed directly */ + VGMEMP_FREE(env, dp); + free(dp); + } +} + +/** Return all dirty pages to dpage list */ +static void +mdb_dlist_free(MDB_txn *txn) +{ + MDB_env *env = txn->mt_env; + MDB_ID2L dl = txn->mt_u.dirty_list; + unsigned i, n = dl[0].mid; + + for (i = 1; i <= n; i++) { + mdb_dpage_free(env, dl[i].mptr); + } + dl[0].mid = 0; +} + +/** Loosen or free a single page. + * Saves single pages to a list for future reuse + * in this same txn. It has been pulled from the freeDB + * and already resides on the dirty list, but has been + * deleted. Use these pages first before pulling again + * from the freeDB. + * + * If the page wasn't dirtied in this txn, just add it + * to this txn's free list. + */ +static int +mdb_page_loose(MDB_cursor *mc, MDB_page *mp) +{ + int loose = 0; + pgno_t pgno = mp->mp_pgno; + MDB_txn *txn = mc->mc_txn; + + if ((mp->mp_flags & P_DIRTY) && mc->mc_dbi != FREE_DBI) { + if (txn->mt_parent) { + MDB_ID2 *dl = txn->mt_u.dirty_list; + /* If txn has a parent, make sure the page is in our + * dirty list. + */ + if (dl[0].mid) { + unsigned x = mdb_mid2l_search(dl, pgno); + if (x <= dl[0].mid && dl[x].mid == pgno) { + if (mp != dl[x].mptr) { /* bad cursor? */ + mc->mc_flags &= ~(C_INITIALIZED|C_EOF); + txn->mt_flags |= MDB_TXN_ERROR; + return MDB_CORRUPTED; + } + /* ok, it's ours */ + loose = 1; + } + } + } else { + /* no parent txn, so it's just ours */ + loose = 1; + } + } + if (loose) { + DPRINTF(("loosen db %d page %"Z"u", DDBI(mc), + mp->mp_pgno)); + NEXT_LOOSE_PAGE(mp) = txn->mt_loose_pgs; + txn->mt_loose_pgs = mp; + txn->mt_loose_count++; + mp->mp_flags |= P_LOOSE; + } else { + int rc = mdb_midl_append(&txn->mt_free_pgs, pgno); + if (rc) + return rc; + } + + return MDB_SUCCESS; +} + +/** Set or clear P_KEEP in dirty, non-overflow, non-sub pages watched by txn. + * @param[in] mc A cursor handle for the current operation. + * @param[in] pflags Flags of the pages to update: + * P_DIRTY to set P_KEEP, P_DIRTY|P_KEEP to clear it. + * @param[in] all No shortcuts. Needed except after a full #mdb_page_flush(). + * @return 0 on success, non-zero on failure. + */ +static int +mdb_pages_xkeep(MDB_cursor *mc, unsigned pflags, int all) +{ + enum { Mask = P_SUBP|P_DIRTY|P_LOOSE|P_KEEP }; + MDB_txn *txn = mc->mc_txn; + MDB_cursor *m3; + MDB_xcursor *mx; + MDB_page *dp, *mp; + MDB_node *leaf; + unsigned i, j; + int rc = MDB_SUCCESS, level; + + /* Mark pages seen by cursors */ + if (mc->mc_flags & C_UNTRACK) + mc = NULL; /* will find mc in mt_cursors */ + for (i = txn->mt_numdbs;; mc = txn->mt_cursors[--i]) { + for (; mc; mc=mc->mc_next) { + if (!(mc->mc_flags & C_INITIALIZED)) + continue; + for (m3 = mc;; m3 = &mx->mx_cursor) { + mp = NULL; + for (j=0; j<m3->mc_snum; j++) { + mp = m3->mc_pg[j]; + if ((mp->mp_flags & Mask) == pflags) + mp->mp_flags ^= P_KEEP; + } + mx = m3->mc_xcursor; + /* Proceed to mx if it is at a sub-database */ + if (! (mx && (mx->mx_cursor.mc_flags & C_INITIALIZED))) + break; + if (! (mp && (mp->mp_flags & P_LEAF))) + break; + leaf = NODEPTR(mp, m3->mc_ki[j-1]); + if (!(leaf->mn_flags & F_SUBDATA)) + break; + } + } + if (i == 0) + break; + } + + if (all) { + /* Mark dirty root pages */ + for (i=0; i<txn->mt_numdbs; i++) { + if (txn->mt_dbflags[i] & DB_DIRTY) { + pgno_t pgno = txn->mt_dbs[i].md_root; + if (pgno == P_INVALID) + continue; + if ((rc = mdb_page_get(txn, pgno, &dp, &level)) != MDB_SUCCESS) + break; + if ((dp->mp_flags & Mask) == pflags && level <= 1) + dp->mp_flags ^= P_KEEP; + } + } + } + + return rc; +} + +static int mdb_page_flush(MDB_txn *txn, int keep); + +/** Spill pages from the dirty list back to disk. + * This is intended to prevent running into #MDB_TXN_FULL situations, + * but note that they may still occur in a few cases: + * 1) our estimate of the txn size could be too small. Currently this + * seems unlikely, except with a large number of #MDB_MULTIPLE items. + * 2) child txns may run out of space if their parents dirtied a + * lot of pages and never spilled them. TODO: we probably should do + * a preemptive spill during #mdb_txn_begin() of a child txn, if + * the parent's dirty_room is below a given threshold. + * + * Otherwise, if not using nested txns, it is expected that apps will + * not run into #MDB_TXN_FULL any more. The pages are flushed to disk + * the same way as for a txn commit, e.g. their P_DIRTY flag is cleared. + * If the txn never references them again, they can be left alone. + * If the txn only reads them, they can be used without any fuss. + * If the txn writes them again, they can be dirtied immediately without + * going thru all of the work of #mdb_page_touch(). Such references are + * handled by #mdb_page_unspill(). + * + * Also note, we never spill DB root pages, nor pages of active cursors, + * because we'll need these back again soon anyway. And in nested txns, + * we can't spill a page in a child txn if it was already spilled in a + * parent txn. That would alter the parent txns' data even though + * the child hasn't committed yet, and we'd have no way to undo it if + * the child aborted. + * + * @param[in] m0 cursor A cursor handle identifying the transaction and + * database for which we are checking space. + * @param[in] key For a put operation, the key being stored. + * @param[in] data For a put operation, the data being stored. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_spill(MDB_cursor *m0, MDB_val *key, MDB_val *data) +{ + MDB_txn *txn = m0->mc_txn; + MDB_page *dp; + MDB_ID2L dl = txn->mt_u.dirty_list; + unsigned int i, j, need; + int rc; + + if (m0->mc_flags & C_SUB) + return MDB_SUCCESS; + + /* Estimate how much space this op will take */ + i = m0->mc_db->md_depth; + /* Named DBs also dirty the main DB */ + if (m0->mc_dbi > MAIN_DBI) + i += txn->mt_dbs[MAIN_DBI].md_depth; + /* For puts, roughly factor in the key+data size */ + if (key) + i += (LEAFSIZE(key, data) + txn->mt_env->me_psize) / txn->mt_env->me_psize; + i += i; /* double it for good measure */ + need = i; + + if (txn->mt_dirty_room > i) + return MDB_SUCCESS; + + if (!txn->mt_spill_pgs) { + txn->mt_spill_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX); + if (!txn->mt_spill_pgs) + return ENOMEM; + } else { + /* purge deleted slots */ + MDB_IDL sl = txn->mt_spill_pgs; + unsigned int num = sl[0]; + j=0; + for (i=1; i<=num; i++) { + if (!(sl[i] & 1)) + sl[++j] = sl[i]; + } + sl[0] = j; + } + + /* Preserve pages which may soon be dirtied again */ + if ((rc = mdb_pages_xkeep(m0, P_DIRTY, 1)) != MDB_SUCCESS) + goto done; + + /* Less aggressive spill - we originally spilled the entire dirty list, + * with a few exceptions for cursor pages and DB root pages. But this + * turns out to be a lot of wasted effort because in a large txn many + * of those pages will need to be used again. So now we spill only 1/8th + * of the dirty pages. Testing revealed this to be a good tradeoff, + * better than 1/2, 1/4, or 1/10. + */ + if (need < MDB_IDL_UM_MAX / 8) + need = MDB_IDL_UM_MAX / 8; + + /* Save the page IDs of all the pages we're flushing */ + /* flush from the tail forward, this saves a lot of shifting later on. */ + for (i=dl[0].mid; i && need; i--) { + MDB_ID pn = dl[i].mid << 1; + dp = dl[i].mptr; + if (dp->mp_flags & (P_LOOSE|P_KEEP)) + continue; + /* Can't spill twice, make sure it's not already in a parent's + * spill list. + */ + if (txn->mt_parent) { + MDB_txn *tx2; + for (tx2 = txn->mt_parent; tx2; tx2 = tx2->mt_parent) { + if (tx2->mt_spill_pgs) { + j = mdb_midl_search(tx2->mt_spill_pgs, pn); + if (j <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[j] == pn) { + dp->mp_flags |= P_KEEP; + break; + } + } + } + if (tx2) + continue; + } + if ((rc = mdb_midl_append(&txn->mt_spill_pgs, pn))) + goto done; + need--; + } + mdb_midl_sort(txn->mt_spill_pgs); + + /* Flush the spilled part of dirty list */ + if ((rc = mdb_page_flush(txn, i)) != MDB_SUCCESS) + goto done; + + /* Reset any dirty pages we kept that page_flush didn't see */ + rc = mdb_pages_xkeep(m0, P_DIRTY|P_KEEP, i); + +done: + txn->mt_flags |= rc ? MDB_TXN_ERROR : MDB_TXN_SPILLS; + return rc; +} + +/** Find oldest txnid still referenced. Expects txn->mt_txnid > 0. */ +static txnid_t +mdb_find_oldest(MDB_txn *txn) +{ + int i; + txnid_t mr, oldest = txn->mt_txnid - 1; + if (txn->mt_env->me_txns) { + MDB_reader *r = txn->mt_env->me_txns->mti_readers; + for (i = txn->mt_env->me_txns->mti_numreaders; --i >= 0; ) { + if (r[i].mr_pid) { + mr = r[i].mr_txnid; + if (oldest > mr) + oldest = mr; + } + } + } + return oldest; +} + +/** Add a page to the txn's dirty list */ +static void +mdb_page_dirty(MDB_txn *txn, MDB_page *mp) +{ + MDB_ID2 mid; + int rc, (*insert)(MDB_ID2L, MDB_ID2 *); + + if (txn->mt_env->me_flags & MDB_WRITEMAP) { + insert = mdb_mid2l_append; + } else { + insert = mdb_mid2l_insert; + } + mid.mid = mp->mp_pgno; + mid.mptr = mp; + rc = insert(txn->mt_u.dirty_list, &mid); + mdb_tassert(txn, rc == 0); + txn->mt_dirty_room--; +} + +/** Allocate page numbers and memory for writing. Maintain me_pglast, + * me_pghead and mt_next_pgno. + * + * If there are free pages available from older transactions, they + * are re-used first. Otherwise allocate a new page at mt_next_pgno. + * Do not modify the freedB, just merge freeDB records into me_pghead[] + * and move me_pglast to say which records were consumed. Only this + * function can create me_pghead and move me_pglast/mt_next_pgno. + * @param[in] mc cursor A cursor handle identifying the transaction and + * database for which we are allocating. + * @param[in] num the number of pages to allocate. + * @param[out] mp Address of the allocated page(s). Requests for multiple pages + * will always be satisfied by a single contiguous chunk of memory. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) +{ +#ifdef MDB_PARANOID /* Seems like we can ignore this now */ + /* Get at most <Max_retries> more freeDB records once me_pghead + * has enough pages. If not enough, use new pages from the map. + * If <Paranoid> and mc is updating the freeDB, only get new + * records if me_pghead is empty. Then the freelist cannot play + * catch-up with itself by growing while trying to save it. + */ + enum { Paranoid = 1, Max_retries = 500 }; +#else + enum { Paranoid = 0, Max_retries = INT_MAX /*infinite*/ }; +#endif + int rc, retry = num * 60; + MDB_txn *txn = mc->mc_txn; + MDB_env *env = txn->mt_env; + pgno_t pgno, *mop = env->me_pghead; + unsigned i, j, mop_len = mop ? mop[0] : 0, n2 = num-1; + MDB_page *np; + txnid_t oldest = 0, last; + MDB_cursor_op op; + MDB_cursor m2; + int found_old = 0; + + /* If there are any loose pages, just use them */ + if (num == 1 && txn->mt_loose_pgs) { + np = txn->mt_loose_pgs; + txn->mt_loose_pgs = NEXT_LOOSE_PAGE(np); + txn->mt_loose_count--; + DPRINTF(("db %d use loose page %"Z"u", DDBI(mc), + np->mp_pgno)); + *mp = np; + return MDB_SUCCESS; + } + + *mp = NULL; + + /* If our dirty list is already full, we can't do anything */ + if (txn->mt_dirty_room == 0) { + rc = MDB_TXN_FULL; + goto fail; + } + + for (op = MDB_FIRST;; op = MDB_NEXT) { + MDB_val key, data; + MDB_node *leaf; + pgno_t *idl; + + /* Seek a big enough contiguous page range. Prefer + * pages at the tail, just truncating the list. + */ + if (mop_len > n2) { + i = mop_len; + do { + pgno = mop[i]; + if (mop[i-n2] == pgno+n2) + goto search_done; + } while (--i > n2); + if (--retry < 0) + break; + } + + if (op == MDB_FIRST) { /* 1st iteration */ + /* Prepare to fetch more and coalesce */ + last = env->me_pglast; + oldest = env->me_pgoldest; + mdb_cursor_init(&m2, txn, FREE_DBI, NULL); + if (last) { + op = MDB_SET_RANGE; + key.mv_data = &last; /* will look up last+1 */ + key.mv_size = sizeof(last); + } + if (Paranoid && mc->mc_dbi == FREE_DBI) + retry = -1; + } + if (Paranoid && retry < 0 && mop_len) + break; + + last++; + /* Do not fetch more if the record will be too recent */ + if (oldest <= last) { + if (!found_old) { + oldest = mdb_find_oldest(txn); + env->me_pgoldest = oldest; + found_old = 1; + } + if (oldest <= last) + break; + } + rc = mdb_cursor_get(&m2, &key, NULL, op); + if (rc) { + if (rc == MDB_NOTFOUND) + break; + goto fail; + } + last = *(txnid_t*)key.mv_data; + if (oldest <= last) { + if (!found_old) { + oldest = mdb_find_oldest(txn); + env->me_pgoldest = oldest; + found_old = 1; + } + if (oldest <= last) + break; + } + np = m2.mc_pg[m2.mc_top]; + leaf = NODEPTR(np, m2.mc_ki[m2.mc_top]); + if ((rc = mdb_node_read(txn, leaf, &data)) != MDB_SUCCESS) + return rc; + + idl = (MDB_ID *) data.mv_data; + i = idl[0]; + if (!mop) { + if (!(env->me_pghead = mop = mdb_midl_alloc(i))) { + rc = ENOMEM; + goto fail; + } + } else { + if ((rc = mdb_midl_need(&env->me_pghead, i)) != 0) + goto fail; + mop = env->me_pghead; + } + env->me_pglast = last; +#if (MDB_DEBUG) > 1 + DPRINTF(("IDL read txn %"Z"u root %"Z"u num %u", + last, txn->mt_dbs[FREE_DBI].md_root, i)); + for (j = i; j; j--) + DPRINTF(("IDL %"Z"u", idl[j])); +#endif + /* Merge in descending sorted order */ + mdb_midl_xmerge(mop, idl); + mop_len = mop[0]; + } + + /* Use new pages from the map when nothing suitable in the freeDB */ + i = 0; + pgno = txn->mt_next_pgno; + if (pgno + num >= env->me_maxpg) { + DPUTS("DB size maxed out"); + rc = MDB_MAP_FULL; + goto fail; + } + +search_done: + if (env->me_flags & MDB_WRITEMAP) { + np = (MDB_page *)(env->me_map + env->me_psize * pgno); + } else { + if (!(np = mdb_page_malloc(txn, num))) { + rc = ENOMEM; + goto fail; + } + } + if (i) { + mop[0] = mop_len -= num; + /* Move any stragglers down */ + for (j = i-num; j < mop_len; ) + mop[++j] = mop[++i]; + } else { + txn->mt_next_pgno = pgno + num; + } + np->mp_pgno = pgno; + mdb_page_dirty(txn, np); + *mp = np; + + return MDB_SUCCESS; + +fail: + txn->mt_flags |= MDB_TXN_ERROR; + return rc; +} + +/** Copy the used portions of a non-overflow page. + * @param[in] dst page to copy into + * @param[in] src page to copy from + * @param[in] psize size of a page + */ +static void +mdb_page_copy(MDB_page *dst, MDB_page *src, unsigned int psize) +{ + enum { Align = sizeof(pgno_t) }; + indx_t upper = src->mp_upper, lower = src->mp_lower, unused = upper-lower; + + /* If page isn't full, just copy the used portion. Adjust + * alignment so memcpy may copy words instead of bytes. + */ + if ((unused &= -Align) && !IS_LEAF2(src)) { + upper = (upper + PAGEBASE) & -Align; + memcpy(dst, src, (lower + PAGEBASE + (Align-1)) & -Align); + memcpy((pgno_t *)((char *)dst+upper), (pgno_t *)((char *)src+upper), + psize - upper); + } else { + memcpy(dst, src, psize - unused); + } +} + +/** Pull a page off the txn's spill list, if present. + * If a page being referenced was spilled to disk in this txn, bring + * it back and make it dirty/writable again. + * @param[in] txn the transaction handle. + * @param[in] mp the page being referenced. It must not be dirty. + * @param[out] ret the writable page, if any. ret is unchanged if + * mp wasn't spilled. + */ +static int +mdb_page_unspill(MDB_txn *txn, MDB_page *mp, MDB_page **ret) +{ + MDB_env *env = txn->mt_env; + const MDB_txn *tx2; + unsigned x; + pgno_t pgno = mp->mp_pgno, pn = pgno << 1; + + for (tx2 = txn; tx2; tx2=tx2->mt_parent) { + if (!tx2->mt_spill_pgs) + continue; + x = mdb_midl_search(tx2->mt_spill_pgs, pn); + if (x <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[x] == pn) { + MDB_page *np; + int num; + if (txn->mt_dirty_room == 0) + return MDB_TXN_FULL; + if (IS_OVERFLOW(mp)) + num = mp->mp_pages; + else + num = 1; + if (env->me_flags & MDB_WRITEMAP) { + np = mp; + } else { + np = mdb_page_malloc(txn, num); + if (!np) + return ENOMEM; + if (num > 1) + memcpy(np, mp, num * env->me_psize); + else + mdb_page_copy(np, mp, env->me_psize); + } + if (tx2 == txn) { + /* If in current txn, this page is no longer spilled. + * If it happens to be the last page, truncate the spill list. + * Otherwise mark it as deleted by setting the LSB. + */ + if (x == txn->mt_spill_pgs[0]) + txn->mt_spill_pgs[0]--; + else + txn->mt_spill_pgs[x] |= 1; + } /* otherwise, if belonging to a parent txn, the + * page remains spilled until child commits + */ + + mdb_page_dirty(txn, np); + np->mp_flags |= P_DIRTY; + *ret = np; + break; + } + } + return MDB_SUCCESS; +} + +/** Touch a page: make it dirty and re-insert into tree with updated pgno. + * @param[in] mc cursor pointing to the page to be touched + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_touch(MDB_cursor *mc) +{ + MDB_page *mp = mc->mc_pg[mc->mc_top], *np; + MDB_txn *txn = mc->mc_txn; + MDB_cursor *m2, *m3; + pgno_t pgno; + int rc; + + if (!F_ISSET(mp->mp_flags, P_DIRTY)) { + if (txn->mt_flags & MDB_TXN_SPILLS) { + np = NULL; + rc = mdb_page_unspill(txn, mp, &np); + if (rc) + goto fail; + if (np) + goto done; + } + if ((rc = mdb_midl_need(&txn->mt_free_pgs, 1)) || + (rc = mdb_page_alloc(mc, 1, &np))) + goto fail; + pgno = np->mp_pgno; + DPRINTF(("touched db %d page %"Z"u -> %"Z"u", DDBI(mc), + mp->mp_pgno, pgno)); + mdb_cassert(mc, mp->mp_pgno != pgno); + mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno); + /* Update the parent page, if any, to point to the new page */ + if (mc->mc_top) { + MDB_page *parent = mc->mc_pg[mc->mc_top-1]; + MDB_node *node = NODEPTR(parent, mc->mc_ki[mc->mc_top-1]); + SETPGNO(node, pgno); + } else { + mc->mc_db->md_root = pgno; + } + } else if (txn->mt_parent && !IS_SUBP(mp)) { + MDB_ID2 mid, *dl = txn->mt_u.dirty_list; + pgno = mp->mp_pgno; + /* If txn has a parent, make sure the page is in our + * dirty list. + */ + if (dl[0].mid) { + unsigned x = mdb_mid2l_search(dl, pgno); + if (x <= dl[0].mid && dl[x].mid == pgno) { + if (mp != dl[x].mptr) { /* bad cursor? */ + mc->mc_flags &= ~(C_INITIALIZED|C_EOF); + txn->mt_flags |= MDB_TXN_ERROR; + return MDB_CORRUPTED; + } + return 0; + } + } + mdb_cassert(mc, dl[0].mid < MDB_IDL_UM_MAX); + /* No - copy it */ + np = mdb_page_malloc(txn, 1); + if (!np) + return ENOMEM; + mid.mid = pgno; + mid.mptr = np; + rc = mdb_mid2l_insert(dl, &mid); + mdb_cassert(mc, rc == 0); + } else { + return 0; + } + + mdb_page_copy(np, mp, txn->mt_env->me_psize); + np->mp_pgno = pgno; + np->mp_flags |= P_DIRTY; + +done: + /* Adjust cursors pointing to mp */ + mc->mc_pg[mc->mc_top] = np; + m2 = txn->mt_cursors[mc->mc_dbi]; + if (mc->mc_flags & C_SUB) { + for (; m2; m2=m2->mc_next) { + m3 = &m2->mc_xcursor->mx_cursor; + if (m3->mc_snum < mc->mc_snum) continue; + if (m3->mc_pg[mc->mc_top] == mp) + m3->mc_pg[mc->mc_top] = np; + } + } else { + for (; m2; m2=m2->mc_next) { + if (m2->mc_snum < mc->mc_snum) continue; + if (m2->mc_pg[mc->mc_top] == mp) { + m2->mc_pg[mc->mc_top] = np; + if ((mc->mc_db->md_flags & MDB_DUPSORT) && + IS_LEAF(np) && + m2->mc_ki[mc->mc_top] == mc->mc_ki[mc->mc_top]) + { + MDB_node *leaf = NODEPTR(np, mc->mc_ki[mc->mc_top]); + if (!(leaf->mn_flags & F_SUBDATA)) + m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); + } + } + } + } +#ifdef VL32 + { + MDB_ID2L rl = mc->mc_txn->mt_rpages; + unsigned x = mdb_mid2l_search(rl, mp->mp_pgno); + if (x <= rl[0].mid && rl[x].mid == mp->mp_pgno) { + munmap(mp, mc->mc_txn->mt_env->me_psize); + while (x < rl[0].mid) { + rl[x] = rl[x+1]; + x++; + } + rl[0].mid--; + } + } +#endif + return 0; + +fail: + txn->mt_flags |= MDB_TXN_ERROR; + return rc; +} + +int +mdb_env_sync(MDB_env *env, int force) +{ + int rc = 0; + if (force || !F_ISSET(env->me_flags, MDB_NOSYNC)) { + if (env->me_flags & MDB_WRITEMAP) { + int flags = ((env->me_flags & MDB_MAPASYNC) && !force) + ? MS_ASYNC : MS_SYNC; + if (MDB_MSYNC(env->me_map, env->me_mapsize, flags)) + rc = ErrCode(); +#ifdef _WIN32 + else if (flags == MS_SYNC && MDB_FDATASYNC(env->me_fd)) + rc = ErrCode(); +#endif + } else { + if (MDB_FDATASYNC(env->me_fd)) + rc = ErrCode(); + } + } + return rc; +} + +/** Back up parent txn's cursors, then grab the originals for tracking */ +static int +mdb_cursor_shadow(MDB_txn *src, MDB_txn *dst) +{ + MDB_cursor *mc, *bk; + MDB_xcursor *mx; + size_t size; + int i; + + for (i = src->mt_numdbs; --i >= 0; ) { + if ((mc = src->mt_cursors[i]) != NULL) { + size = sizeof(MDB_cursor); + if (mc->mc_xcursor) + size += sizeof(MDB_xcursor); + for (; mc; mc = bk->mc_next) { + bk = malloc(size); + if (!bk) + return ENOMEM; + *bk = *mc; + mc->mc_backup = bk; + mc->mc_db = &dst->mt_dbs[i]; + /* Kill pointers into src - and dst to reduce abuse: The + * user may not use mc until dst ends. Otherwise we'd... + */ + mc->mc_txn = NULL; /* ...set this to dst */ + mc->mc_dbflag = NULL; /* ...and &dst->mt_dbflags[i] */ + if ((mx = mc->mc_xcursor) != NULL) { + *(MDB_xcursor *)(bk+1) = *mx; + mx->mx_cursor.mc_txn = NULL; /* ...and dst. */ + } + mc->mc_next = dst->mt_cursors[i]; + dst->mt_cursors[i] = mc; + } + } + } + return MDB_SUCCESS; +} + +/** Close this write txn's cursors, give parent txn's cursors back to parent. + * @param[in] txn the transaction handle. + * @param[in] merge true to keep changes to parent cursors, false to revert. + * @return 0 on success, non-zero on failure. + */ +static void +mdb_cursors_close(MDB_txn *txn, unsigned merge) +{ + MDB_cursor **cursors = txn->mt_cursors, *mc, *next, *bk; + MDB_xcursor *mx; + int i; + + for (i = txn->mt_numdbs; --i >= 0; ) { + for (mc = cursors[i]; mc; mc = next) { + next = mc->mc_next; + if ((bk = mc->mc_backup) != NULL) { + if (merge) { + /* Commit changes to parent txn */ + mc->mc_next = bk->mc_next; + mc->mc_backup = bk->mc_backup; + mc->mc_txn = bk->mc_txn; + mc->mc_db = bk->mc_db; + mc->mc_dbflag = bk->mc_dbflag; + if ((mx = mc->mc_xcursor) != NULL) + mx->mx_cursor.mc_txn = bk->mc_txn; + } else { + /* Abort nested txn */ + *mc = *bk; + if ((mx = mc->mc_xcursor) != NULL) + *mx = *(MDB_xcursor *)(bk+1); + } + mc = bk; + } + /* Only malloced cursors are permanently tracked. */ + free(mc); + } + cursors[i] = NULL; + } +} + +#if !(MDB_DEBUG) +#define mdb_txn_reset0(txn, act) mdb_txn_reset0(txn) +#endif +static void +mdb_txn_reset0(MDB_txn *txn, const char *act); + +#if !(MDB_PIDLOCK) /* Currently the same as defined(_WIN32) */ +enum Pidlock_op { + Pidset, Pidcheck +}; +#else +enum Pidlock_op { + Pidset = F_SETLK, Pidcheck = F_GETLK +}; +#endif + +/** Set or check a pid lock. Set returns 0 on success. + * Check returns 0 if the process is certainly dead, nonzero if it may + * be alive (the lock exists or an error happened so we do not know). + * + * On Windows Pidset is a no-op, we merely check for the existence + * of the process with the given pid. On POSIX we use a single byte + * lock on the lockfile, set at an offset equal to the pid. + */ +static int +mdb_reader_pid(MDB_env *env, enum Pidlock_op op, MDB_PID_T pid) +{ +#if !(MDB_PIDLOCK) /* Currently the same as defined(_WIN32) */ + int ret = 0; + HANDLE h; + if (op == Pidcheck) { + h = OpenProcess(env->me_pidquery, FALSE, pid); + /* No documented "no such process" code, but other program use this: */ + if (!h) + return ErrCode() != ERROR_INVALID_PARAMETER; + /* A process exists until all handles to it close. Has it exited? */ + ret = WaitForSingleObject(h, 0) != 0; + CloseHandle(h); + } + return ret; +#else + for (;;) { + int rc; + struct flock lock_info; + memset(&lock_info, 0, sizeof(lock_info)); + lock_info.l_type = F_WRLCK; + lock_info.l_whence = SEEK_SET; + lock_info.l_start = pid; + lock_info.l_len = 1; + if ((rc = fcntl(env->me_lfd, op, &lock_info)) == 0) { + if (op == F_GETLK && lock_info.l_type != F_UNLCK) + rc = -1; + } else if ((rc = ErrCode()) == EINTR) { + continue; + } + return rc; + } +#endif +} + +/** Common code for #mdb_txn_begin() and #mdb_txn_renew(). + * @param[in] txn the transaction handle to initialize + * @return 0 on success, non-zero on failure. + */ +static int +mdb_txn_renew0(MDB_txn *txn) +{ + MDB_env *env = txn->mt_env; + MDB_txninfo *ti = env->me_txns; + MDB_meta *meta; + unsigned int i, nr; + uint16_t x; + int rc, new_notls = 0; + + /* Setup db info */ + txn->mt_numdbs = env->me_numdbs; + txn->mt_dbxs = env->me_dbxs; /* mostly static anyway */ + + if (txn->mt_flags & MDB_TXN_RDONLY) { + if (!ti) { + meta = env->me_metas[ mdb_env_pick_meta(env) ]; + txn->mt_txnid = meta->mm_txnid; + txn->mt_u.reader = NULL; + } else { + MDB_reader *r = (env->me_flags & MDB_NOTLS) ? txn->mt_u.reader : + pthread_getspecific(env->me_txkey); + if (r) { + if (r->mr_pid != env->me_pid || r->mr_txnid != (txnid_t)-1) + return MDB_BAD_RSLOT; + } else { + MDB_PID_T pid = env->me_pid; + MDB_THR_T tid = pthread_self(); + + if (!env->me_live_reader) { + rc = mdb_reader_pid(env, Pidset, pid); + if (rc) + return rc; + env->me_live_reader = 1; + } + + LOCK_MUTEX_R(env); + nr = ti->mti_numreaders; + for (i=0; i<nr; i++) + if (ti->mti_readers[i].mr_pid == 0) + break; + if (i == env->me_maxreaders) { + UNLOCK_MUTEX_R(env); + return MDB_READERS_FULL; + } + ti->mti_readers[i].mr_pid = pid; + ti->mti_readers[i].mr_tid = tid; + if (i == nr) + ti->mti_numreaders = ++nr; + /* Save numreaders for un-mutexed mdb_env_close() */ + env->me_numreaders = nr; + UNLOCK_MUTEX_R(env); + + r = &ti->mti_readers[i]; + new_notls = (env->me_flags & MDB_NOTLS); + if (!new_notls && (rc=pthread_setspecific(env->me_txkey, r))) { + r->mr_pid = 0; + return rc; + } + } + txn->mt_txnid = r->mr_txnid = ti->mti_txnid; + txn->mt_u.reader = r; + meta = env->me_metas[txn->mt_txnid & 1]; + } + } else { + if (ti) { + LOCK_MUTEX_W(env); + + txn->mt_txnid = ti->mti_txnid; + meta = env->me_metas[txn->mt_txnid & 1]; + } else { + meta = env->me_metas[ mdb_env_pick_meta(env) ]; + txn->mt_txnid = meta->mm_txnid; + } + txn->mt_txnid++; +#if MDB_DEBUG + if (txn->mt_txnid == mdb_debug_start) + mdb_debug = 1; +#endif + txn->mt_dirty_room = MDB_IDL_UM_MAX; + txn->mt_u.dirty_list = env->me_dirty_list; + txn->mt_u.dirty_list[0].mid = 0; + txn->mt_free_pgs = env->me_free_pgs; + txn->mt_free_pgs[0] = 0; + txn->mt_spill_pgs = NULL; + env->me_txn = txn; + memcpy(txn->mt_dbiseqs, env->me_dbiseqs, env->me_maxdbs * sizeof(unsigned int)); + } + + /* Copy the DB info and flags */ + memcpy(txn->mt_dbs, meta->mm_dbs, 2 * sizeof(MDB_db)); + + /* Moved to here to avoid a data race in read TXNs */ + txn->mt_next_pgno = meta->mm_last_pg+1; + + for (i=2; i<txn->mt_numdbs; i++) { + x = env->me_dbflags[i]; + txn->mt_dbs[i].md_flags = x & PERSISTENT_FLAGS; + txn->mt_dbflags[i] = (x & MDB_VALID) ? DB_VALID|DB_STALE : 0; + } + txn->mt_dbflags[0] = txn->mt_dbflags[1] = DB_VALID; + + if (env->me_maxpg < txn->mt_next_pgno) { + mdb_txn_reset0(txn, "renew0-mapfail"); + if (new_notls) { + txn->mt_u.reader->mr_pid = 0; + txn->mt_u.reader = NULL; + } + return MDB_MAP_RESIZED; + } + + return MDB_SUCCESS; +} + +int +mdb_txn_renew(MDB_txn *txn) +{ + int rc; + + if (!txn || txn->mt_dbxs) /* A reset txn has mt_dbxs==NULL */ + return EINVAL; + + if (txn->mt_env->me_flags & MDB_FATAL_ERROR) { + DPUTS("environment had fatal error, must shutdown!"); + return MDB_PANIC; + } + + rc = mdb_txn_renew0(txn); + if (rc == MDB_SUCCESS) { + DPRINTF(("renew txn %"Z"u%c %p on mdbenv %p, root page %"Z"u", + txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w', + (void *)txn, (void *)txn->mt_env, txn->mt_dbs[MAIN_DBI].md_root)); + } + return rc; +} + +int +mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret) +{ + MDB_txn *txn; + MDB_ntxn *ntxn; + int rc, size, tsize = sizeof(MDB_txn); + + if (env->me_flags & MDB_FATAL_ERROR) { + DPUTS("environment had fatal error, must shutdown!"); + return MDB_PANIC; + } + if ((env->me_flags & MDB_RDONLY) && !(flags & MDB_RDONLY)) + return EACCES; + if (parent) { + /* Nested transactions: Max 1 child, write txns only, no writemap */ + if (parent->mt_child || + (flags & MDB_RDONLY) || + (parent->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_ERROR)) || + (env->me_flags & MDB_WRITEMAP)) + { + return (parent->mt_flags & MDB_TXN_RDONLY) ? EINVAL : MDB_BAD_TXN; + } + tsize = sizeof(MDB_ntxn); + } + size = tsize + env->me_maxdbs * (sizeof(MDB_db)+1); + if (!(flags & MDB_RDONLY)) { + if (!parent) { + txn = env->me_txn0; + txn->mt_flags = 0; + goto ok; + } + size += env->me_maxdbs * sizeof(MDB_cursor *); + /* child txns use parent's dbiseqs */ + if (!parent) + size += env->me_maxdbs * sizeof(unsigned int); + } + + if ((txn = calloc(1, size)) == NULL) { + DPRINTF(("calloc: %s", strerror(errno))); + return ENOMEM; + } +#ifdef VL32 + if (!parent) { + txn->mt_rpages = calloc(MDB_IDL_UM_SIZE, sizeof(MDB_ID2)); + if (!txn->mt_rpages) { + free(txn); + return ENOMEM; + } + } +#endif + txn->mt_dbs = (MDB_db *) ((char *)txn + tsize); + if (flags & MDB_RDONLY) { + txn->mt_flags |= MDB_TXN_RDONLY; + txn->mt_dbflags = (unsigned char *)(txn->mt_dbs + env->me_maxdbs); + txn->mt_dbiseqs = env->me_dbiseqs; + } else { + txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs); + if (parent) { + txn->mt_dbiseqs = parent->mt_dbiseqs; + txn->mt_dbflags = (unsigned char *)(txn->mt_cursors + env->me_maxdbs); + } else { + txn->mt_dbiseqs = (unsigned int *)(txn->mt_cursors + env->me_maxdbs); + txn->mt_dbflags = (unsigned char *)(txn->mt_dbiseqs + env->me_maxdbs); + } + } + txn->mt_env = env; + +ok: + if (parent) { + unsigned int i; + txn->mt_u.dirty_list = malloc(sizeof(MDB_ID2)*MDB_IDL_UM_SIZE); + if (!txn->mt_u.dirty_list || + !(txn->mt_free_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX))) + { + free(txn->mt_u.dirty_list); + free(txn); + return ENOMEM; + } + txn->mt_txnid = parent->mt_txnid; + txn->mt_dirty_room = parent->mt_dirty_room; + txn->mt_u.dirty_list[0].mid = 0; + txn->mt_spill_pgs = NULL; + txn->mt_next_pgno = parent->mt_next_pgno; + parent->mt_child = txn; + txn->mt_parent = parent; + txn->mt_numdbs = parent->mt_numdbs; + txn->mt_flags = parent->mt_flags; + txn->mt_dbxs = parent->mt_dbxs; +#ifdef VL32 + txn->mt_rpages = parent->mt_rpages; +#endif + memcpy(txn->mt_dbs, parent->mt_dbs, txn->mt_numdbs * sizeof(MDB_db)); + /* Copy parent's mt_dbflags, but clear DB_NEW */ + for (i=0; i<txn->mt_numdbs; i++) + txn->mt_dbflags[i] = parent->mt_dbflags[i] & ~DB_NEW; + rc = 0; + ntxn = (MDB_ntxn *)txn; + ntxn->mnt_pgstate = env->me_pgstate; /* save parent me_pghead & co */ + if (env->me_pghead) { + size = MDB_IDL_SIZEOF(env->me_pghead); + env->me_pghead = mdb_midl_alloc(env->me_pghead[0]); + if (env->me_pghead) + memcpy(env->me_pghead, ntxn->mnt_pgstate.mf_pghead, size); + else + rc = ENOMEM; + } + if (!rc) + rc = mdb_cursor_shadow(parent, txn); + if (rc) + mdb_txn_reset0(txn, "beginchild-fail"); + } else { + rc = mdb_txn_renew0(txn); + } + if (rc) { + if (txn != env->me_txn0) + free(txn); + } else { + *ret = txn; + DPRINTF(("begin txn %"Z"u%c %p on mdbenv %p, root page %"Z"u", + txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w', + (void *) txn, (void *) env, txn->mt_dbs[MAIN_DBI].md_root)); + } + + return rc; +} + +MDB_env * +mdb_txn_env(MDB_txn *txn) +{ + if(!txn) return NULL; + return txn->mt_env; +} + +/** Export or close DBI handles opened in this txn. */ +static void +mdb_dbis_update(MDB_txn *txn, int keep) +{ + int i; + MDB_dbi n = txn->mt_numdbs; + MDB_env *env = txn->mt_env; + unsigned char *tdbflags = txn->mt_dbflags; + + for (i = n; --i >= 2;) { + if (tdbflags[i] & DB_NEW) { + if (keep) { + env->me_dbflags[i] = txn->mt_dbs[i].md_flags | MDB_VALID; + } else { + char *ptr = env->me_dbxs[i].md_name.mv_data; + if (ptr) { + env->me_dbxs[i].md_name.mv_data = NULL; + env->me_dbxs[i].md_name.mv_size = 0; + env->me_dbflags[i] = 0; + env->me_dbiseqs[i]++; + free(ptr); + } + } + } + } + if (keep && env->me_numdbs < n) + env->me_numdbs = n; +} + +/** Common code for #mdb_txn_reset() and #mdb_txn_abort(). + * May be called twice for readonly txns: First reset it, then abort. + * @param[in] txn the transaction handle to reset + * @param[in] act why the transaction is being reset + */ +static void +mdb_txn_reset0(MDB_txn *txn, const char *act) +{ + MDB_env *env = txn->mt_env; + + /* Close any DBI handles opened in this txn */ + mdb_dbis_update(txn, 0); + + DPRINTF(("%s txn %"Z"u%c %p on mdbenv %p, root page %"Z"u", + act, txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w', + (void *) txn, (void *)env, txn->mt_dbs[MAIN_DBI].md_root)); + + if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) { + if (txn->mt_u.reader) { + txn->mt_u.reader->mr_txnid = (txnid_t)-1; + if (!(env->me_flags & MDB_NOTLS)) + txn->mt_u.reader = NULL; /* txn does not own reader */ + } + txn->mt_numdbs = 0; /* close nothing if called again */ + txn->mt_dbxs = NULL; /* mark txn as reset */ + } else { + mdb_cursors_close(txn, 0); + + if (!(env->me_flags & MDB_WRITEMAP)) { + mdb_dlist_free(txn); + } + mdb_midl_free(env->me_pghead); + + if (txn->mt_parent) { + txn->mt_parent->mt_child = NULL; + env->me_pgstate = ((MDB_ntxn *)txn)->mnt_pgstate; + mdb_midl_free(txn->mt_free_pgs); + mdb_midl_free(txn->mt_spill_pgs); + free(txn->mt_u.dirty_list); + return; + } + + if (mdb_midl_shrink(&txn->mt_free_pgs)) + env->me_free_pgs = txn->mt_free_pgs; + env->me_pghead = NULL; + env->me_pglast = 0; + + env->me_txn = NULL; + /* The writer mutex was locked in mdb_txn_begin. */ + if (env->me_txns) + UNLOCK_MUTEX_W(env); + } +#ifdef VL32 + { + unsigned i, n = txn->mt_rpages[0].mid; + for (i = 1; i <= n; i++) { +#ifdef _WIN32 + UnmapViewOfFile(txn->mt_rpages[i].mptr);) +#else + MDB_page *mp = txn->mt_rpages[i].mptr; + int size = txn->mt_env->me_psize; + if (IS_OVERFLOW(mp)) size *= mp->mp_pages; + munmap(mp, size); +#endif + } + } + txn->mt_rpages[0].mid = 0; +#endif +} + +void +mdb_txn_reset(MDB_txn *txn) +{ + if (txn == NULL) + return; + + /* This call is only valid for read-only txns */ + if (!(txn->mt_flags & MDB_TXN_RDONLY)) + return; + + mdb_txn_reset0(txn, "reset"); +} + +void +mdb_txn_abort(MDB_txn *txn) +{ + if (txn == NULL) + return; + + if (txn->mt_child) + mdb_txn_abort(txn->mt_child); + + mdb_txn_reset0(txn, "abort"); + /* Free reader slot tied to this txn (if MDB_NOTLS && writable FS) */ + if ((txn->mt_flags & MDB_TXN_RDONLY) && txn->mt_u.reader) + txn->mt_u.reader->mr_pid = 0; + + if (txn != txn->mt_env->me_txn0) { +#ifdef VL32 + free(txn->mt_rpages); +#endif + free(txn); + } +} + +/** Save the freelist as of this transaction to the freeDB. + * This changes the freelist. Keep trying until it stabilizes. + */ +static int +mdb_freelist_save(MDB_txn *txn) +{ + /* env->me_pghead[] can grow and shrink during this call. + * env->me_pglast and txn->mt_free_pgs[] can only grow. + * Page numbers cannot disappear from txn->mt_free_pgs[]. + */ + MDB_cursor mc; + MDB_env *env = txn->mt_env; + int rc, maxfree_1pg = env->me_maxfree_1pg, more = 1; + txnid_t pglast = 0, head_id = 0; + pgno_t freecnt = 0, *free_pgs, *mop; + ssize_t head_room = 0, total_room = 0, mop_len, clean_limit; + + mdb_cursor_init(&mc, txn, FREE_DBI, NULL); + + if (env->me_pghead) { + /* Make sure first page of freeDB is touched and on freelist */ + rc = mdb_page_search(&mc, NULL, MDB_PS_FIRST|MDB_PS_MODIFY); + if (rc && rc != MDB_NOTFOUND) + return rc; + } + + if (!env->me_pghead && txn->mt_loose_pgs) { + /* Put loose page numbers in mt_free_pgs, since + * we may be unable to return them to me_pghead. + */ + MDB_page *mp = txn->mt_loose_pgs; + if ((rc = mdb_midl_need(&txn->mt_free_pgs, txn->mt_loose_count)) != 0) + return rc; + for (; mp; mp = NEXT_LOOSE_PAGE(mp)) + mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno); + txn->mt_loose_pgs = NULL; + txn->mt_loose_count = 0; + } + + /* MDB_RESERVE cancels meminit in ovpage malloc (when no WRITEMAP) */ + clean_limit = (env->me_flags & (MDB_NOMEMINIT|MDB_WRITEMAP)) + ? SSIZE_MAX : maxfree_1pg; + + for (;;) { + /* Come back here after each Put() in case freelist changed */ + MDB_val key, data; + pgno_t *pgs; + ssize_t j; + + /* If using records from freeDB which we have not yet + * deleted, delete them and any we reserved for me_pghead. + */ + while (pglast < env->me_pglast) { + rc = mdb_cursor_first(&mc, &key, NULL); + if (rc) + return rc; + pglast = head_id = *(txnid_t *)key.mv_data; + total_room = head_room = 0; + mdb_tassert(txn, pglast <= env->me_pglast); + rc = mdb_cursor_del(&mc, 0); + if (rc) + return rc; + } + + /* Save the IDL of pages freed by this txn, to a single record */ + if (freecnt < txn->mt_free_pgs[0]) { + if (!freecnt) { + /* Make sure last page of freeDB is touched and on freelist */ + rc = mdb_page_search(&mc, NULL, MDB_PS_LAST|MDB_PS_MODIFY); + if (rc && rc != MDB_NOTFOUND) + return rc; + } + free_pgs = txn->mt_free_pgs; + /* Write to last page of freeDB */ + key.mv_size = sizeof(txn->mt_txnid); + key.mv_data = &txn->mt_txnid; + do { + freecnt = free_pgs[0]; + data.mv_size = MDB_IDL_SIZEOF(free_pgs); + rc = mdb_cursor_put(&mc, &key, &data, MDB_RESERVE); + if (rc) + return rc; + /* Retry if mt_free_pgs[] grew during the Put() */ + free_pgs = txn->mt_free_pgs; + } while (freecnt < free_pgs[0]); + mdb_midl_sort(free_pgs); + memcpy(data.mv_data, free_pgs, data.mv_size); +#if (MDB_DEBUG) > 1 + { + unsigned int i = free_pgs[0]; + DPRINTF(("IDL write txn %"Z"u root %"Z"u num %u", + txn->mt_txnid, txn->mt_dbs[FREE_DBI].md_root, i)); + for (; i; i--) + DPRINTF(("IDL %"Z"u", free_pgs[i])); + } +#endif + continue; + } + + mop = env->me_pghead; + mop_len = (mop ? mop[0] : 0) + txn->mt_loose_count; + + /* Reserve records for me_pghead[]. Split it if multi-page, + * to avoid searching freeDB for a page range. Use keys in + * range [1,me_pglast]: Smaller than txnid of oldest reader. + */ + if (total_room >= mop_len) { + if (total_room == mop_len || --more < 0) + break; + } else if (head_room >= maxfree_1pg && head_id > 1) { + /* Keep current record (overflow page), add a new one */ + head_id--; + head_room = 0; + } + /* (Re)write {key = head_id, IDL length = head_room} */ + total_room -= head_room; + head_room = mop_len - total_room; + if (head_room > maxfree_1pg && head_id > 1) { + /* Overflow multi-page for part of me_pghead */ + head_room /= head_id; /* amortize page sizes */ + head_room += maxfree_1pg - head_room % (maxfree_1pg + 1); + } else if (head_room < 0) { + /* Rare case, not bothering to delete this record */ + head_room = 0; + } + key.mv_size = sizeof(head_id); + key.mv_data = &head_id; + data.mv_size = (head_room + 1) * sizeof(pgno_t); + rc = mdb_cursor_put(&mc, &key, &data, MDB_RESERVE); + if (rc) + return rc; + /* IDL is initially empty, zero out at least the length */ + pgs = (pgno_t *)data.mv_data; + j = head_room > clean_limit ? head_room : 0; + do { + pgs[j] = 0; + } while (--j >= 0); + total_room += head_room; + } + + /* Return loose page numbers to me_pghead, though usually none are + * left at this point. The pages themselves remain in dirty_list. + */ + if (txn->mt_loose_pgs) { + MDB_page *mp = txn->mt_loose_pgs; + unsigned count = txn->mt_loose_count; + MDB_IDL loose; + /* Room for loose pages + temp IDL with same */ + if ((rc = mdb_midl_need(&env->me_pghead, 2*count+1)) != 0) + return rc; + mop = env->me_pghead; + loose = mop + MDB_IDL_ALLOCLEN(mop) - count; + for (count = 0; mp; mp = NEXT_LOOSE_PAGE(mp)) + loose[ ++count ] = mp->mp_pgno; + loose[0] = count; + mdb_midl_sort(loose); + mdb_midl_xmerge(mop, loose); + txn->mt_loose_pgs = NULL; + txn->mt_loose_count = 0; + mop_len = mop[0]; + } + + /* Fill in the reserved me_pghead records */ + rc = MDB_SUCCESS; + if (mop_len) { + MDB_val key, data; + + mop += mop_len; + rc = mdb_cursor_first(&mc, &key, &data); + for (; !rc; rc = mdb_cursor_next(&mc, &key, &data, MDB_NEXT)) { + txnid_t id = *(txnid_t *)key.mv_data; + ssize_t len = (ssize_t)(data.mv_size / sizeof(MDB_ID)) - 1; + MDB_ID save; + + mdb_tassert(txn, len >= 0 && id <= env->me_pglast); + key.mv_data = &id; + if (len > mop_len) { + len = mop_len; + data.mv_size = (len + 1) * sizeof(MDB_ID); + } + data.mv_data = mop -= len; + save = mop[0]; + mop[0] = len; + rc = mdb_cursor_put(&mc, &key, &data, MDB_CURRENT); + mop[0] = save; + if (rc || !(mop_len -= len)) + break; + } + } + return rc; +} + +/** Flush (some) dirty pages to the map, after clearing their dirty flag. + * @param[in] txn the transaction that's being committed + * @param[in] keep number of initial pages in dirty_list to keep dirty. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_flush(MDB_txn *txn, int keep) +{ + MDB_env *env = txn->mt_env; + MDB_ID2L dl = txn->mt_u.dirty_list; + unsigned psize = env->me_psize, j; + int i, pagecount = dl[0].mid, rc; + size_t size = 0, pos = 0; + pgno_t pgno = 0; + MDB_page *dp = NULL; +#ifdef _WIN32 + OVERLAPPED ov; +#else + struct iovec iov[MDB_COMMIT_PAGES]; + ssize_t wpos = 0, wsize = 0, wres; + size_t next_pos = 1; /* impossible pos, so pos != next_pos */ + int n = 0; +#endif + + j = i = keep; + + if (env->me_flags & MDB_WRITEMAP) { + /* Clear dirty flags */ + while (++i <= pagecount) { + dp = dl[i].mptr; + /* Don't flush this page yet */ + if (dp->mp_flags & (P_LOOSE|P_KEEP)) { + dp->mp_flags &= ~P_KEEP; + dl[++j] = dl[i]; + continue; + } + dp->mp_flags &= ~P_DIRTY; + } + goto done; + } + + /* Write the pages */ + for (;;) { + if (++i <= pagecount) { + dp = dl[i].mptr; + /* Don't flush this page yet */ + if (dp->mp_flags & (P_LOOSE|P_KEEP)) { + dp->mp_flags &= ~P_KEEP; + dl[i].mid = 0; + continue; + } + pgno = dl[i].mid; + /* clear dirty flag */ + dp->mp_flags &= ~P_DIRTY; + pos = pgno * psize; + size = psize; + if (IS_OVERFLOW(dp)) size *= dp->mp_pages; + } +#ifdef _WIN32 + else break; + + /* Windows actually supports scatter/gather I/O, but only on + * unbuffered file handles. Since we're relying on the OS page + * cache for all our data, that's self-defeating. So we just + * write pages one at a time. We use the ov structure to set + * the write offset, to at least save the overhead of a Seek + * system call. + */ + DPRINTF(("committing page %"Z"u", pgno)); + memset(&ov, 0, sizeof(ov)); + ov.Offset = pos & 0xffffffff; + ov.OffsetHigh = pos >> 16 >> 16; + if (!WriteFile(env->me_fd, dp, size, NULL, &ov)) { + rc = ErrCode(); + DPRINTF(("WriteFile: %d", rc)); + return rc; + } +#else + /* Write up to MDB_COMMIT_PAGES dirty pages at a time. */ + if (pos!=next_pos || n==MDB_COMMIT_PAGES || wsize+size>MAX_WRITE) { + if (n) { + /* Write previous page(s) */ +#ifdef MDB_USE_PWRITEV + wres = pwritev(env->me_fd, iov, n, wpos); +#else + if (n == 1) { + wres = pwrite(env->me_fd, iov[0].iov_base, wsize, wpos); + } else { + if (lseek(env->me_fd, wpos, SEEK_SET) == -1) { + rc = ErrCode(); + DPRINTF(("lseek: %s", strerror(rc))); + return rc; + } + wres = writev(env->me_fd, iov, n); + } +#endif + if (wres != wsize) { + if (wres < 0) { + rc = ErrCode(); + DPRINTF(("Write error: %s", strerror(rc))); + } else { + rc = EIO; /* TODO: Use which error code? */ + DPUTS("short write, filesystem full?"); + } + return rc; + } + n = 0; + } + if (i > pagecount) + break; + wpos = pos; + wsize = 0; + } + DPRINTF(("committing page %"Z"u", pgno)); + next_pos = pos + size; + iov[n].iov_len = size; + iov[n].iov_base = (char *)dp; + wsize += size; + n++; +#endif /* _WIN32 */ + } + + /* MIPS has cache coherency issues, this is a no-op everywhere else + * Note: for any size >= on-chip cache size, entire on-chip cache is + * flushed. + */ + CACHEFLUSH(env->me_map, txn->mt_next_pgno * env->me_psize, DCACHE); + + for (i = keep; ++i <= pagecount; ) { + dp = dl[i].mptr; + /* This is a page we skipped above */ + if (!dl[i].mid) { + dl[++j] = dl[i]; + dl[j].mid = dp->mp_pgno; + continue; + } + mdb_dpage_free(env, dp); + } + +done: + i--; + txn->mt_dirty_room += i - j; + dl[0].mid = j; + return MDB_SUCCESS; +} + +int +mdb_txn_commit(MDB_txn *txn) +{ + int rc; + unsigned int i; + MDB_env *env; + + if (txn == NULL || txn->mt_env == NULL) + return EINVAL; + + if (txn->mt_child) { + rc = mdb_txn_commit(txn->mt_child); + txn->mt_child = NULL; + if (rc) + goto fail; + } + + env = txn->mt_env; + + if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) { + mdb_dbis_update(txn, 1); + txn->mt_numdbs = 2; /* so txn_abort() doesn't close any new handles */ + mdb_txn_abort(txn); + return MDB_SUCCESS; + } + + if (F_ISSET(txn->mt_flags, MDB_TXN_ERROR)) { + DPUTS("error flag is set, can't commit"); + if (txn->mt_parent) + txn->mt_parent->mt_flags |= MDB_TXN_ERROR; + rc = MDB_BAD_TXN; + goto fail; + } + + if (txn->mt_parent) { + MDB_txn *parent = txn->mt_parent; + MDB_page **lp; + MDB_ID2L dst, src; + MDB_IDL pspill; + unsigned x, y, len, ps_len; + + /* Append our free list to parent's */ + rc = mdb_midl_append_list(&parent->mt_free_pgs, txn->mt_free_pgs); + if (rc) + goto fail; + mdb_midl_free(txn->mt_free_pgs); + /* Failures after this must either undo the changes + * to the parent or set MDB_TXN_ERROR in the parent. + */ + + parent->mt_next_pgno = txn->mt_next_pgno; + parent->mt_flags = txn->mt_flags; + + /* Merge our cursors into parent's and close them */ + mdb_cursors_close(txn, 1); + + /* Update parent's DB table. */ + memcpy(parent->mt_dbs, txn->mt_dbs, txn->mt_numdbs * sizeof(MDB_db)); + parent->mt_numdbs = txn->mt_numdbs; + parent->mt_dbflags[0] = txn->mt_dbflags[0]; + parent->mt_dbflags[1] = txn->mt_dbflags[1]; + for (i=2; i<txn->mt_numdbs; i++) { + /* preserve parent's DB_NEW status */ + x = parent->mt_dbflags[i] & DB_NEW; + parent->mt_dbflags[i] = txn->mt_dbflags[i] | x; + } + + dst = parent->mt_u.dirty_list; + src = txn->mt_u.dirty_list; + /* Remove anything in our dirty list from parent's spill list */ + if ((pspill = parent->mt_spill_pgs) && (ps_len = pspill[0])) { + x = y = ps_len; + pspill[0] = (pgno_t)-1; + /* Mark our dirty pages as deleted in parent spill list */ + for (i=0, len=src[0].mid; ++i <= len; ) { + MDB_ID pn = src[i].mid << 1; + while (pn > pspill[x]) + x--; + if (pn == pspill[x]) { + pspill[x] = 1; + y = --x; + } + } + /* Squash deleted pagenums if we deleted any */ + for (x=y; ++x <= ps_len; ) + if (!(pspill[x] & 1)) + pspill[++y] = pspill[x]; + pspill[0] = y; + } + + /* Find len = length of merging our dirty list with parent's */ + x = dst[0].mid; + dst[0].mid = 0; /* simplify loops */ + if (parent->mt_parent) { + len = x + src[0].mid; + y = mdb_mid2l_search(src, dst[x].mid + 1) - 1; + for (i = x; y && i; y--) { + pgno_t yp = src[y].mid; + while (yp < dst[i].mid) + i--; + if (yp == dst[i].mid) { + i--; + len--; + } + } + } else { /* Simplify the above for single-ancestor case */ + len = MDB_IDL_UM_MAX - txn->mt_dirty_room; + } + /* Merge our dirty list with parent's */ + y = src[0].mid; + for (i = len; y; dst[i--] = src[y--]) { + pgno_t yp = src[y].mid; + while (yp < dst[x].mid) + dst[i--] = dst[x--]; + if (yp == dst[x].mid) + free(dst[x--].mptr); + } + mdb_tassert(txn, i == x); + dst[0].mid = len; + free(txn->mt_u.dirty_list); + parent->mt_dirty_room = txn->mt_dirty_room; + if (txn->mt_spill_pgs) { + if (parent->mt_spill_pgs) { + /* TODO: Prevent failure here, so parent does not fail */ + rc = mdb_midl_append_list(&parent->mt_spill_pgs, txn->mt_spill_pgs); + if (rc) + parent->mt_flags |= MDB_TXN_ERROR; + mdb_midl_free(txn->mt_spill_pgs); + mdb_midl_sort(parent->mt_spill_pgs); + } else { + parent->mt_spill_pgs = txn->mt_spill_pgs; + } + } + + /* Append our loose page list to parent's */ + for (lp = &parent->mt_loose_pgs; *lp; lp = &NEXT_LOOSE_PAGE(lp)) + ; + *lp = txn->mt_loose_pgs; + parent->mt_loose_count += txn->mt_loose_count; + + parent->mt_child = NULL; + mdb_midl_free(((MDB_ntxn *)txn)->mnt_pgstate.mf_pghead); + free(txn); + return rc; + } + + if (txn != env->me_txn) { + DPUTS("attempt to commit unknown transaction"); + rc = EINVAL; + goto fail; + } + + mdb_cursors_close(txn, 0); + + if (!txn->mt_u.dirty_list[0].mid && + !(txn->mt_flags & (MDB_TXN_DIRTY|MDB_TXN_SPILLS))) + goto done; + + DPRINTF(("committing txn %"Z"u %p on mdbenv %p, root page %"Z"u", + txn->mt_txnid, (void*)txn, (void*)env, txn->mt_dbs[MAIN_DBI].md_root)); + + /* Update DB root pointers */ + if (txn->mt_numdbs > 2) { + MDB_cursor mc; + MDB_dbi i; + MDB_val data; + data.mv_size = sizeof(MDB_db); + + mdb_cursor_init(&mc, txn, MAIN_DBI, NULL); + for (i = 2; i < txn->mt_numdbs; i++) { + if (txn->mt_dbflags[i] & DB_DIRTY) { + if (TXN_DBI_CHANGED(txn, i)) { + rc = MDB_BAD_DBI; + goto fail; + } + data.mv_data = &txn->mt_dbs[i]; + rc = mdb_cursor_put(&mc, &txn->mt_dbxs[i].md_name, &data, 0); + if (rc) + goto fail; + } + } + } + + rc = mdb_freelist_save(txn); + if (rc) + goto fail; + + mdb_midl_free(env->me_pghead); + env->me_pghead = NULL; + if (mdb_midl_shrink(&txn->mt_free_pgs)) + env->me_free_pgs = txn->mt_free_pgs; + +#if (MDB_DEBUG) > 2 + mdb_audit(txn); +#endif + + if ((rc = mdb_page_flush(txn, 0)) || + (rc = mdb_env_sync(env, 0)) || + (rc = mdb_env_write_meta(txn))) + goto fail; + + /* Free P_LOOSE pages left behind in dirty_list */ + if (!(env->me_flags & MDB_WRITEMAP)) + mdb_dlist_free(txn); + +done: + env->me_pglast = 0; + env->me_txn = NULL; + mdb_dbis_update(txn, 1); + + if (env->me_txns) + UNLOCK_MUTEX_W(env); + if (txn != env->me_txn0) + free(txn); + + return MDB_SUCCESS; + +fail: + mdb_txn_abort(txn); + return rc; +} + +/** Read the environment parameters of a DB environment before + * mapping it into memory. + * @param[in] env the environment handle + * @param[out] meta address of where to store the meta information + * @return 0 on success, non-zero on failure. + */ +static int ESECT +mdb_env_read_header(MDB_env *env, MDB_meta *meta) +{ + MDB_metabuf pbuf; + MDB_page *p; + MDB_meta *m; + int i, rc, off; + enum { Size = sizeof(pbuf) }; + + /* We don't know the page size yet, so use a minimum value. + * Read both meta pages so we can use the latest one. + */ + + for (i=off=0; i<2; i++, off = meta->mm_psize) { +#ifdef _WIN32 + DWORD len; + OVERLAPPED ov; + memset(&ov, 0, sizeof(ov)); + ov.Offset = off; + rc = ReadFile(env->me_fd, &pbuf, Size, &len, &ov) ? (int)len : -1; + if (rc == -1 && ErrCode() == ERROR_HANDLE_EOF) + rc = 0; +#else + rc = pread(env->me_fd, &pbuf, Size, off); +#endif + if (rc != Size) { + if (rc == 0 && off == 0) + return ENOENT; + rc = rc < 0 ? (int) ErrCode() : MDB_INVALID; + DPRINTF(("read: %s", mdb_strerror(rc))); + return rc; + } + + p = (MDB_page *)&pbuf; + + if (!F_ISSET(p->mp_flags, P_META)) { + DPRINTF(("page %"Z"u not a meta page", p->mp_pgno)); + return MDB_INVALID; + } + + m = METADATA(p); + if (m->mm_magic != MDB_MAGIC) { + DPUTS("meta has invalid magic"); + return MDB_INVALID; + } + + if (m->mm_version != MDB_DATA_VERSION) { + DPRINTF(("database is version %u, expected version %u", + m->mm_version, MDB_DATA_VERSION)); + return MDB_VERSION_MISMATCH; + } + + if (off == 0 || m->mm_txnid > meta->mm_txnid) + *meta = *m; + } + return 0; +} + +static void ESECT +mdb_env_init_meta0(MDB_env *env, MDB_meta *meta) +{ + meta->mm_magic = MDB_MAGIC; + meta->mm_version = MDB_DATA_VERSION; + meta->mm_mapsize = env->me_mapsize; + meta->mm_psize = env->me_psize; + meta->mm_last_pg = 1; + meta->mm_flags = env->me_flags & 0xffff; + meta->mm_flags |= MDB_INTEGERKEY; + meta->mm_dbs[0].md_root = P_INVALID; + meta->mm_dbs[1].md_root = P_INVALID; +} + +/** Write the environment parameters of a freshly created DB environment. + * @param[in] env the environment handle + * @param[out] meta address of where to store the meta information + * @return 0 on success, non-zero on failure. + */ +static int ESECT +mdb_env_init_meta(MDB_env *env, MDB_meta *meta) +{ + MDB_page *p, *q; + int rc; + unsigned int psize; +#ifdef _WIN32 + DWORD len; + OVERLAPPED ov; + memset(&ov, 0, sizeof(ov)); +#define DO_PWRITE(rc, fd, ptr, size, len, pos) do { \ + ov.Offset = pos; \ + rc = WriteFile(fd, ptr, size, &len, &ov); } while(0) +#else + int len; +#define DO_PWRITE(rc, fd, ptr, size, len, pos) do { \ + len = pwrite(fd, ptr, size, pos); \ + rc = (len >= 0); } while(0) +#endif + + DPUTS("writing new meta page"); + + psize = env->me_psize; + + mdb_env_init_meta0(env, meta); + + p = calloc(2, psize); + p->mp_pgno = 0; + p->mp_flags = P_META; + *(MDB_meta *)METADATA(p) = *meta; + + q = (MDB_page *)((char *)p + psize); + q->mp_pgno = 1; + q->mp_flags = P_META; + *(MDB_meta *)METADATA(q) = *meta; + + DO_PWRITE(rc, env->me_fd, p, psize * 2, len, 0); + if (!rc) + rc = ErrCode(); + else if ((unsigned) len == psize * 2) + rc = MDB_SUCCESS; + else + rc = ENOSPC; + free(p); + return rc; +} + +/** Update the environment info to commit a transaction. + * @param[in] txn the transaction that's being committed + * @return 0 on success, non-zero on failure. + */ +static int +mdb_env_write_meta(MDB_txn *txn) +{ + MDB_env *env; + MDB_meta meta, metab, *mp; + size_t mapsize; + off_t off; + int rc, len, toggle; + char *ptr; + HANDLE mfd; +#ifdef _WIN32 + OVERLAPPED ov; +#else + int r2; +#endif + + toggle = txn->mt_txnid & 1; + DPRINTF(("writing meta page %d for root page %"Z"u", + toggle, txn->mt_dbs[MAIN_DBI].md_root)); + + env = txn->mt_env; + mp = env->me_metas[toggle]; + mapsize = env->me_metas[toggle ^ 1]->mm_mapsize; + /* Persist any increases of mapsize config */ + if (mapsize < env->me_mapsize) + mapsize = env->me_mapsize; + + if (env->me_flags & MDB_WRITEMAP) { + mp->mm_mapsize = mapsize; + mp->mm_dbs[0] = txn->mt_dbs[0]; + mp->mm_dbs[1] = txn->mt_dbs[1]; + mp->mm_last_pg = txn->mt_next_pgno - 1; + mp->mm_txnid = txn->mt_txnid; + if (!(env->me_flags & (MDB_NOMETASYNC|MDB_NOSYNC))) { + unsigned meta_size = env->me_psize; + rc = (env->me_flags & MDB_MAPASYNC) ? MS_ASYNC : MS_SYNC; + ptr = env->me_map; + if (toggle) { +#ifndef _WIN32 /* POSIX msync() requires ptr = start of OS page */ + if (meta_size < env->me_os_psize) + meta_size += meta_size; + else +#endif + ptr += meta_size; + } + if (MDB_MSYNC(ptr, meta_size, rc)) { + rc = ErrCode(); + goto fail; + } + } + goto done; + } + metab.mm_txnid = env->me_metas[toggle]->mm_txnid; + metab.mm_last_pg = env->me_metas[toggle]->mm_last_pg; + + meta.mm_mapsize = mapsize; + meta.mm_dbs[0] = txn->mt_dbs[0]; + meta.mm_dbs[1] = txn->mt_dbs[1]; + meta.mm_last_pg = txn->mt_next_pgno - 1; + meta.mm_txnid = txn->mt_txnid; + + off = offsetof(MDB_meta, mm_mapsize); + ptr = (char *)&meta + off; + len = sizeof(MDB_meta) - off; + if (toggle) + off += env->me_psize; + off += PAGEHDRSZ; + + /* Write to the SYNC fd */ + mfd = env->me_flags & (MDB_NOSYNC|MDB_NOMETASYNC) ? + env->me_fd : env->me_mfd; +#ifdef _WIN32 + { + memset(&ov, 0, sizeof(ov)); + ov.Offset = off; + if (!WriteFile(mfd, ptr, len, (DWORD *)&rc, &ov)) + rc = -1; + } +#else + rc = pwrite(mfd, ptr, len, off); +#endif + if (rc != len) { + rc = rc < 0 ? ErrCode() : EIO; + DPUTS("write failed, disk error?"); + /* On a failure, the pagecache still contains the new data. + * Write some old data back, to prevent it from being used. + * Use the non-SYNC fd; we know it will fail anyway. + */ + meta.mm_last_pg = metab.mm_last_pg; + meta.mm_txnid = metab.mm_txnid; +#ifdef _WIN32 + memset(&ov, 0, sizeof(ov)); + ov.Offset = off; + WriteFile(env->me_fd, ptr, len, NULL, &ov); +#else + r2 = pwrite(env->me_fd, ptr, len, off); + (void)r2; /* Silence warnings. We don't care about pwrite's return value */ +#endif +fail: + env->me_flags |= MDB_FATAL_ERROR; + return rc; + } + /* MIPS has cache coherency issues, this is a no-op everywhere else */ + CACHEFLUSH(env->me_map + off, len, DCACHE); +done: + /* Memory ordering issues are irrelevant; since the entire writer + * is wrapped by wmutex, all of these changes will become visible + * after the wmutex is unlocked. Since the DB is multi-version, + * readers will get consistent data regardless of how fresh or + * how stale their view of these values is. + */ + if (env->me_txns) + env->me_txns->mti_txnid = txn->mt_txnid; + + return MDB_SUCCESS; +} + +/** Check both meta pages to see which one is newer. + * @param[in] env the environment handle + * @return meta toggle (0 or 1). + */ +static int +mdb_env_pick_meta(const MDB_env *env) +{ + return (env->me_metas[0]->mm_txnid < env->me_metas[1]->mm_txnid); +} + +int ESECT +mdb_env_create(MDB_env **env) +{ + MDB_env *e; + + e = calloc(1, sizeof(MDB_env)); + if (!e) + return ENOMEM; + + e->me_maxreaders = DEFAULT_READERS; + e->me_maxdbs = e->me_numdbs = 2; + e->me_fd = INVALID_HANDLE_VALUE; + e->me_lfd = INVALID_HANDLE_VALUE; + e->me_mfd = INVALID_HANDLE_VALUE; +#ifdef MDB_USE_POSIX_SEM + e->me_rmutex = SEM_FAILED; + e->me_wmutex = SEM_FAILED; +#endif + e->me_pid = getpid(); + GET_PAGESIZE(e->me_os_psize); + VGMEMP_CREATE(e,0,0); + *env = e; + return MDB_SUCCESS; +} + +static int ESECT +mdb_env_map(MDB_env *env, void *addr) +{ + MDB_page *p; + unsigned int flags = env->me_flags; +#ifdef _WIN32 + int rc; + HANDLE mh; + LONG sizelo, sizehi; + size_t msize; + + if (flags & MDB_RDONLY) { + /* Don't set explicit map size, use whatever exists */ + msize = 0; + sizelo = 0; + sizehi = 0; + } else { + msize = env->me_mapsize; + sizelo = msize & 0xffffffff; + sizehi = msize >> 16 >> 16; /* only needed on Win64 */ + + /* Windows won't create mappings for zero length files. + * and won't map more than the file size. + * Just set the maxsize right now. + */ + if (SetFilePointer(env->me_fd, sizelo, &sizehi, 0) != (DWORD)sizelo + || !SetEndOfFile(env->me_fd) + || SetFilePointer(env->me_fd, 0, NULL, 0) != 0) + return ErrCode(); + } + + mh = CreateFileMapping(env->me_fd, NULL, flags & MDB_WRITEMAP ? + PAGE_READWRITE : PAGE_READONLY, + sizehi, sizelo, NULL); + if (!mh) + return ErrCode(); +#ifdef VL32 + msize = 2 * env->me_psize; +#endif + env->me_map = MapViewOfFileEx(mh, flags & MDB_WRITEMAP ? + FILE_MAP_WRITE : FILE_MAP_READ, + 0, 0, msize, addr); + rc = env->me_map ? 0 : ErrCode(); +#ifdef VL32 + env->me_fmh = mh; +#else + CloseHandle(mh); +#endif + if (rc) + return rc; +#else +#ifdef VL32 + (void) flags; + env->me_map = mmap(addr, 2 * env->me_psize, PROT_READ, MAP_SHARED, + env->me_fd, 0); + if (env->me_map == MAP_FAILED) { + env->me_map = NULL; + return ErrCode(); + } +#else + int prot = PROT_READ; + if (flags & MDB_WRITEMAP) { + prot |= PROT_WRITE; + if (ftruncate(env->me_fd, env->me_mapsize) < 0) + return ErrCode(); + } + env->me_map = mmap(addr, env->me_mapsize, prot, MAP_SHARED, + env->me_fd, 0); + if (env->me_map == MAP_FAILED) { + env->me_map = NULL; + return ErrCode(); + } + + if (flags & MDB_NORDAHEAD) { + /* Turn off readahead. It's harmful when the DB is larger than RAM. */ +#ifdef MADV_RANDOM + madvise(env->me_map, env->me_mapsize, MADV_RANDOM); +#else +#ifdef POSIX_MADV_RANDOM + posix_madvise(env->me_map, env->me_mapsize, POSIX_MADV_RANDOM); +#endif /* POSIX_MADV_RANDOM */ +#endif /* MADV_RANDOM */ + } +#endif /* _WIN32 */ + + /* Can happen because the address argument to mmap() is just a + * hint. mmap() can pick another, e.g. if the range is in use. + * The MAP_FIXED flag would prevent that, but then mmap could + * instead unmap existing pages to make room for the new map. + */ + if (addr && env->me_map != addr) + return EBUSY; /* TODO: Make a new MDB_* error code? */ +#endif + + p = (MDB_page *)env->me_map; + env->me_metas[0] = METADATA(p); + env->me_metas[1] = (MDB_meta *)((char *)env->me_metas[0] + env->me_psize); + + return MDB_SUCCESS; +} + +int ESECT +mdb_env_set_mapsize(MDB_env *env, size_t size) +{ + /* If env is already open, caller is responsible for making + * sure there are no active txns. + */ + if (env->me_map) { + int rc; + void *old; + if (env->me_txn) + return EINVAL; + if (!size) + size = env->me_metas[mdb_env_pick_meta(env)]->mm_mapsize; + else if (size < env->me_mapsize) { + /* If the configured size is smaller, make sure it's + * still big enough. Silently round up to minimum if not. + */ + size_t minsize = (env->me_metas[mdb_env_pick_meta(env)]->mm_last_pg + 1) * env->me_psize; + if (size < minsize) + size = minsize; + } + munmap(env->me_map, env->me_mapsize); + env->me_mapsize = size; + old = (env->me_flags & MDB_FIXEDMAP) ? env->me_map : NULL; + rc = mdb_env_map(env, old); + if (rc) + return rc; + } + env->me_mapsize = size; + if (env->me_psize) + env->me_maxpg = env->me_mapsize / env->me_psize; + return MDB_SUCCESS; +} + +int ESECT +mdb_env_set_maxdbs(MDB_env *env, MDB_dbi dbs) +{ + if (env->me_map) + return EINVAL; + env->me_maxdbs = dbs + 2; /* Named databases + main and free DB */ + return MDB_SUCCESS; +} + +int ESECT +mdb_env_set_maxreaders(MDB_env *env, unsigned int readers) +{ + if (env->me_map || readers < 1) + return EINVAL; + env->me_maxreaders = readers; + return MDB_SUCCESS; +} + +int ESECT +mdb_env_get_maxreaders(MDB_env *env, unsigned int *readers) +{ + if (!env || !readers) + return EINVAL; + *readers = env->me_maxreaders; + return MDB_SUCCESS; +} + +/** Further setup required for opening an LMDB environment + */ +static int ESECT +mdb_env_open2(MDB_env *env) +{ + unsigned int flags = env->me_flags; + int i, newenv = 0, rc; + MDB_meta meta; + +#ifdef _WIN32 + /* See if we should use QueryLimited */ + rc = GetVersion(); + if ((rc & 0xff) > 5) + env->me_pidquery = MDB_PROCESS_QUERY_LIMITED_INFORMATION; + else + env->me_pidquery = PROCESS_QUERY_INFORMATION; +#endif /* _WIN32 */ + + memset(&meta, 0, sizeof(meta)); + + if ((i = mdb_env_read_header(env, &meta)) != 0) { + if (i != ENOENT) + return i; + DPUTS("new mdbenv"); + newenv = 1; + env->me_psize = env->me_os_psize; + if (env->me_psize > MAX_PAGESIZE) + env->me_psize = MAX_PAGESIZE; + } else { + env->me_psize = meta.mm_psize; + } + + /* Was a mapsize configured? */ + if (!env->me_mapsize) { + /* If this is a new environment, take the default, + * else use the size recorded in the existing env. + */ + env->me_mapsize = newenv ? DEFAULT_MAPSIZE : meta.mm_mapsize; + } else if (env->me_mapsize < meta.mm_mapsize) { + /* If the configured size is smaller, make sure it's + * still big enough. Silently round up to minimum if not. + */ + size_t minsize = (meta.mm_last_pg + 1) * meta.mm_psize; + if (env->me_mapsize < minsize) + env->me_mapsize = minsize; + } + + rc = mdb_env_map(env, (flags & MDB_FIXEDMAP) ? meta.mm_address : NULL); + if (rc) + return rc; + + if (newenv) { + if (flags & MDB_FIXEDMAP) + meta.mm_address = env->me_map; + i = mdb_env_init_meta(env, &meta); + if (i != MDB_SUCCESS) { + return i; + } + } + + env->me_maxfree_1pg = (env->me_psize - PAGEHDRSZ) / sizeof(pgno_t) - 1; + env->me_nodemax = (((env->me_psize - PAGEHDRSZ) / MDB_MINKEYS) & -2) + - sizeof(indx_t); +#if !(MDB_MAXKEYSIZE) + env->me_maxkey = env->me_nodemax - (NODESIZE + sizeof(MDB_db)); +#endif + env->me_maxpg = env->me_mapsize / env->me_psize; + +#if MDB_DEBUG + { + int toggle = mdb_env_pick_meta(env); + MDB_db *db = &env->me_metas[toggle]->mm_dbs[MAIN_DBI]; + + DPRINTF(("opened database version %u, pagesize %u", + env->me_metas[0]->mm_version, env->me_psize)); + DPRINTF(("using meta page %d", toggle)); + DPRINTF(("depth: %u", db->md_depth)); + DPRINTF(("entries: %"Z"u", db->md_entries)); + DPRINTF(("branch pages: %"Z"u", db->md_branch_pages)); + DPRINTF(("leaf pages: %"Z"u", db->md_leaf_pages)); + DPRINTF(("overflow pages: %"Z"u", db->md_overflow_pages)); + DPRINTF(("root: %"Z"u", db->md_root)); + } +#endif + + return MDB_SUCCESS; +} + + +/** Release a reader thread's slot in the reader lock table. + * This function is called automatically when a thread exits. + * @param[in] ptr This points to the slot in the reader lock table. + */ +static void +mdb_env_reader_dest(void *ptr) +{ + MDB_reader *reader = ptr; + + reader->mr_pid = 0; +} + +#ifdef _WIN32 +/** Junk for arranging thread-specific callbacks on Windows. This is + * necessarily platform and compiler-specific. Windows supports up + * to 1088 keys. Let's assume nobody opens more than 64 environments + * in a single process, for now. They can override this if needed. + */ +#ifndef MAX_TLS_KEYS +#define MAX_TLS_KEYS 64 +#endif +static pthread_key_t mdb_tls_keys[MAX_TLS_KEYS]; +static int mdb_tls_nkeys; + +static void NTAPI mdb_tls_callback(PVOID module, DWORD reason, PVOID ptr) +{ + int i; + switch(reason) { + case DLL_PROCESS_ATTACH: break; + case DLL_THREAD_ATTACH: break; + case DLL_THREAD_DETACH: + for (i=0; i<mdb_tls_nkeys; i++) { + MDB_reader *r = pthread_getspecific(mdb_tls_keys[i]); + if (r) { + mdb_env_reader_dest(r); + } + } + break; + case DLL_PROCESS_DETACH: break; + } +} +#ifdef __GNUC__ +#ifdef _WIN64 +const PIMAGE_TLS_CALLBACK mdb_tls_cbp __attribute__((section (".CRT$XLB"))) = mdb_tls_callback; +#else +PIMAGE_TLS_CALLBACK mdb_tls_cbp __attribute__((section (".CRT$XLB"))) = mdb_tls_callback; +#endif +#else +#ifdef _WIN64 +/* Force some symbol references. + * _tls_used forces the linker to create the TLS directory if not already done + * mdb_tls_cbp prevents whole-program-optimizer from dropping the symbol. + */ +#pragma comment(linker, "/INCLUDE:_tls_used") +#pragma comment(linker, "/INCLUDE:mdb_tls_cbp") +#pragma const_seg(".CRT$XLB") +extern const PIMAGE_TLS_CALLBACK mdb_tls_cbp; +const PIMAGE_TLS_CALLBACK mdb_tls_cbp = mdb_tls_callback; +#pragma const_seg() +#else /* WIN32 */ +#pragma comment(linker, "/INCLUDE:__tls_used") +#pragma comment(linker, "/INCLUDE:_mdb_tls_cbp") +#pragma data_seg(".CRT$XLB") +PIMAGE_TLS_CALLBACK mdb_tls_cbp = mdb_tls_callback; +#pragma data_seg() +#endif /* WIN 32/64 */ +#endif /* !__GNUC__ */ +#endif + +/** Downgrade the exclusive lock on the region back to shared */ +static int ESECT +mdb_env_share_locks(MDB_env *env, int *excl) +{ + int rc = 0, toggle = mdb_env_pick_meta(env); + + env->me_txns->mti_txnid = env->me_metas[toggle]->mm_txnid; + +#ifdef _WIN32 + { + OVERLAPPED ov; + /* First acquire a shared lock. The Unlock will + * then release the existing exclusive lock. + */ + memset(&ov, 0, sizeof(ov)); + if (!LockFileEx(env->me_lfd, 0, 0, 1, 0, &ov)) { + rc = ErrCode(); + } else { + UnlockFile(env->me_lfd, 0, 0, 1, 0); + *excl = 0; + } + } +#else + { + struct flock lock_info; + /* The shared lock replaces the existing lock */ + memset((void *)&lock_info, 0, sizeof(lock_info)); + lock_info.l_type = F_RDLCK; + lock_info.l_whence = SEEK_SET; + lock_info.l_start = 0; + lock_info.l_len = 1; + while ((rc = fcntl(env->me_lfd, F_SETLK, &lock_info)) && + (rc = ErrCode()) == EINTR) ; + *excl = rc ? -1 : 0; /* error may mean we lost the lock */ + } +#endif + + return rc; +} + +/** Try to get exlusive lock, otherwise shared. + * Maintain *excl = -1: no/unknown lock, 0: shared, 1: exclusive. + */ +static int ESECT +mdb_env_excl_lock(MDB_env *env, int *excl) +{ + int rc = 0; +#ifdef _WIN32 + if (LockFile(env->me_lfd, 0, 0, 1, 0)) { + *excl = 1; + } else { + OVERLAPPED ov; + memset(&ov, 0, sizeof(ov)); + if (LockFileEx(env->me_lfd, 0, 0, 1, 0, &ov)) { + *excl = 0; + } else { + rc = ErrCode(); + } + } +#else + struct flock lock_info; + memset((void *)&lock_info, 0, sizeof(lock_info)); + lock_info.l_type = F_WRLCK; + lock_info.l_whence = SEEK_SET; + lock_info.l_start = 0; + lock_info.l_len = 1; + while ((rc = fcntl(env->me_lfd, F_SETLK, &lock_info)) && + (rc = ErrCode()) == EINTR) ; + if (!rc) { + *excl = 1; + } else +# ifdef MDB_USE_POSIX_SEM + if (*excl < 0) /* always true when !MDB_USE_POSIX_SEM */ +# endif + { + lock_info.l_type = F_RDLCK; + while ((rc = fcntl(env->me_lfd, F_SETLKW, &lock_info)) && + (rc = ErrCode()) == EINTR) ; + if (rc == 0) + *excl = 0; + } +#endif + return rc; +} + +#ifdef MDB_USE_HASH +/* + * hash_64 - 64 bit Fowler/Noll/Vo-0 FNV-1a hash code + * + * @(#) $Revision: 5.1 $ + * @(#) $Id: hash_64a.c,v 5.1 2009/06/30 09:01:38 chongo Exp $ + * @(#) $Source: /usr/local/src/cmd/fnv/RCS/hash_64a.c,v $ + * + * http://www.isthe.com/chongo/tech/comp/fnv/index.html + * + *** + * + * Please do not copyright this code. This code is in the public domain. + * + * LANDON CURT NOLL DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, + * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO + * EVENT SHALL LANDON CURT NOLL BE LIABLE FOR ANY SPECIAL, INDIRECT OR + * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF + * USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR + * OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THIS SOFTWARE. + * + * By: + * chongo <Landon Curt Noll> /\oo/\ + * http://www.isthe.com/chongo/ + * + * Share and Enjoy! :-) + */ + +typedef unsigned long long mdb_hash_t; +#define MDB_HASH_INIT ((mdb_hash_t)0xcbf29ce484222325ULL) + +/** perform a 64 bit Fowler/Noll/Vo FNV-1a hash on a buffer + * @param[in] val value to hash + * @param[in] hval initial value for hash + * @return 64 bit hash + * + * NOTE: To use the recommended 64 bit FNV-1a hash, use MDB_HASH_INIT as the + * hval arg on the first call. + */ +static mdb_hash_t +mdb_hash_val(MDB_val *val, mdb_hash_t hval) +{ + unsigned char *s = (unsigned char *)val->mv_data; /* unsigned string */ + unsigned char *end = s + val->mv_size; + /* + * FNV-1a hash each octet of the string + */ + while (s < end) { + /* xor the bottom with the current octet */ + hval ^= (mdb_hash_t)*s++; + + /* multiply by the 64 bit FNV magic prime mod 2^64 */ + hval += (hval << 1) + (hval << 4) + (hval << 5) + + (hval << 7) + (hval << 8) + (hval << 40); + } + /* return our new hash value */ + return hval; +} + +/** Hash the string and output the encoded hash. + * This uses modified RFC1924 Ascii85 encoding to accommodate systems with + * very short name limits. We don't care about the encoding being reversible, + * we just want to preserve as many bits of the input as possible in a + * small printable string. + * @param[in] str string to hash + * @param[out] encbuf an array of 11 chars to hold the hash + */ +static const char mdb_a85[]= "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~"; + +static void +mdb_pack85(unsigned long l, char *out) +{ + int i; + + for (i=0; i<5; i++) { + *out++ = mdb_a85[l % 85]; + l /= 85; + } +} + +static void +mdb_hash_enc(MDB_val *val, char *encbuf) +{ + mdb_hash_t h = mdb_hash_val(val, MDB_HASH_INIT); + + mdb_pack85(h, encbuf); + mdb_pack85(h>>32, encbuf+5); + encbuf[10] = '\0'; +} +#endif + +/** Open and/or initialize the lock region for the environment. + * @param[in] env The LMDB environment. + * @param[in] lpath The pathname of the file used for the lock region. + * @param[in] mode The Unix permissions for the file, if we create it. + * @param[out] excl Resulting file lock type: -1 none, 0 shared, 1 exclusive + * @param[in,out] excl In -1, out lock type: -1 none, 0 shared, 1 exclusive + * @return 0 on success, non-zero on failure. + */ +static int ESECT +mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl) +{ +#ifdef _WIN32 +# define MDB_ERRCODE_ROFS ERROR_WRITE_PROTECT +#else +# define MDB_ERRCODE_ROFS EROFS +#ifdef O_CLOEXEC /* Linux: Open file and set FD_CLOEXEC atomically */ +# define MDB_CLOEXEC O_CLOEXEC +#else + int fdflags; +# define MDB_CLOEXEC 0 +#endif +#endif + int rc; + off_t size, rsize; + +#ifdef _WIN32 + env->me_lfd = CreateFile(lpath, GENERIC_READ|GENERIC_WRITE, + FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, NULL); +#else + env->me_lfd = open(lpath, O_RDWR|O_CREAT|MDB_CLOEXEC, mode); +#endif + if (env->me_lfd == INVALID_HANDLE_VALUE) { + rc = ErrCode(); + if (rc == MDB_ERRCODE_ROFS && (env->me_flags & MDB_RDONLY)) { + return MDB_SUCCESS; + } + goto fail_errno; + } +#if ! ((MDB_CLOEXEC) || defined(_WIN32)) + /* Lose record locks when exec*() */ + if ((fdflags = fcntl(env->me_lfd, F_GETFD) | FD_CLOEXEC) >= 0) + fcntl(env->me_lfd, F_SETFD, fdflags); +#endif + + if (!(env->me_flags & MDB_NOTLS)) { + rc = pthread_key_create(&env->me_txkey, mdb_env_reader_dest); + if (rc) + goto fail; + env->me_flags |= MDB_ENV_TXKEY; +#ifdef _WIN32 + /* Windows TLS callbacks need help finding their TLS info. */ + if (mdb_tls_nkeys >= MAX_TLS_KEYS) { + rc = MDB_TLS_FULL; + goto fail; + } + mdb_tls_keys[mdb_tls_nkeys++] = env->me_txkey; +#endif + } + + /* Try to get exclusive lock. If we succeed, then + * nobody is using the lock region and we should initialize it. + */ + if ((rc = mdb_env_excl_lock(env, excl))) goto fail; + +#ifdef _WIN32 + size = GetFileSize(env->me_lfd, NULL); +#else + size = lseek(env->me_lfd, 0, SEEK_END); + if (size == -1) goto fail_errno; +#endif + rsize = (env->me_maxreaders-1) * sizeof(MDB_reader) + sizeof(MDB_txninfo); + if (size < rsize && *excl > 0) { +#ifdef _WIN32 + if (SetFilePointer(env->me_lfd, rsize, NULL, FILE_BEGIN) != (DWORD)rsize + || !SetEndOfFile(env->me_lfd)) + goto fail_errno; +#else + if (ftruncate(env->me_lfd, rsize) != 0) goto fail_errno; +#endif + } else { + rsize = size; + size = rsize - sizeof(MDB_txninfo); + env->me_maxreaders = size/sizeof(MDB_reader) + 1; + } + { +#ifdef _WIN32 + HANDLE mh; + mh = CreateFileMapping(env->me_lfd, NULL, PAGE_READWRITE, + 0, 0, NULL); + if (!mh) goto fail_errno; + env->me_txns = MapViewOfFileEx(mh, FILE_MAP_WRITE, 0, 0, rsize, NULL); + CloseHandle(mh); + if (!env->me_txns) goto fail_errno; +#else + void *m = mmap(NULL, rsize, PROT_READ|PROT_WRITE, MAP_SHARED, + env->me_lfd, 0); + if (m == MAP_FAILED) goto fail_errno; + env->me_txns = m; +#endif + } + if (*excl > 0) { +#ifdef _WIN32 + BY_HANDLE_FILE_INFORMATION stbuf; + struct { + DWORD volume; + DWORD nhigh; + DWORD nlow; + } idbuf; + MDB_val val; + char encbuf[11]; + + if (!mdb_sec_inited) { + InitializeSecurityDescriptor(&mdb_null_sd, + SECURITY_DESCRIPTOR_REVISION); + SetSecurityDescriptorDacl(&mdb_null_sd, TRUE, 0, FALSE); + mdb_all_sa.nLength = sizeof(SECURITY_ATTRIBUTES); + mdb_all_sa.bInheritHandle = FALSE; + mdb_all_sa.lpSecurityDescriptor = &mdb_null_sd; + mdb_sec_inited = 1; + } + if (!GetFileInformationByHandle(env->me_lfd, &stbuf)) goto fail_errno; + idbuf.volume = stbuf.dwVolumeSerialNumber; + idbuf.nhigh = stbuf.nFileIndexHigh; + idbuf.nlow = stbuf.nFileIndexLow; + val.mv_data = &idbuf; + val.mv_size = sizeof(idbuf); + mdb_hash_enc(&val, encbuf); + sprintf(env->me_txns->mti_rmname, "Global\\MDBr%s", encbuf); + sprintf(env->me_txns->mti_wmname, "Global\\MDBw%s", encbuf); + env->me_rmutex = CreateMutex(&mdb_all_sa, FALSE, env->me_txns->mti_rmname); + if (!env->me_rmutex) goto fail_errno; + env->me_wmutex = CreateMutex(&mdb_all_sa, FALSE, env->me_txns->mti_wmname); + if (!env->me_wmutex) goto fail_errno; +#elif defined(MDB_USE_POSIX_SEM) + struct stat stbuf; + struct { + dev_t dev; + ino_t ino; + } idbuf; + MDB_val val; + char encbuf[11]; + +#if defined(__NetBSD__) +#define MDB_SHORT_SEMNAMES 1 /* limited to 14 chars */ +#endif + if (fstat(env->me_lfd, &stbuf)) goto fail_errno; + idbuf.dev = stbuf.st_dev; + idbuf.ino = stbuf.st_ino; + val.mv_data = &idbuf; + val.mv_size = sizeof(idbuf); + mdb_hash_enc(&val, encbuf); +#ifdef MDB_SHORT_SEMNAMES + encbuf[9] = '\0'; /* drop name from 15 chars to 14 chars */ +#endif + sprintf(env->me_txns->mti_rmname, "/MDBr%s", encbuf); + sprintf(env->me_txns->mti_wmname, "/MDBw%s", encbuf); + /* Clean up after a previous run, if needed: Try to + * remove both semaphores before doing anything else. + */ + sem_unlink(env->me_txns->mti_rmname); + sem_unlink(env->me_txns->mti_wmname); + env->me_rmutex = sem_open(env->me_txns->mti_rmname, + O_CREAT|O_EXCL, mode, 1); + if (env->me_rmutex == SEM_FAILED) goto fail_errno; + env->me_wmutex = sem_open(env->me_txns->mti_wmname, + O_CREAT|O_EXCL, mode, 1); + if (env->me_wmutex == SEM_FAILED) goto fail_errno; +#else /* MDB_USE_POSIX_SEM */ + pthread_mutexattr_t mattr; + + if ((rc = pthread_mutexattr_init(&mattr)) + || (rc = pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED)) + || (rc = pthread_mutex_init(&env->me_txns->mti_mutex, &mattr)) + || (rc = pthread_mutex_init(&env->me_txns->mti_wmutex, &mattr))) + goto fail; + pthread_mutexattr_destroy(&mattr); +#endif /* _WIN32 || MDB_USE_POSIX_SEM */ + + env->me_txns->mti_magic = MDB_MAGIC; + env->me_txns->mti_format = MDB_LOCK_FORMAT; + env->me_txns->mti_txnid = 0; + env->me_txns->mti_numreaders = 0; + + } else { + if (env->me_txns->mti_magic != MDB_MAGIC) { + DPUTS("lock region has invalid magic"); + rc = MDB_INVALID; + goto fail; + } + if (env->me_txns->mti_format != MDB_LOCK_FORMAT) { + DPRINTF(("lock region has format+version 0x%x, expected 0x%x", + env->me_txns->mti_format, MDB_LOCK_FORMAT)); + rc = MDB_VERSION_MISMATCH; + goto fail; + } + rc = ErrCode(); + if (rc && rc != EACCES && rc != EAGAIN) { + goto fail; + } +#ifdef _WIN32 + env->me_rmutex = OpenMutex(SYNCHRONIZE, FALSE, env->me_txns->mti_rmname); + if (!env->me_rmutex) goto fail_errno; + env->me_wmutex = OpenMutex(SYNCHRONIZE, FALSE, env->me_txns->mti_wmname); + if (!env->me_wmutex) goto fail_errno; +#elif defined(MDB_USE_POSIX_SEM) + env->me_rmutex = sem_open(env->me_txns->mti_rmname, 0); + if (env->me_rmutex == SEM_FAILED) goto fail_errno; + env->me_wmutex = sem_open(env->me_txns->mti_wmname, 0); + if (env->me_wmutex == SEM_FAILED) goto fail_errno; +#endif + } + return MDB_SUCCESS; + +fail_errno: + rc = ErrCode(); +fail: + return rc; +} + + /** The name of the lock file in the DB environment */ +#define LOCKNAME "/lock.mdb" + /** The name of the data file in the DB environment */ +#define DATANAME "/data.mdb" + /** The suffix of the lock file when no subdir is used */ +#define LOCKSUFF "-lock" + /** Only a subset of the @ref mdb_env flags can be changed + * at runtime. Changing other flags requires closing the + * environment and re-opening it with the new flags. + */ +#define CHANGEABLE (MDB_NOSYNC|MDB_NOMETASYNC|MDB_MAPASYNC|MDB_NOMEMINIT) +#define CHANGELESS (MDB_FIXEDMAP|MDB_NOSUBDIR|MDB_RDONLY|MDB_WRITEMAP| \ + MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD) + +#if VALID_FLAGS & PERSISTENT_FLAGS & (CHANGEABLE|CHANGELESS) +# error "Persistent DB flags & env flags overlap, but both go in mm_flags" +#endif + +int ESECT +mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode) +{ + int oflags, rc, len, excl = -1; + char *lpath, *dpath; + + if (env->me_fd!=INVALID_HANDLE_VALUE || (flags & ~(CHANGEABLE|CHANGELESS))) + return EINVAL; + +#ifdef VL32 + if (flags & MDB_WRITEMAP) { + /* silently ignore WRITEMAP in 32 bit mode */ + flags ^= MDB_WRITEMAP; + } + if (flags & MDB_FIXEDMAP) { + /* cannot support FIXEDMAP */ + return EINVAL; + } +#endif + + len = strlen(path); + if (flags & MDB_NOSUBDIR) { + rc = len + sizeof(LOCKSUFF) + len + 1; + } else { + rc = len + sizeof(LOCKNAME) + len + sizeof(DATANAME); + } + lpath = malloc(rc); + if (!lpath) + return ENOMEM; + if (flags & MDB_NOSUBDIR) { + dpath = lpath + len + sizeof(LOCKSUFF); + sprintf(lpath, "%s" LOCKSUFF, path); + strcpy(dpath, path); + } else { + dpath = lpath + len + sizeof(LOCKNAME); + sprintf(lpath, "%s" LOCKNAME, path); + sprintf(dpath, "%s" DATANAME, path); + } + + rc = MDB_SUCCESS; + flags |= env->me_flags; + if (flags & MDB_RDONLY) { + /* silently ignore WRITEMAP when we're only getting read access */ + flags &= ~MDB_WRITEMAP; + } else { + if (!((env->me_free_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX)) && + (env->me_dirty_list = calloc(MDB_IDL_UM_SIZE, sizeof(MDB_ID2))))) + rc = ENOMEM; + } + env->me_flags = flags |= MDB_ENV_ACTIVE; + if (rc) + goto leave; + + env->me_path = strdup(path); + env->me_dbxs = calloc(env->me_maxdbs, sizeof(MDB_dbx)); + env->me_dbflags = calloc(env->me_maxdbs, sizeof(uint16_t)); + env->me_dbiseqs = calloc(env->me_maxdbs, sizeof(unsigned int)); + if (!(env->me_dbxs && env->me_path && env->me_dbflags && env->me_dbiseqs)) { + rc = ENOMEM; + goto leave; + } + + /* For RDONLY, get lockfile after we know datafile exists */ + if (!(flags & (MDB_RDONLY|MDB_NOLOCK))) { + rc = mdb_env_setup_locks(env, lpath, mode, &excl); + if (rc) + goto leave; + } + +#ifdef _WIN32 + if (F_ISSET(flags, MDB_RDONLY)) { + oflags = GENERIC_READ; + len = OPEN_EXISTING; + } else { + oflags = GENERIC_READ|GENERIC_WRITE; + len = OPEN_ALWAYS; + } + mode = FILE_ATTRIBUTE_NORMAL; + env->me_fd = CreateFile(dpath, oflags, FILE_SHARE_READ|FILE_SHARE_WRITE, + NULL, len, mode, NULL); +#else + if (F_ISSET(flags, MDB_RDONLY)) + oflags = O_RDONLY; + else + oflags = O_RDWR | O_CREAT; + + env->me_fd = open(dpath, oflags, mode); +#endif + if (env->me_fd == INVALID_HANDLE_VALUE) { + rc = ErrCode(); + goto leave; + } + + if ((flags & (MDB_RDONLY|MDB_NOLOCK)) == MDB_RDONLY) { + rc = mdb_env_setup_locks(env, lpath, mode, &excl); + if (rc) + goto leave; + } + + if ((rc = mdb_env_open2(env)) == MDB_SUCCESS) { + if (flags & (MDB_RDONLY|MDB_WRITEMAP)) { + env->me_mfd = env->me_fd; + } else { + /* Synchronous fd for meta writes. Needed even with + * MDB_NOSYNC/MDB_NOMETASYNC, in case these get reset. + */ +#ifdef _WIN32 + len = OPEN_EXISTING; + env->me_mfd = CreateFile(dpath, oflags, + FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, len, + mode | FILE_FLAG_WRITE_THROUGH, NULL); +#else + oflags &= ~O_CREAT; + env->me_mfd = open(dpath, oflags | MDB_DSYNC, mode); +#endif + if (env->me_mfd == INVALID_HANDLE_VALUE) { + rc = ErrCode(); + goto leave; + } + } + DPRINTF(("opened dbenv %p", (void *) env)); + if (excl > 0) { + rc = mdb_env_share_locks(env, &excl); + if (rc) + goto leave; + } + if (!((flags & MDB_RDONLY) || + (env->me_pbuf = calloc(1, env->me_psize)))) + rc = ENOMEM; + if (!(flags & MDB_RDONLY)) { + MDB_txn *txn; + int tsize = sizeof(MDB_txn), size = tsize + env->me_maxdbs * + (sizeof(MDB_db)+sizeof(MDB_cursor)+sizeof(unsigned int)+1); + txn = calloc(1, size); + if (txn) { + txn->mt_dbs = (MDB_db *)((char *)txn + tsize); + txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs); + txn->mt_dbiseqs = (unsigned int *)(txn->mt_cursors + env->me_maxdbs); + txn->mt_dbflags = (unsigned char *)(txn->mt_dbiseqs + env->me_maxdbs); + txn->mt_env = env; +#ifdef VL32 + txn->mt_rpages = calloc(MDB_IDL_UM_SIZE, sizeof(MDB_ID2)); + if (!txn->mt_rpages) { + free(txn); + rc = ENOMEM; + } +#endif + env->me_txn0 = txn; + } else { + rc = ENOMEM; + } + } + } + +leave: + if (rc) { + mdb_env_close0(env, excl); + } + free(lpath); + return rc; +} + +/** Destroy resources from mdb_env_open(), clear our readers & DBIs */ +static void ESECT +mdb_env_close0(MDB_env *env, int excl) +{ + int i; + + if (!(env->me_flags & MDB_ENV_ACTIVE)) + return; + + /* Doing this here since me_dbxs may not exist during mdb_env_close */ + for (i = env->me_maxdbs; --i > MAIN_DBI; ) + free(env->me_dbxs[i].md_name.mv_data); + + free(env->me_pbuf); + free(env->me_dbiseqs); + free(env->me_dbflags); + free(env->me_dbxs); + free(env->me_path); + free(env->me_dirty_list); + free(env->me_txn0); + mdb_midl_free(env->me_free_pgs); + + if (env->me_flags & MDB_ENV_TXKEY) { + pthread_key_delete(env->me_txkey); +#ifdef _WIN32 + /* Delete our key from the global list */ + for (i=0; i<mdb_tls_nkeys; i++) + if (mdb_tls_keys[i] == env->me_txkey) { + mdb_tls_keys[i] = mdb_tls_keys[mdb_tls_nkeys-1]; + mdb_tls_nkeys--; + break; + } +#endif + } + + if (env->me_map) { +#ifdef VL32 + munmap(env->me_map, 2*env->me_psize); +#else + munmap(env->me_map, env->me_mapsize); +#endif + } + if (env->me_mfd != env->me_fd && env->me_mfd != INVALID_HANDLE_VALUE) + (void) close(env->me_mfd); + if (env->me_fd != INVALID_HANDLE_VALUE) + (void) close(env->me_fd); + if (env->me_txns) { + MDB_PID_T pid = env->me_pid; + /* Clearing readers is done in this function because + * me_txkey with its destructor must be disabled first. + */ + for (i = env->me_numreaders; --i >= 0; ) + if (env->me_txns->mti_readers[i].mr_pid == pid) + env->me_txns->mti_readers[i].mr_pid = 0; +#ifdef _WIN32 + if (env->me_rmutex) { + CloseHandle(env->me_rmutex); + if (env->me_wmutex) CloseHandle(env->me_wmutex); + } + /* Windows automatically destroys the mutexes when + * the last handle closes. + */ +#elif defined(MDB_USE_POSIX_SEM) + if (env->me_rmutex != SEM_FAILED) { + sem_close(env->me_rmutex); + if (env->me_wmutex != SEM_FAILED) + sem_close(env->me_wmutex); + /* If we have the filelock: If we are the + * only remaining user, clean up semaphores. + */ + if (excl == 0) + mdb_env_excl_lock(env, &excl); + if (excl > 0) { + sem_unlink(env->me_txns->mti_rmname); + sem_unlink(env->me_txns->mti_wmname); + } + } +#endif + munmap((void *)env->me_txns, (env->me_maxreaders-1)*sizeof(MDB_reader)+sizeof(MDB_txninfo)); + } + if (env->me_lfd != INVALID_HANDLE_VALUE) { +#ifdef _WIN32 + if (excl >= 0) { + /* Unlock the lockfile. Windows would have unlocked it + * after closing anyway, but not necessarily at once. + */ + UnlockFile(env->me_lfd, 0, 0, 1, 0); + } +#endif + (void) close(env->me_lfd); + } + + env->me_flags &= ~(MDB_ENV_ACTIVE|MDB_ENV_TXKEY); +} + + +void ESECT +mdb_env_close(MDB_env *env) +{ + MDB_page *dp; + + if (env == NULL) + return; + + VGMEMP_DESTROY(env); + while ((dp = env->me_dpages) != NULL) { + VGMEMP_DEFINED(&dp->mp_next, sizeof(dp->mp_next)); + env->me_dpages = dp->mp_next; + free(dp); + } + + mdb_env_close0(env, 0); + free(env); +} + +/** Compare two items pointing at aligned size_t's */ +static int +mdb_cmp_long(const MDB_val *a, const MDB_val *b) +{ + return (*(size_t *)a->mv_data < *(size_t *)b->mv_data) ? -1 : + *(size_t *)a->mv_data > *(size_t *)b->mv_data; +} + +/** Compare two items pointing at aligned unsigned int's */ +static int +mdb_cmp_int(const MDB_val *a, const MDB_val *b) +{ + return (*(unsigned int *)a->mv_data < *(unsigned int *)b->mv_data) ? -1 : + *(unsigned int *)a->mv_data > *(unsigned int *)b->mv_data; +} + +/** Compare two items pointing at unsigned ints of unknown alignment. + * Nodes and keys are guaranteed to be 2-byte aligned. + */ +static int +mdb_cmp_cint(const MDB_val *a, const MDB_val *b) +{ +#if BYTE_ORDER == LITTLE_ENDIAN + unsigned short *u, *c; + int x; + + u = (unsigned short *) ((char *) a->mv_data + a->mv_size); + c = (unsigned short *) ((char *) b->mv_data + a->mv_size); + do { + x = *--u - *--c; + } while(!x && u > (unsigned short *)a->mv_data); + return x; +#else + unsigned short *u, *c, *end; + int x; + + end = (unsigned short *) ((char *) a->mv_data + a->mv_size); + u = (unsigned short *)a->mv_data; + c = (unsigned short *)b->mv_data; + do { + x = *u++ - *c++; + } while(!x && u < end); + return x; +#endif +} + +/** Compare two items pointing at size_t's of unknown alignment. */ +#ifdef MISALIGNED_OK +# define mdb_cmp_clong mdb_cmp_long +#else +# define mdb_cmp_clong mdb_cmp_cint +#endif + +/** Compare two items lexically */ +static int +mdb_cmp_memn(const MDB_val *a, const MDB_val *b) +{ + int diff; + ssize_t len_diff; + unsigned int len; + + len = a->mv_size; + len_diff = (ssize_t) a->mv_size - (ssize_t) b->mv_size; + if (len_diff > 0) { + len = b->mv_size; + len_diff = 1; + } + + diff = memcmp(a->mv_data, b->mv_data, len); + return diff ? diff : len_diff<0 ? -1 : len_diff; +} + +/** Compare two items in reverse byte order */ +static int +mdb_cmp_memnr(const MDB_val *a, const MDB_val *b) +{ + const unsigned char *p1, *p2, *p1_lim; + ssize_t len_diff; + int diff; + + p1_lim = (const unsigned char *)a->mv_data; + p1 = (const unsigned char *)a->mv_data + a->mv_size; + p2 = (const unsigned char *)b->mv_data + b->mv_size; + + len_diff = (ssize_t) a->mv_size - (ssize_t) b->mv_size; + if (len_diff > 0) { + p1_lim += len_diff; + len_diff = 1; + } + + while (p1 > p1_lim) { + diff = *--p1 - *--p2; + if (diff) + return diff; + } + return len_diff<0 ? -1 : len_diff; +} + +/** Search for key within a page, using binary search. + * Returns the smallest entry larger or equal to the key. + * If exactp is non-null, stores whether the found entry was an exact match + * in *exactp (1 or 0). + * Updates the cursor index with the index of the found entry. + * If no entry larger or equal to the key is found, returns NULL. + */ +static MDB_node * +mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp) +{ + unsigned int i = 0, nkeys; + int low, high; + int rc = 0; + MDB_page *mp = mc->mc_pg[mc->mc_top]; + MDB_node *node = NULL; + MDB_val nodekey; + MDB_cmp_func *cmp; + DKBUF; + + nkeys = NUMKEYS(mp); + + DPRINTF(("searching %u keys in %s %spage %"Z"u", + nkeys, IS_LEAF(mp) ? "leaf" : "branch", IS_SUBP(mp) ? "sub-" : "", + mdb_dbg_pgno(mp))); + + low = IS_LEAF(mp) ? 0 : 1; + high = nkeys - 1; + cmp = mc->mc_dbx->md_cmp; + + /* Branch pages have no data, so if using integer keys, + * alignment is guaranteed. Use faster mdb_cmp_int. + */ + if (cmp == mdb_cmp_cint && IS_BRANCH(mp)) { + if (NODEPTR(mp, 1)->mn_ksize == sizeof(size_t)) + cmp = mdb_cmp_long; + else + cmp = mdb_cmp_int; + } + + if (IS_LEAF2(mp)) { + nodekey.mv_size = mc->mc_db->md_pad; + node = NODEPTR(mp, 0); /* fake */ + while (low <= high) { + i = (low + high) >> 1; + nodekey.mv_data = LEAF2KEY(mp, i, nodekey.mv_size); + rc = cmp(key, &nodekey); + DPRINTF(("found leaf index %u [%s], rc = %i", + i, DKEY(&nodekey), rc)); + if (rc == 0) + break; + if (rc > 0) + low = i + 1; + else + high = i - 1; + } + } else { + while (low <= high) { + i = (low + high) >> 1; + + node = NODEPTR(mp, i); + nodekey.mv_size = NODEKSZ(node); + nodekey.mv_data = NODEKEY(node); + + rc = cmp(key, &nodekey); +#if MDB_DEBUG + if (IS_LEAF(mp)) + DPRINTF(("found leaf index %u [%s], rc = %i", + i, DKEY(&nodekey), rc)); + else + DPRINTF(("found branch index %u [%s -> %"Z"u], rc = %i", + i, DKEY(&nodekey), NODEPGNO(node), rc)); +#endif + if (rc == 0) + break; + if (rc > 0) + low = i + 1; + else + high = i - 1; + } + } + + if (rc > 0) { /* Found entry is less than the key. */ + i++; /* Skip to get the smallest entry larger than key. */ + if (!IS_LEAF2(mp)) + node = NODEPTR(mp, i); + } + if (exactp) + *exactp = (rc == 0 && nkeys > 0); + /* store the key index */ + mc->mc_ki[mc->mc_top] = i; + if (i >= nkeys) + /* There is no entry larger or equal to the key. */ + return NULL; + + /* nodeptr is fake for LEAF2 */ + return node; +} + +#if 0 +static void +mdb_cursor_adjust(MDB_cursor *mc, func) +{ + MDB_cursor *m2; + + for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { + if (m2->mc_pg[m2->mc_top] == mc->mc_pg[mc->mc_top]) { + func(mc, m2); + } + } +} +#endif + +/** Pop a page off the top of the cursor's stack. */ +static void +mdb_cursor_pop(MDB_cursor *mc) +{ + if (mc->mc_snum) { +#if MDB_DEBUG + MDB_page *top = mc->mc_pg[mc->mc_top]; +#endif + mc->mc_snum--; + if (mc->mc_snum) + mc->mc_top--; + + DPRINTF(("popped page %"Z"u off db %d cursor %p", top->mp_pgno, + DDBI(mc), (void *) mc)); + } +} + +/** Push a page onto the top of the cursor's stack. */ +static int +mdb_cursor_push(MDB_cursor *mc, MDB_page *mp) +{ + DPRINTF(("pushing page %"Z"u on db %d cursor %p", mp->mp_pgno, + DDBI(mc), (void *) mc)); + + if (mc->mc_snum >= CURSOR_STACK) { + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return MDB_CURSOR_FULL; + } + + mc->mc_top = mc->mc_snum++; + mc->mc_pg[mc->mc_top] = mp; + mc->mc_ki[mc->mc_top] = 0; + + return MDB_SUCCESS; +} + +/** Find the address of the page corresponding to a given page number. + * @param[in] txn the transaction for this access. + * @param[in] pgno the page number for the page to retrieve. + * @param[out] ret address of a pointer where the page's address will be stored. + * @param[out] lvl dirty_list inheritance level of found page. 1=current txn, 0=mapped page. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_get(MDB_txn *txn, pgno_t pgno, MDB_page **ret, int *lvl) +{ + MDB_env *env = txn->mt_env; + MDB_page *p = NULL; + int level; + + if (!((txn->mt_flags & MDB_TXN_RDONLY) | (env->me_flags & MDB_WRITEMAP))) { + MDB_txn *tx2 = txn; + level = 1; + do { + MDB_ID2L dl = tx2->mt_u.dirty_list; + unsigned x; + /* Spilled pages were dirtied in this txn and flushed + * because the dirty list got full. Bring this page + * back in from the map (but don't unspill it here, + * leave that unless page_touch happens again). + */ + if (tx2->mt_spill_pgs) { + MDB_ID pn = pgno << 1; + x = mdb_midl_search(tx2->mt_spill_pgs, pn); + if (x <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[x] == pn) { + p = (MDB_page *)(env->me_map + env->me_psize * pgno); + goto done; + } + } + if (dl[0].mid) { + unsigned x = mdb_mid2l_search(dl, pgno); + if (x <= dl[0].mid && dl[x].mid == pgno) { + p = dl[x].mptr; + goto done; + } + } + level++; + } while ((tx2 = tx2->mt_parent) != NULL); + } + + if (pgno < txn->mt_next_pgno) { + level = 0; +#ifdef VL32 + { + unsigned x = mdb_mid2l_search(txn->mt_rpages, pgno); + if (x <= txn->mt_rpages[0].mid && txn->mt_rpages[x].mid == pgno) { + p = txn->mt_rpages[x].mptr; + goto done; + } + if (txn->mt_rpages[0].mid >= MDB_IDL_UM_MAX) { + /* unmap some other page */ + mdb_tassert(txn, 0); + } + if (txn->mt_rpages[0].mid < MDB_IDL_UM_SIZE) { + MDB_ID2 id2; + size_t len = env->me_psize; + int np; +#ifdef _WIN32 + size_t off = pgno * env->me_psize; + DWORD lo, hi; + lo = off & 0xffffffff; + hi = off >> 16 >> 16; + p = MapViewOfFile(env->me_fmh, FILE_MAP_READ, hi, lo, len); + if (p == NULL) + return ErrCode(); + if (IS_OVERFLOW(p)) { + np = p->mp_pages; + UnmapViewOfFile(p); + len *= np; + p = MapViewOfFile(env->me_fmh, FILE_MAP_READ, hi, lo, len); + if (p == NULL) + return ErrCode(); + } +#else + off_t off = pgno * env->me_psize; + p = mmap(NULL, len, PROT_READ, MAP_SHARED, env->me_fd, off); + if (p == MAP_FAILED) + return errno; + if (IS_OVERFLOW(p)) { + np = p->mp_pages; + munmap(p, len); + len *= np; + p = mmap(NULL, len, PROT_READ, MAP_SHARED, env->me_fd, off); + if (p == MAP_FAILED) + return errno; + } +#endif + id2.mid = pgno; + id2.mptr = p; + mdb_mid2l_insert(txn->mt_rpages, &id2); + goto done; + } + } +#else + p = (MDB_page *)(env->me_map + env->me_psize * pgno); +#endif + } else { + DPRINTF(("page %"Z"u not found", pgno)); + txn->mt_flags |= MDB_TXN_ERROR; + return MDB_PAGE_NOTFOUND; + } + +done: + *ret = p; + if (lvl) + *lvl = level; + return MDB_SUCCESS; +} + +/** Finish #mdb_page_search() / #mdb_page_search_lowest(). + * The cursor is at the root page, set up the rest of it. + */ +static int +mdb_page_search_root(MDB_cursor *mc, MDB_val *key, int flags) +{ + MDB_page *mp = mc->mc_pg[mc->mc_top]; + int rc; + DKBUF; + + while (IS_BRANCH(mp)) { + MDB_node *node; + indx_t i; + + DPRINTF(("branch page %"Z"u has %u keys", mp->mp_pgno, NUMKEYS(mp))); + mdb_cassert(mc, NUMKEYS(mp) > 1); + DPRINTF(("found index 0 to page %"Z"u", NODEPGNO(NODEPTR(mp, 0)))); + + if (flags & (MDB_PS_FIRST|MDB_PS_LAST)) { + i = 0; + if (flags & MDB_PS_LAST) + i = NUMKEYS(mp) - 1; + } else { + int exact; + node = mdb_node_search(mc, key, &exact); + if (node == NULL) + i = NUMKEYS(mp) - 1; + else { + i = mc->mc_ki[mc->mc_top]; + if (!exact) { + mdb_cassert(mc, i > 0); + i--; + } + } + DPRINTF(("following index %u for key [%s]", i, DKEY(key))); + } + + mdb_cassert(mc, i < NUMKEYS(mp)); + node = NODEPTR(mp, i); + + if ((rc = mdb_page_get(mc->mc_txn, NODEPGNO(node), &mp, NULL)) != 0) + return rc; + + mc->mc_ki[mc->mc_top] = i; + if ((rc = mdb_cursor_push(mc, mp))) + return rc; + + if (flags & MDB_PS_MODIFY) { + if ((rc = mdb_page_touch(mc)) != 0) + return rc; + mp = mc->mc_pg[mc->mc_top]; + } + } + + if (!IS_LEAF(mp)) { + DPRINTF(("internal error, index points to a %02X page!?", + mp->mp_flags)); + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return MDB_CORRUPTED; + } + + DPRINTF(("found leaf page %"Z"u for key [%s]", mp->mp_pgno, + key ? DKEY(key) : "null")); + mc->mc_flags |= C_INITIALIZED; + mc->mc_flags &= ~C_EOF; + + return MDB_SUCCESS; +} + +/** Search for the lowest key under the current branch page. + * This just bypasses a NUMKEYS check in the current page + * before calling mdb_page_search_root(), because the callers + * are all in situations where the current page is known to + * be underfilled. + */ +static int +mdb_page_search_lowest(MDB_cursor *mc) +{ + MDB_page *mp = mc->mc_pg[mc->mc_top]; + MDB_node *node = NODEPTR(mp, 0); + int rc; + + if ((rc = mdb_page_get(mc->mc_txn, NODEPGNO(node), &mp, NULL)) != 0) + return rc; + + mc->mc_ki[mc->mc_top] = 0; + if ((rc = mdb_cursor_push(mc, mp))) + return rc; + return mdb_page_search_root(mc, NULL, MDB_PS_FIRST); +} + +/** Search for the page a given key should be in. + * Push it and its parent pages on the cursor stack. + * @param[in,out] mc the cursor for this operation. + * @param[in] key the key to search for, or NULL for first/last page. + * @param[in] flags If MDB_PS_MODIFY is set, visited pages in the DB + * are touched (updated with new page numbers). + * If MDB_PS_FIRST or MDB_PS_LAST is set, find first or last leaf. + * This is used by #mdb_cursor_first() and #mdb_cursor_last(). + * If MDB_PS_ROOTONLY set, just fetch root node, no further lookups. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_search(MDB_cursor *mc, MDB_val *key, int flags) +{ + int rc; + pgno_t root; + + /* Make sure the txn is still viable, then find the root from + * the txn's db table and set it as the root of the cursor's stack. + */ + if (F_ISSET(mc->mc_txn->mt_flags, MDB_TXN_ERROR)) { + DPUTS("transaction has failed, must abort"); + return MDB_BAD_TXN; + } else { + /* Make sure we're using an up-to-date root */ + if (*mc->mc_dbflag & DB_STALE) { + MDB_cursor mc2; + if (TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbi)) + return MDB_BAD_DBI; + mdb_cursor_init(&mc2, mc->mc_txn, MAIN_DBI, NULL); + rc = mdb_page_search(&mc2, &mc->mc_dbx->md_name, 0); + if (rc) + return rc; + { + MDB_val data; + int exact = 0; + uint16_t flags; + MDB_node *leaf = mdb_node_search(&mc2, + &mc->mc_dbx->md_name, &exact); + if (!exact) + return MDB_NOTFOUND; + rc = mdb_node_read(mc->mc_txn, leaf, &data); + if (rc) + return rc; + memcpy(&flags, ((char *) data.mv_data + offsetof(MDB_db, md_flags)), + sizeof(uint16_t)); + /* The txn may not know this DBI, or another process may + * have dropped and recreated the DB with other flags. + */ + if ((mc->mc_db->md_flags & PERSISTENT_FLAGS) != flags) + return MDB_INCOMPATIBLE; + memcpy(mc->mc_db, data.mv_data, sizeof(MDB_db)); + } + *mc->mc_dbflag &= ~DB_STALE; + } + root = mc->mc_db->md_root; + + if (root == P_INVALID) { /* Tree is empty. */ + DPUTS("tree is empty"); + return MDB_NOTFOUND; + } + } + + mdb_cassert(mc, root > 1); + if (!mc->mc_pg[0] || mc->mc_pg[0]->mp_pgno != root) + if ((rc = mdb_page_get(mc->mc_txn, root, &mc->mc_pg[0], NULL)) != 0) + return rc; + + mc->mc_snum = 1; + mc->mc_top = 0; + + DPRINTF(("db %d root page %"Z"u has flags 0x%X", + DDBI(mc), root, mc->mc_pg[0]->mp_flags)); + + if (flags & MDB_PS_MODIFY) { + if ((rc = mdb_page_touch(mc))) + return rc; + } + + if (flags & MDB_PS_ROOTONLY) + return MDB_SUCCESS; + + return mdb_page_search_root(mc, key, flags); +} + +static int +mdb_ovpage_free(MDB_cursor *mc, MDB_page *mp) +{ + MDB_txn *txn = mc->mc_txn; + pgno_t pg = mp->mp_pgno; + unsigned x = 0, ovpages = mp->mp_pages; + MDB_env *env = txn->mt_env; + MDB_IDL sl = txn->mt_spill_pgs; + MDB_ID pn = pg << 1; + int rc; + + DPRINTF(("free ov page %"Z"u (%d)", pg, ovpages)); + /* If the page is dirty or on the spill list we just acquired it, + * so we should give it back to our current free list, if any. + * Otherwise put it onto the list of pages we freed in this txn. + * + * Won't create me_pghead: me_pglast must be inited along with it. + * Unsupported in nested txns: They would need to hide the page + * range in ancestor txns' dirty and spilled lists. + */ + if (env->me_pghead && + !txn->mt_parent && + ((mp->mp_flags & P_DIRTY) || + (sl && (x = mdb_midl_search(sl, pn)) <= sl[0] && sl[x] == pn))) + { + unsigned i, j; + pgno_t *mop; + MDB_ID2 *dl, ix, iy; + rc = mdb_midl_need(&env->me_pghead, ovpages); + if (rc) + return rc; + if (!(mp->mp_flags & P_DIRTY)) { + /* This page is no longer spilled */ + if (x == sl[0]) + sl[0]--; + else + sl[x] |= 1; + goto release; + } + /* Remove from dirty list */ + dl = txn->mt_u.dirty_list; + x = dl[0].mid--; + for (ix = dl[x]; ix.mptr != mp; ix = iy) { + if (x > 1) { + x--; + iy = dl[x]; + dl[x] = ix; + } else { + mdb_cassert(mc, x > 1); + j = ++(dl[0].mid); + dl[j] = ix; /* Unsorted. OK when MDB_TXN_ERROR. */ + txn->mt_flags |= MDB_TXN_ERROR; + return MDB_CORRUPTED; + } + } + if (!(env->me_flags & MDB_WRITEMAP)) + mdb_dpage_free(env, mp); +release: + /* Insert in me_pghead */ + mop = env->me_pghead; + j = mop[0] + ovpages; + for (i = mop[0]; i && mop[i] < pg; i--) + mop[j--] = mop[i]; + while (j>i) + mop[j--] = pg++; + mop[0] += ovpages; + } else { + rc = mdb_midl_append_range(&txn->mt_free_pgs, pg, ovpages); + if (rc) + return rc; + } + mc->mc_db->md_overflow_pages -= ovpages; + return 0; +} + +/** Return the data associated with a given node. + * @param[in] txn The transaction for this operation. + * @param[in] leaf The node being read. + * @param[out] data Updated to point to the node's data. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_node_read(MDB_txn *txn, MDB_node *leaf, MDB_val *data) +{ + MDB_page *omp; /* overflow page */ + pgno_t pgno; + int rc; + + if (!F_ISSET(leaf->mn_flags, F_BIGDATA)) { + data->mv_size = NODEDSZ(leaf); + data->mv_data = NODEDATA(leaf); + return MDB_SUCCESS; + } + + /* Read overflow data. + */ + data->mv_size = NODEDSZ(leaf); + memcpy(&pgno, NODEDATA(leaf), sizeof(pgno)); + if ((rc = mdb_page_get(txn, pgno, &omp, NULL)) != 0) { + DPRINTF(("read overflow page %"Z"u failed", pgno)); + return rc; + } + data->mv_data = METADATA(omp); + + return MDB_SUCCESS; +} + +int +mdb_get(MDB_txn *txn, MDB_dbi dbi, + MDB_val *key, MDB_val *data) +{ + MDB_cursor mc; + MDB_xcursor mx; + int exact = 0; + DKBUF; + + DPRINTF(("===> get db %u key [%s]", dbi, DKEY(key))); + + if (!key || !data || dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + + if (txn->mt_flags & MDB_TXN_ERROR) + return MDB_BAD_TXN; + + mdb_cursor_init(&mc, txn, dbi, &mx); + return mdb_cursor_set(&mc, key, data, MDB_SET, &exact); +} + +/** Find a sibling for a page. + * Replaces the page at the top of the cursor's stack with the + * specified sibling, if one exists. + * @param[in] mc The cursor for this operation. + * @param[in] move_right Non-zero if the right sibling is requested, + * otherwise the left sibling. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_cursor_sibling(MDB_cursor *mc, int move_right) +{ + int rc; + MDB_node *indx; + MDB_page *mp; +#ifdef VL32 + MDB_page *op; +#endif + + if (mc->mc_snum < 2) { + return MDB_NOTFOUND; /* root has no siblings */ + } + +#ifdef VL32 + op = mc->mc_pg[mc->mc_top]; +#endif + mdb_cursor_pop(mc); + DPRINTF(("parent page is page %"Z"u, index %u", + mc->mc_pg[mc->mc_top]->mp_pgno, mc->mc_ki[mc->mc_top])); + + if (move_right ? (mc->mc_ki[mc->mc_top] + 1u >= NUMKEYS(mc->mc_pg[mc->mc_top])) + : (mc->mc_ki[mc->mc_top] == 0)) { + DPRINTF(("no more keys left, moving to %s sibling", + move_right ? "right" : "left")); + if ((rc = mdb_cursor_sibling(mc, move_right)) != MDB_SUCCESS) { + /* undo cursor_pop before returning */ + mc->mc_top++; + mc->mc_snum++; + return rc; + } + } else { + if (move_right) + mc->mc_ki[mc->mc_top]++; + else + mc->mc_ki[mc->mc_top]--; + DPRINTF(("just moving to %s index key %u", + move_right ? "right" : "left", mc->mc_ki[mc->mc_top])); + } + mdb_cassert(mc, IS_BRANCH(mc->mc_pg[mc->mc_top])); + +#ifdef VL32 + { + MDB_ID2L rl = mc->mc_txn->mt_rpages; + unsigned x = mdb_mid2l_search(rl, op->mp_pgno); + if (x <= rl[0].mid && rl[x].mid == op->mp_pgno) { + munmap(op, mc->mc_txn->mt_env->me_psize); + while (x < rl[0].mid) { + rl[x] = rl[x+1]; + x++; + } + rl[0].mid--; + } + } +#endif + + indx = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if ((rc = mdb_page_get(mc->mc_txn, NODEPGNO(indx), &mp, NULL)) != 0) { + /* mc will be inconsistent if caller does mc_snum++ as above */ + mc->mc_flags &= ~(C_INITIALIZED|C_EOF); + return rc; + } + + mdb_cursor_push(mc, mp); + if (!move_right) + mc->mc_ki[mc->mc_top] = NUMKEYS(mp)-1; + + return MDB_SUCCESS; +} + +/** Move the cursor to the next data item. */ +static int +mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) +{ + MDB_page *mp; + MDB_node *leaf; + int rc; + + if (mc->mc_flags & C_EOF) { + return MDB_NOTFOUND; + } + + mdb_cassert(mc, mc->mc_flags & C_INITIALIZED); + + mp = mc->mc_pg[mc->mc_top]; + + if (mc->mc_db->md_flags & MDB_DUPSORT) { + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + if (op == MDB_NEXT || op == MDB_NEXT_DUP) { + rc = mdb_cursor_next(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_NEXT); + if (op != MDB_NEXT || rc != MDB_NOTFOUND) { + if (rc == MDB_SUCCESS) + MDB_GET_KEY(leaf, key); + return rc; + } + } + } else { + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + if (op == MDB_NEXT_DUP) + return MDB_NOTFOUND; + } + } + + DPRINTF(("cursor_next: top page is %"Z"u in cursor %p", + mdb_dbg_pgno(mp), (void *) mc)); + if (mc->mc_flags & C_DEL) + goto skip; + + if (mc->mc_ki[mc->mc_top] + 1u >= NUMKEYS(mp)) { + DPUTS("=====> move to next sibling page"); + if ((rc = mdb_cursor_sibling(mc, 1)) != MDB_SUCCESS) { + mc->mc_flags |= C_EOF; + return rc; + } + mp = mc->mc_pg[mc->mc_top]; + DPRINTF(("next page is %"Z"u, key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top])); + } else + mc->mc_ki[mc->mc_top]++; + +skip: + DPRINTF(("==> cursor points to page %"Z"u with %u keys, key index %u", + mdb_dbg_pgno(mp), NUMKEYS(mp), mc->mc_ki[mc->mc_top])); + + if (IS_LEAF2(mp)) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); + return MDB_SUCCESS; + } + + mdb_cassert(mc, IS_LEAF(mp)); + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + mdb_xcursor_init1(mc, leaf); + } + if (data) { + if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS) + return rc; + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); + if (rc != MDB_SUCCESS) + return rc; + } + } + + MDB_GET_KEY(leaf, key); + return MDB_SUCCESS; +} + +/** Move the cursor to the previous data item. */ +static int +mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) +{ + MDB_page *mp; + MDB_node *leaf; + int rc; + + mdb_cassert(mc, mc->mc_flags & C_INITIALIZED); + + mp = mc->mc_pg[mc->mc_top]; + + if (mc->mc_db->md_flags & MDB_DUPSORT) { + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + if (op == MDB_PREV || op == MDB_PREV_DUP) { + rc = mdb_cursor_prev(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_PREV); + if (op != MDB_PREV || rc != MDB_NOTFOUND) { + if (rc == MDB_SUCCESS) { + MDB_GET_KEY(leaf, key); + mc->mc_flags &= ~C_EOF; + } + return rc; + } + } else { + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + if (op == MDB_PREV_DUP) + return MDB_NOTFOUND; + } + } + } + + DPRINTF(("cursor_prev: top page is %"Z"u in cursor %p", + mdb_dbg_pgno(mp), (void *) mc)); + + if (mc->mc_ki[mc->mc_top] == 0) { + DPUTS("=====> move to prev sibling page"); + if ((rc = mdb_cursor_sibling(mc, 0)) != MDB_SUCCESS) { + return rc; + } + mp = mc->mc_pg[mc->mc_top]; + mc->mc_ki[mc->mc_top] = NUMKEYS(mp) - 1; + DPRINTF(("prev page is %"Z"u, key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top])); + } else + mc->mc_ki[mc->mc_top]--; + + mc->mc_flags &= ~C_EOF; + + DPRINTF(("==> cursor points to page %"Z"u with %u keys, key index %u", + mdb_dbg_pgno(mp), NUMKEYS(mp), mc->mc_ki[mc->mc_top])); + + if (IS_LEAF2(mp)) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); + return MDB_SUCCESS; + } + + mdb_cassert(mc, IS_LEAF(mp)); + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + mdb_xcursor_init1(mc, leaf); + } + if (data) { + if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS) + return rc; + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + rc = mdb_cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL); + if (rc != MDB_SUCCESS) + return rc; + } + } + + MDB_GET_KEY(leaf, key); + return MDB_SUCCESS; +} + +/** Set the cursor on a specific data item. */ +static int +mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data, + MDB_cursor_op op, int *exactp) +{ + int rc; + MDB_page *mp; + MDB_node *leaf = NULL; + DKBUF; + + if (key->mv_size == 0) + return MDB_BAD_VALSIZE; + + if (mc->mc_xcursor) + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + + /* See if we're already on the right page */ + if (mc->mc_flags & C_INITIALIZED) { + MDB_val nodekey; + + mp = mc->mc_pg[mc->mc_top]; + if (!NUMKEYS(mp)) { + mc->mc_ki[mc->mc_top] = 0; + return MDB_NOTFOUND; + } + if (mp->mp_flags & P_LEAF2) { + nodekey.mv_size = mc->mc_db->md_pad; + nodekey.mv_data = LEAF2KEY(mp, 0, nodekey.mv_size); + } else { + leaf = NODEPTR(mp, 0); + MDB_GET_KEY2(leaf, nodekey); + } + rc = mc->mc_dbx->md_cmp(key, &nodekey); + if (rc == 0) { + /* Probably happens rarely, but first node on the page + * was the one we wanted. + */ + mc->mc_ki[mc->mc_top] = 0; + if (exactp) + *exactp = 1; + goto set1; + } + if (rc > 0) { + unsigned int i; + unsigned int nkeys = NUMKEYS(mp); + if (nkeys > 1) { + if (mp->mp_flags & P_LEAF2) { + nodekey.mv_data = LEAF2KEY(mp, + nkeys-1, nodekey.mv_size); + } else { + leaf = NODEPTR(mp, nkeys-1); + MDB_GET_KEY2(leaf, nodekey); + } + rc = mc->mc_dbx->md_cmp(key, &nodekey); + if (rc == 0) { + /* last node was the one we wanted */ + mc->mc_ki[mc->mc_top] = nkeys-1; + if (exactp) + *exactp = 1; + goto set1; + } + if (rc < 0) { + if (mc->mc_ki[mc->mc_top] < NUMKEYS(mp)) { + /* This is definitely the right page, skip search_page */ + if (mp->mp_flags & P_LEAF2) { + nodekey.mv_data = LEAF2KEY(mp, + mc->mc_ki[mc->mc_top], nodekey.mv_size); + } else { + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + MDB_GET_KEY2(leaf, nodekey); + } + rc = mc->mc_dbx->md_cmp(key, &nodekey); + if (rc == 0) { + /* current node was the one we wanted */ + if (exactp) + *exactp = 1; + goto set1; + } + } + rc = 0; + goto set2; + } + } + /* If any parents have right-sibs, search. + * Otherwise, there's nothing further. + */ + for (i=0; i<mc->mc_top; i++) + if (mc->mc_ki[i] < + NUMKEYS(mc->mc_pg[i])-1) + break; + if (i == mc->mc_top) { + /* There are no other pages */ + mc->mc_ki[mc->mc_top] = nkeys; + return MDB_NOTFOUND; + } + } + if (!mc->mc_top) { + /* There are no other pages */ + mc->mc_ki[mc->mc_top] = 0; + if (op == MDB_SET_RANGE && !exactp) { + rc = 0; + goto set1; + } else + return MDB_NOTFOUND; + } + } + + rc = mdb_page_search(mc, key, 0); + if (rc != MDB_SUCCESS) + return rc; + + mp = mc->mc_pg[mc->mc_top]; + mdb_cassert(mc, IS_LEAF(mp)); + +set2: + leaf = mdb_node_search(mc, key, exactp); + if (exactp != NULL && !*exactp) { + /* MDB_SET specified and not an exact match. */ + return MDB_NOTFOUND; + } + + if (leaf == NULL) { + DPUTS("===> inexact leaf not found, goto sibling"); + if ((rc = mdb_cursor_sibling(mc, 1)) != MDB_SUCCESS) + return rc; /* no entries matched */ + mp = mc->mc_pg[mc->mc_top]; + mdb_cassert(mc, IS_LEAF(mp)); + leaf = NODEPTR(mp, 0); + } + +set1: + mc->mc_flags |= C_INITIALIZED; + mc->mc_flags &= ~C_EOF; + + if (IS_LEAF2(mp)) { + if (op == MDB_SET_RANGE || op == MDB_SET_KEY) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); + } + return MDB_SUCCESS; + } + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + mdb_xcursor_init1(mc, leaf); + } + if (data) { + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + if (op == MDB_SET || op == MDB_SET_KEY || op == MDB_SET_RANGE) { + rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); + } else { + int ex2, *ex2p; + if (op == MDB_GET_BOTH) { + ex2p = &ex2; + ex2 = 0; + } else { + ex2p = NULL; + } + rc = mdb_cursor_set(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_SET_RANGE, ex2p); + if (rc != MDB_SUCCESS) + return rc; + } + } else if (op == MDB_GET_BOTH || op == MDB_GET_BOTH_RANGE) { + MDB_val d2; + if ((rc = mdb_node_read(mc->mc_txn, leaf, &d2)) != MDB_SUCCESS) + return rc; + rc = mc->mc_dbx->md_dcmp(data, &d2); + if (rc) { + if (op == MDB_GET_BOTH || rc > 0) + return MDB_NOTFOUND; + rc = 0; + *data = d2; + } + + } else { + if (mc->mc_xcursor) + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS) + return rc; + } + } + + /* The key already matches in all other cases */ + if (op == MDB_SET_RANGE || op == MDB_SET_KEY) + MDB_GET_KEY(leaf, key); + DPRINTF(("==> cursor placed on key [%s]", DKEY(key))); + + return rc; +} + +/** Move the cursor to the first item in the database. */ +static int +mdb_cursor_first(MDB_cursor *mc, MDB_val *key, MDB_val *data) +{ + int rc; + MDB_node *leaf; + + if (mc->mc_xcursor) + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + + if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) { + rc = mdb_page_search(mc, NULL, MDB_PS_FIRST); + if (rc != MDB_SUCCESS) + return rc; + } + mdb_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top])); + + leaf = NODEPTR(mc->mc_pg[mc->mc_top], 0); + mc->mc_flags |= C_INITIALIZED; + mc->mc_flags &= ~C_EOF; + + mc->mc_ki[mc->mc_top] = 0; + + if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mc->mc_pg[mc->mc_top], 0, key->mv_size); + return MDB_SUCCESS; + } + + if (data) { + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + mdb_xcursor_init1(mc, leaf); + rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); + if (rc) + return rc; + } else { + if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS) + return rc; + } + } + MDB_GET_KEY(leaf, key); + return MDB_SUCCESS; +} + +/** Move the cursor to the last item in the database. */ +static int +mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data) +{ + int rc; + MDB_node *leaf; + + if (mc->mc_xcursor) + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + + if (!(mc->mc_flags & C_EOF)) { + + if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) { + rc = mdb_page_search(mc, NULL, MDB_PS_LAST); + if (rc != MDB_SUCCESS) + return rc; + } + mdb_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top])); + + } + mc->mc_ki[mc->mc_top] = NUMKEYS(mc->mc_pg[mc->mc_top]) - 1; + mc->mc_flags |= C_INITIALIZED|C_EOF; + leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + + if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top], key->mv_size); + return MDB_SUCCESS; + } + + if (data) { + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + mdb_xcursor_init1(mc, leaf); + rc = mdb_cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL); + if (rc) + return rc; + } else { + if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS) + return rc; + } + } + + MDB_GET_KEY(leaf, key); + return MDB_SUCCESS; +} + +int +mdb_cursor_get(MDB_cursor *mc, MDB_val *key, MDB_val *data, + MDB_cursor_op op) +{ + int rc; + int exact = 0; + int (*mfunc)(MDB_cursor *mc, MDB_val *key, MDB_val *data); + + if (mc == NULL) + return EINVAL; + + if (mc->mc_txn->mt_flags & MDB_TXN_ERROR) + return MDB_BAD_TXN; + + switch (op) { + case MDB_GET_CURRENT: + if (!(mc->mc_flags & C_INITIALIZED)) { + rc = EINVAL; + } else { + MDB_page *mp = mc->mc_pg[mc->mc_top]; + int nkeys = NUMKEYS(mp); + if (!nkeys || mc->mc_ki[mc->mc_top] >= nkeys) { + mc->mc_ki[mc->mc_top] = nkeys; + rc = MDB_NOTFOUND; + break; + } + rc = MDB_SUCCESS; + if (IS_LEAF2(mp)) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); + } else { + MDB_node *leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + MDB_GET_KEY(leaf, key); + if (data) { + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + if (mc->mc_flags & C_DEL) + mdb_xcursor_init1(mc, leaf); + rc = mdb_cursor_get(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_GET_CURRENT); + } else { + rc = mdb_node_read(mc->mc_txn, leaf, data); + } + } + } + } + break; + case MDB_GET_BOTH: + case MDB_GET_BOTH_RANGE: + if (data == NULL) { + rc = EINVAL; + break; + } + if (mc->mc_xcursor == NULL) { + rc = MDB_INCOMPATIBLE; + break; + } + /* FALLTHRU */ + case MDB_SET: + case MDB_SET_KEY: + case MDB_SET_RANGE: + if (key == NULL) { + rc = EINVAL; + } else { + rc = mdb_cursor_set(mc, key, data, op, + op == MDB_SET_RANGE ? NULL : &exact); + } + break; + case MDB_GET_MULTIPLE: + if (data == NULL || !(mc->mc_flags & C_INITIALIZED)) { + rc = EINVAL; + break; + } + if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) { + rc = MDB_INCOMPATIBLE; + break; + } + rc = MDB_SUCCESS; + if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) || + (mc->mc_xcursor->mx_cursor.mc_flags & C_EOF)) + break; + goto fetchm; + case MDB_NEXT_MULTIPLE: + if (data == NULL) { + rc = EINVAL; + break; + } + if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) { + rc = MDB_INCOMPATIBLE; + break; + } + if (!(mc->mc_flags & C_INITIALIZED)) + rc = mdb_cursor_first(mc, key, data); + else + rc = mdb_cursor_next(mc, key, data, MDB_NEXT_DUP); + if (rc == MDB_SUCCESS) { + if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { + MDB_cursor *mx; +fetchm: + mx = &mc->mc_xcursor->mx_cursor; + data->mv_size = NUMKEYS(mx->mc_pg[mx->mc_top]) * + mx->mc_db->md_pad; + data->mv_data = METADATA(mx->mc_pg[mx->mc_top]); + mx->mc_ki[mx->mc_top] = NUMKEYS(mx->mc_pg[mx->mc_top])-1; + } else { + rc = MDB_NOTFOUND; + } + } + break; + case MDB_NEXT: + case MDB_NEXT_DUP: + case MDB_NEXT_NODUP: + if (!(mc->mc_flags & C_INITIALIZED)) + rc = mdb_cursor_first(mc, key, data); + else + rc = mdb_cursor_next(mc, key, data, op); + break; + case MDB_PREV: + case MDB_PREV_DUP: + case MDB_PREV_NODUP: + if (!(mc->mc_flags & C_INITIALIZED)) { + rc = mdb_cursor_last(mc, key, data); + if (rc) + break; + mc->mc_flags |= C_INITIALIZED; + mc->mc_ki[mc->mc_top]++; + } + rc = mdb_cursor_prev(mc, key, data, op); + break; + case MDB_FIRST: + rc = mdb_cursor_first(mc, key, data); + break; + case MDB_FIRST_DUP: + mfunc = mdb_cursor_first; + mmove: + if (data == NULL || !(mc->mc_flags & C_INITIALIZED)) { + rc = EINVAL; + break; + } + if (mc->mc_xcursor == NULL) { + rc = MDB_INCOMPATIBLE; + break; + } + { + MDB_node *leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { + MDB_GET_KEY(leaf, key); + rc = mdb_node_read(mc->mc_txn, leaf, data); + break; + } + } + if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) { + rc = EINVAL; + break; + } + rc = mfunc(&mc->mc_xcursor->mx_cursor, data, NULL); + break; + case MDB_LAST: + rc = mdb_cursor_last(mc, key, data); + break; + case MDB_LAST_DUP: + mfunc = mdb_cursor_last; + goto mmove; + default: + DPRINTF(("unhandled/unimplemented cursor operation %u", op)); + rc = EINVAL; + break; + } + + if (mc->mc_flags & C_DEL) + mc->mc_flags ^= C_DEL; + + return rc; +} + +/** Touch all the pages in the cursor stack. Set mc_top. + * Makes sure all the pages are writable, before attempting a write operation. + * @param[in] mc The cursor to operate on. + */ +static int +mdb_cursor_touch(MDB_cursor *mc) +{ + int rc = MDB_SUCCESS; + + if (mc->mc_dbi > MAIN_DBI && !(*mc->mc_dbflag & DB_DIRTY)) { + MDB_cursor mc2; + MDB_xcursor mcx; + if (TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbi)) + return MDB_BAD_DBI; + mdb_cursor_init(&mc2, mc->mc_txn, MAIN_DBI, &mcx); + rc = mdb_page_search(&mc2, &mc->mc_dbx->md_name, MDB_PS_MODIFY); + if (rc) + return rc; + *mc->mc_dbflag |= DB_DIRTY; + } + mc->mc_top = 0; + if (mc->mc_snum) { + do { + rc = mdb_page_touch(mc); + } while (!rc && ++(mc->mc_top) < mc->mc_snum); + mc->mc_top = mc->mc_snum-1; + } + return rc; +} + +/** Do not spill pages to disk if txn is getting full, may fail instead */ +#define MDB_NOSPILL 0x8000 + +int +mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, + unsigned int flags) +{ + enum { MDB_NO_ROOT = MDB_LAST_ERRCODE+10 }; /* internal code */ + MDB_env *env; + MDB_node *leaf = NULL; + MDB_page *fp, *mp; + uint16_t fp_flags; + MDB_val xdata, *rdata, dkey, olddata; + MDB_db dummy; + int do_sub = 0, insert_key, insert_data; + unsigned int mcount = 0, dcount = 0, nospill; + size_t nsize; + int rc, rc2; + unsigned int nflags; + DKBUF; + + if (mc == NULL || key == NULL) + return EINVAL; + + env = mc->mc_txn->mt_env; + + /* Check this first so counter will always be zero on any + * early failures. + */ + if (flags & MDB_MULTIPLE) { + dcount = data[1].mv_size; + data[1].mv_size = 0; + if (!F_ISSET(mc->mc_db->md_flags, MDB_DUPFIXED)) + return MDB_INCOMPATIBLE; + } + + nospill = flags & MDB_NOSPILL; + flags &= ~MDB_NOSPILL; + + if (mc->mc_txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_ERROR)) + return (mc->mc_txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN; + + if (key->mv_size-1 >= ENV_MAXKEY(env)) + return MDB_BAD_VALSIZE; + +#if SIZE_MAX > MAXDATASIZE + if (data->mv_size > ((mc->mc_db->md_flags & MDB_DUPSORT) ? ENV_MAXKEY(env) : MAXDATASIZE)) + return MDB_BAD_VALSIZE; +#else + if ((mc->mc_db->md_flags & MDB_DUPSORT) && data->mv_size > ENV_MAXKEY(env)) + return MDB_BAD_VALSIZE; +#endif + + DPRINTF(("==> put db %d key [%s], size %"Z"u, data size %"Z"u", + DDBI(mc), DKEY(key), key ? key->mv_size : 0, data->mv_size)); + + dkey.mv_size = 0; + + if (flags == MDB_CURRENT) { + if (!(mc->mc_flags & C_INITIALIZED)) + return EINVAL; + rc = MDB_SUCCESS; + } else if (mc->mc_db->md_root == P_INVALID) { + /* new database, cursor has nothing to point to */ + mc->mc_snum = 0; + mc->mc_top = 0; + mc->mc_flags &= ~C_INITIALIZED; + rc = MDB_NO_ROOT; + } else { + int exact = 0; + MDB_val d2; + if (flags & MDB_APPEND) { + MDB_val k2; + rc = mdb_cursor_last(mc, &k2, &d2); + if (rc == 0) { + rc = mc->mc_dbx->md_cmp(key, &k2); + if (rc > 0) { + rc = MDB_NOTFOUND; + mc->mc_ki[mc->mc_top]++; + } else { + /* new key is <= last key */ + rc = MDB_KEYEXIST; + } + } + } else { + rc = mdb_cursor_set(mc, key, &d2, MDB_SET, &exact); + } + if ((flags & MDB_NOOVERWRITE) && rc == 0) { + DPRINTF(("duplicate key [%s]", DKEY(key))); + *data = d2; + return MDB_KEYEXIST; + } + if (rc && rc != MDB_NOTFOUND) + return rc; + } + + if (mc->mc_flags & C_DEL) + mc->mc_flags ^= C_DEL; + + /* Cursor is positioned, check for room in the dirty list */ + if (!nospill) { + if (flags & MDB_MULTIPLE) { + rdata = &xdata; + xdata.mv_size = data->mv_size * dcount; + } else { + rdata = data; + } + if ((rc2 = mdb_page_spill(mc, key, rdata))) + return rc2; + } + + if (rc == MDB_NO_ROOT) { + MDB_page *np; + /* new database, write a root leaf page */ + DPUTS("allocating new root leaf page"); + if ((rc2 = mdb_page_new(mc, P_LEAF, 1, &np))) { + return rc2; + } + mdb_cursor_push(mc, np); + mc->mc_db->md_root = np->mp_pgno; + mc->mc_db->md_depth++; + *mc->mc_dbflag |= DB_DIRTY; + if ((mc->mc_db->md_flags & (MDB_DUPSORT|MDB_DUPFIXED)) + == MDB_DUPFIXED) + np->mp_flags |= P_LEAF2; + mc->mc_flags |= C_INITIALIZED; + } else { + /* make sure all cursor pages are writable */ + rc2 = mdb_cursor_touch(mc); + if (rc2) + return rc2; + } + + insert_key = insert_data = rc; + if (insert_key) { + /* The key does not exist */ + DPRINTF(("inserting key at index %i", mc->mc_ki[mc->mc_top])); + if ((mc->mc_db->md_flags & MDB_DUPSORT) && + LEAFSIZE(key, data) > env->me_nodemax) + { + /* Too big for a node, insert in sub-DB. Set up an empty + * "old sub-page" for prep_subDB to expand to a full page. + */ + fp_flags = P_LEAF|P_DIRTY; + fp = env->me_pbuf; + fp->mp_pad = data->mv_size; /* used if MDB_DUPFIXED */ + fp->mp_lower = fp->mp_upper = (PAGEHDRSZ-PAGEBASE); + olddata.mv_size = PAGEHDRSZ; + goto prep_subDB; + } + } else { + /* there's only a key anyway, so this is a no-op */ + if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { + char *ptr; + unsigned int ksize = mc->mc_db->md_pad; + if (key->mv_size != ksize) + return MDB_BAD_VALSIZE; + ptr = LEAF2KEY(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top], ksize); + memcpy(ptr, key->mv_data, ksize); +fix_parent: + /* if overwriting slot 0 of leaf, need to + * update branch key if there is a parent page + */ + if (mc->mc_top && !mc->mc_ki[mc->mc_top]) { + unsigned short top = mc->mc_top; + mc->mc_top--; + /* slot 0 is always an empty key, find real slot */ + while (mc->mc_top && !mc->mc_ki[mc->mc_top]) + mc->mc_top--; + if (mc->mc_ki[mc->mc_top]) + rc2 = mdb_update_key(mc, key); + else + rc2 = MDB_SUCCESS; + mc->mc_top = top; + if (rc2) + return rc2; + } + return MDB_SUCCESS; + } + +more: + leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + olddata.mv_size = NODEDSZ(leaf); + olddata.mv_data = NODEDATA(leaf); + + /* DB has dups? */ + if (F_ISSET(mc->mc_db->md_flags, MDB_DUPSORT)) { + /* Prepare (sub-)page/sub-DB to accept the new item, + * if needed. fp: old sub-page or a header faking + * it. mp: new (sub-)page. offset: growth in page + * size. xdata: node data with new page or DB. + */ + unsigned i, offset = 0; + mp = fp = xdata.mv_data = env->me_pbuf; + mp->mp_pgno = mc->mc_pg[mc->mc_top]->mp_pgno; + + /* Was a single item before, must convert now */ + if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { + /* Just overwrite the current item */ + if (flags == MDB_CURRENT) + goto current; + +#if UINT_MAX < SIZE_MAX + if (mc->mc_dbx->md_dcmp == mdb_cmp_int && olddata.mv_size == sizeof(size_t)) + mc->mc_dbx->md_dcmp = mdb_cmp_clong; +#endif + /* does data match? */ + if (!mc->mc_dbx->md_dcmp(data, &olddata)) { + if (flags & MDB_NODUPDATA) + return MDB_KEYEXIST; + /* overwrite it */ + goto current; + } + + /* Back up original data item */ + dkey.mv_size = olddata.mv_size; + dkey.mv_data = memcpy(fp+1, olddata.mv_data, olddata.mv_size); + + /* Make sub-page header for the dup items, with dummy body */ + fp->mp_flags = P_LEAF|P_DIRTY|P_SUBP; + fp->mp_lower = (PAGEHDRSZ-PAGEBASE); + xdata.mv_size = PAGEHDRSZ + dkey.mv_size + data->mv_size; + if (mc->mc_db->md_flags & MDB_DUPFIXED) { + fp->mp_flags |= P_LEAF2; + fp->mp_pad = data->mv_size; + xdata.mv_size += 2 * data->mv_size; /* leave space for 2 more */ + } else { + xdata.mv_size += 2 * (sizeof(indx_t) + NODESIZE) + + (dkey.mv_size & 1) + (data->mv_size & 1); + } + fp->mp_upper = xdata.mv_size - PAGEBASE; + olddata.mv_size = xdata.mv_size; /* pretend olddata is fp */ + } else if (leaf->mn_flags & F_SUBDATA) { + /* Data is on sub-DB, just store it */ + flags |= F_DUPDATA|F_SUBDATA; + goto put_sub; + } else { + /* Data is on sub-page */ + fp = olddata.mv_data; + switch (flags) { + default: + if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) { + offset = EVEN(NODESIZE + sizeof(indx_t) + + data->mv_size); + break; + } + offset = fp->mp_pad; + if (SIZELEFT(fp) < offset) { + offset *= 4; /* space for 4 more */ + break; + } + /* FALLTHRU: Big enough MDB_DUPFIXED sub-page */ + case MDB_CURRENT: + fp->mp_flags |= P_DIRTY; + COPY_PGNO(fp->mp_pgno, mp->mp_pgno); + mc->mc_xcursor->mx_cursor.mc_pg[0] = fp; + flags |= F_DUPDATA; + goto put_sub; + } + xdata.mv_size = olddata.mv_size + offset; + } + + fp_flags = fp->mp_flags; + if (NODESIZE + NODEKSZ(leaf) + xdata.mv_size > env->me_nodemax) { + /* Too big for a sub-page, convert to sub-DB */ + fp_flags &= ~P_SUBP; +prep_subDB: + if (mc->mc_db->md_flags & MDB_DUPFIXED) { + fp_flags |= P_LEAF2; + dummy.md_pad = fp->mp_pad; + dummy.md_flags = MDB_DUPFIXED; + if (mc->mc_db->md_flags & MDB_INTEGERDUP) + dummy.md_flags |= MDB_INTEGERKEY; + } else { + dummy.md_pad = 0; + dummy.md_flags = 0; + } + dummy.md_depth = 1; + dummy.md_branch_pages = 0; + dummy.md_leaf_pages = 1; + dummy.md_overflow_pages = 0; + dummy.md_entries = NUMKEYS(fp); + xdata.mv_size = sizeof(MDB_db); + xdata.mv_data = &dummy; + if ((rc = mdb_page_alloc(mc, 1, &mp))) + return rc; + offset = env->me_psize - olddata.mv_size; + flags |= F_DUPDATA|F_SUBDATA; + dummy.md_root = mp->mp_pgno; + } + if (mp != fp) { + mp->mp_flags = fp_flags | P_DIRTY; + mp->mp_pad = fp->mp_pad; + mp->mp_lower = fp->mp_lower; + mp->mp_upper = fp->mp_upper + offset; + if (fp_flags & P_LEAF2) { + memcpy(METADATA(mp), METADATA(fp), NUMKEYS(fp) * fp->mp_pad); + } else { + memcpy((char *)mp + mp->mp_upper + PAGEBASE, (char *)fp + fp->mp_upper + PAGEBASE, + olddata.mv_size - fp->mp_upper - PAGEBASE); + for (i=0; i<NUMKEYS(fp); i++) + mp->mp_ptrs[i] = fp->mp_ptrs[i] + offset; + } + } + + rdata = &xdata; + flags |= F_DUPDATA; + do_sub = 1; + if (!insert_key) + mdb_node_del(mc, 0); + goto new_sub; + } +current: + /* overflow page overwrites need special handling */ + if (F_ISSET(leaf->mn_flags, F_BIGDATA)) { + MDB_page *omp; + pgno_t pg; + int level, ovpages, dpages = OVPAGES(data->mv_size, env->me_psize); + + memcpy(&pg, olddata.mv_data, sizeof(pg)); + if ((rc2 = mdb_page_get(mc->mc_txn, pg, &omp, &level)) != 0) + return rc2; + ovpages = omp->mp_pages; + + /* Is the ov page large enough? */ + if (ovpages >= dpages) { + if (!(omp->mp_flags & P_DIRTY) && + (level || (env->me_flags & MDB_WRITEMAP))) + { + rc = mdb_page_unspill(mc->mc_txn, omp, &omp); + if (rc) + return rc; + level = 0; /* dirty in this txn or clean */ + } + /* Is it dirty? */ + if (omp->mp_flags & P_DIRTY) { + /* yes, overwrite it. Note in this case we don't + * bother to try shrinking the page if the new data + * is smaller than the overflow threshold. + */ + if (level > 1) { + /* It is writable only in a parent txn */ + size_t sz = (size_t) env->me_psize * ovpages, off; + MDB_page *np = mdb_page_malloc(mc->mc_txn, ovpages); + MDB_ID2 id2; + if (!np) + return ENOMEM; + id2.mid = pg; + id2.mptr = np; + rc2 = mdb_mid2l_insert(mc->mc_txn->mt_u.dirty_list, &id2); + mdb_cassert(mc, rc2 == 0); + if (!(flags & MDB_RESERVE)) { + /* Copy end of page, adjusting alignment so + * compiler may copy words instead of bytes. + */ + off = (PAGEHDRSZ + data->mv_size) & -sizeof(size_t); + memcpy((size_t *)((char *)np + off), + (size_t *)((char *)omp + off), sz - off); + sz = PAGEHDRSZ; + } + memcpy(np, omp, sz); /* Copy beginning of page */ + omp = np; + } + SETDSZ(leaf, data->mv_size); + if (F_ISSET(flags, MDB_RESERVE)) + data->mv_data = METADATA(omp); + else + memcpy(METADATA(omp), data->mv_data, data->mv_size); + return MDB_SUCCESS; + } + } + if ((rc2 = mdb_ovpage_free(mc, omp)) != MDB_SUCCESS) + return rc2; + } else if (data->mv_size == olddata.mv_size) { + /* same size, just replace it. Note that we could + * also reuse this node if the new data is smaller, + * but instead we opt to shrink the node in that case. + */ + if (F_ISSET(flags, MDB_RESERVE)) + data->mv_data = olddata.mv_data; + else if (!(mc->mc_flags & C_SUB)) + memcpy(olddata.mv_data, data->mv_data, data->mv_size); + else { + memcpy(NODEKEY(leaf), key->mv_data, key->mv_size); + goto fix_parent; + } + return MDB_SUCCESS; + } + mdb_node_del(mc, 0); + } + + rdata = data; + +new_sub: + nflags = flags & NODE_ADD_FLAGS; + nsize = IS_LEAF2(mc->mc_pg[mc->mc_top]) ? key->mv_size : mdb_leaf_size(env, key, rdata); + if (SIZELEFT(mc->mc_pg[mc->mc_top]) < nsize) { + if (( flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA ) + nflags &= ~MDB_APPEND; /* sub-page may need room to grow */ + if (!insert_key) + nflags |= MDB_SPLIT_REPLACE; + rc = mdb_page_split(mc, key, rdata, P_INVALID, nflags); + } else { + /* There is room already in this leaf page. */ + rc = mdb_node_add(mc, mc->mc_ki[mc->mc_top], key, rdata, 0, nflags); + if (rc == 0 && insert_key) { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2, *m3; + MDB_dbi dbi = mc->mc_dbi; + unsigned i = mc->mc_top; + MDB_page *mp = mc->mc_pg[i]; + + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (mc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3 == mc || m3->mc_snum < mc->mc_snum) continue; + if (m3->mc_pg[i] == mp && m3->mc_ki[i] >= mc->mc_ki[i]) { + m3->mc_ki[i]++; + } + } + } + } + + if (rc == MDB_SUCCESS) { + /* Now store the actual data in the child DB. Note that we're + * storing the user data in the keys field, so there are strict + * size limits on dupdata. The actual data fields of the child + * DB are all zero size. + */ + if (do_sub) { + int xflags; + size_t ecount; +put_sub: + xdata.mv_size = 0; + xdata.mv_data = ""; + leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if (flags & MDB_CURRENT) { + xflags = MDB_CURRENT|MDB_NOSPILL; + } else { + mdb_xcursor_init1(mc, leaf); + xflags = (flags & MDB_NODUPDATA) ? + MDB_NOOVERWRITE|MDB_NOSPILL : MDB_NOSPILL; + } + /* converted, write the original data first */ + if (dkey.mv_size) { + rc = mdb_cursor_put(&mc->mc_xcursor->mx_cursor, &dkey, &xdata, xflags); + if (rc) + goto bad_sub; + { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2; + unsigned i = mc->mc_top; + MDB_page *mp = mc->mc_pg[i]; + + for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { + if (m2 == mc || m2->mc_snum < mc->mc_snum) continue; + if (!(m2->mc_flags & C_INITIALIZED)) continue; + if (m2->mc_pg[i] == mp && m2->mc_ki[i] == mc->mc_ki[i]) { + mdb_xcursor_init1(m2, leaf); + } + } + } + /* we've done our job */ + dkey.mv_size = 0; + } + ecount = mc->mc_xcursor->mx_db.md_entries; + if (flags & MDB_APPENDDUP) + xflags |= MDB_APPEND; + rc = mdb_cursor_put(&mc->mc_xcursor->mx_cursor, data, &xdata, xflags); + if (flags & F_SUBDATA) { + void *db = NODEDATA(leaf); + memcpy(db, &mc->mc_xcursor->mx_db, sizeof(MDB_db)); + } + insert_data = mc->mc_xcursor->mx_db.md_entries - ecount; + } + /* Increment count unless we just replaced an existing item. */ + if (insert_data) + mc->mc_db->md_entries++; + if (insert_key) { + /* Invalidate txn if we created an empty sub-DB */ + if (rc) + goto bad_sub; + /* If we succeeded and the key didn't exist before, + * make sure the cursor is marked valid. + */ + mc->mc_flags |= C_INITIALIZED; + } + if (flags & MDB_MULTIPLE) { + if (!rc) { + mcount++; + /* let caller know how many succeeded, if any */ + data[1].mv_size = mcount; + if (mcount < dcount) { + data[0].mv_data = (char *)data[0].mv_data + data[0].mv_size; + insert_key = insert_data = 0; + goto more; + } + } + } + return rc; +bad_sub: + if (rc == MDB_KEYEXIST) /* should not happen, we deleted that item */ + rc = MDB_CORRUPTED; + } + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return rc; +} + +int +mdb_cursor_del(MDB_cursor *mc, unsigned int flags) +{ + MDB_node *leaf; + MDB_page *mp; + int rc; + + if (mc->mc_txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_ERROR)) + return (mc->mc_txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN; + + if (!(mc->mc_flags & C_INITIALIZED)) + return EINVAL; + + if (mc->mc_ki[mc->mc_top] >= NUMKEYS(mc->mc_pg[mc->mc_top])) + return MDB_NOTFOUND; + + if (!(flags & MDB_NOSPILL) && (rc = mdb_page_spill(mc, NULL, NULL))) + return rc; + + rc = mdb_cursor_touch(mc); + if (rc) + return rc; + + mp = mc->mc_pg[mc->mc_top]; + if (IS_LEAF2(mp)) + goto del_key; + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + if (flags & MDB_NODUPDATA) { + /* mdb_cursor_del0() will subtract the final entry */ + mc->mc_db->md_entries -= mc->mc_xcursor->mx_db.md_entries - 1; + } else { + if (!F_ISSET(leaf->mn_flags, F_SUBDATA)) { + mc->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); + } + rc = mdb_cursor_del(&mc->mc_xcursor->mx_cursor, MDB_NOSPILL); + if (rc) + return rc; + /* If sub-DB still has entries, we're done */ + if (mc->mc_xcursor->mx_db.md_entries) { + if (leaf->mn_flags & F_SUBDATA) { + /* update subDB info */ + void *db = NODEDATA(leaf); + memcpy(db, &mc->mc_xcursor->mx_db, sizeof(MDB_db)); + } else { + MDB_cursor *m2; + /* shrink fake page */ + mdb_node_shrink(mp, mc->mc_ki[mc->mc_top]); + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + mc->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); + /* fix other sub-DB cursors pointed at this fake page */ + for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { + if (m2 == mc || m2->mc_snum < mc->mc_snum) continue; + if (m2->mc_pg[mc->mc_top] == mp && + m2->mc_ki[mc->mc_top] == mc->mc_ki[mc->mc_top]) + m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); + } + } + mc->mc_db->md_entries--; + mc->mc_flags |= C_DEL; + return rc; + } + /* otherwise fall thru and delete the sub-DB */ + } + + if (leaf->mn_flags & F_SUBDATA) { + /* add all the child DB's pages to the free list */ + rc = mdb_drop0(&mc->mc_xcursor->mx_cursor, 0); + if (rc) + goto fail; + } + } + + /* add overflow pages to free list */ + if (F_ISSET(leaf->mn_flags, F_BIGDATA)) { + MDB_page *omp; + pgno_t pg; + + memcpy(&pg, NODEDATA(leaf), sizeof(pg)); + if ((rc = mdb_page_get(mc->mc_txn, pg, &omp, NULL)) || + (rc = mdb_ovpage_free(mc, omp))) + goto fail; + } + +del_key: + return mdb_cursor_del0(mc); + +fail: + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return rc; +} + +/** Allocate and initialize new pages for a database. + * @param[in] mc a cursor on the database being added to. + * @param[in] flags flags defining what type of page is being allocated. + * @param[in] num the number of pages to allocate. This is usually 1, + * unless allocating overflow pages for a large record. + * @param[out] mp Address of a page, or NULL on failure. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp) +{ + MDB_page *np; + int rc; + + if ((rc = mdb_page_alloc(mc, num, &np))) + return rc; + DPRINTF(("allocated new mpage %"Z"u, page size %u", + np->mp_pgno, mc->mc_txn->mt_env->me_psize)); + np->mp_flags = flags | P_DIRTY; + np->mp_lower = (PAGEHDRSZ-PAGEBASE); + np->mp_upper = mc->mc_txn->mt_env->me_psize - PAGEBASE; + + if (IS_BRANCH(np)) + mc->mc_db->md_branch_pages++; + else if (IS_LEAF(np)) + mc->mc_db->md_leaf_pages++; + else if (IS_OVERFLOW(np)) { + mc->mc_db->md_overflow_pages += num; + np->mp_pages = num; + } + *mp = np; + + return 0; +} + +/** Calculate the size of a leaf node. + * The size depends on the environment's page size; if a data item + * is too large it will be put onto an overflow page and the node + * size will only include the key and not the data. Sizes are always + * rounded up to an even number of bytes, to guarantee 2-byte alignment + * of the #MDB_node headers. + * @param[in] env The environment handle. + * @param[in] key The key for the node. + * @param[in] data The data for the node. + * @return The number of bytes needed to store the node. + */ +static size_t +mdb_leaf_size(MDB_env *env, MDB_val *key, MDB_val *data) +{ + size_t sz; + + sz = LEAFSIZE(key, data); + if (sz > env->me_nodemax) { + /* put on overflow page */ + sz -= data->mv_size - sizeof(pgno_t); + } + + return EVEN(sz + sizeof(indx_t)); +} + +/** Calculate the size of a branch node. + * The size should depend on the environment's page size but since + * we currently don't support spilling large keys onto overflow + * pages, it's simply the size of the #MDB_node header plus the + * size of the key. Sizes are always rounded up to an even number + * of bytes, to guarantee 2-byte alignment of the #MDB_node headers. + * @param[in] env The environment handle. + * @param[in] key The key for the node. + * @return The number of bytes needed to store the node. + */ +static size_t +mdb_branch_size(MDB_env *env, MDB_val *key) +{ + size_t sz; + + sz = INDXSIZE(key); + if (sz > env->me_nodemax) { + /* put on overflow page */ + /* not implemented */ + /* sz -= key->size - sizeof(pgno_t); */ + } + + return sz + sizeof(indx_t); +} + +/** Add a node to the page pointed to by the cursor. + * @param[in] mc The cursor for this operation. + * @param[in] indx The index on the page where the new node should be added. + * @param[in] key The key for the new node. + * @param[in] data The data for the new node, if any. + * @param[in] pgno The page number, if adding a branch node. + * @param[in] flags Flags for the node. + * @return 0 on success, non-zero on failure. Possible errors are: + * <ul> + * <li>ENOMEM - failed to allocate overflow pages for the node. + * <li>MDB_PAGE_FULL - there is insufficient room in the page. This error + * should never happen since all callers already calculate the + * page's free space before calling this function. + * </ul> + */ +static int +mdb_node_add(MDB_cursor *mc, indx_t indx, + MDB_val *key, MDB_val *data, pgno_t pgno, unsigned int flags) +{ + unsigned int i; + size_t node_size = NODESIZE; + ssize_t room; + indx_t ofs; + MDB_node *node; + MDB_page *mp = mc->mc_pg[mc->mc_top]; + MDB_page *ofp = NULL; /* overflow page */ + DKBUF; + + mdb_cassert(mc, mp->mp_upper >= mp->mp_lower); + + DPRINTF(("add to %s %spage %"Z"u index %i, data size %"Z"u key size %"Z"u [%s]", + IS_LEAF(mp) ? "leaf" : "branch", + IS_SUBP(mp) ? "sub-" : "", + mdb_dbg_pgno(mp), indx, data ? data->mv_size : 0, + key ? key->mv_size : 0, key ? DKEY(key) : "null")); + + if (IS_LEAF2(mp)) { + /* Move higher keys up one slot. */ + int ksize = mc->mc_db->md_pad, dif; + char *ptr = LEAF2KEY(mp, indx, ksize); + dif = NUMKEYS(mp) - indx; + if (dif > 0) + memmove(ptr+ksize, ptr, dif*ksize); + /* insert new key */ + memcpy(ptr, key->mv_data, ksize); + + /* Just using these for counting */ + mp->mp_lower += sizeof(indx_t); + mp->mp_upper -= ksize - sizeof(indx_t); + return MDB_SUCCESS; + } + + room = (ssize_t)SIZELEFT(mp) - (ssize_t)sizeof(indx_t); + if (key != NULL) + node_size += key->mv_size; + if (IS_LEAF(mp)) { + mdb_cassert(mc, data); + if (F_ISSET(flags, F_BIGDATA)) { + /* Data already on overflow page. */ + node_size += sizeof(pgno_t); + } else if (node_size + data->mv_size > mc->mc_txn->mt_env->me_nodemax) { + int ovpages = OVPAGES(data->mv_size, mc->mc_txn->mt_env->me_psize); + int rc; + /* Put data on overflow page. */ + DPRINTF(("data size is %"Z"u, node would be %"Z"u, put data on overflow page", + data->mv_size, node_size+data->mv_size)); + node_size = EVEN(node_size + sizeof(pgno_t)); + if ((ssize_t)node_size > room) + goto full; + if ((rc = mdb_page_new(mc, P_OVERFLOW, ovpages, &ofp))) + return rc; + DPRINTF(("allocated overflow page %"Z"u", ofp->mp_pgno)); + flags |= F_BIGDATA; + goto update; + } else { + node_size += data->mv_size; + } + } + node_size = EVEN(node_size); + if ((ssize_t)node_size > room) + goto full; + +update: + /* Move higher pointers up one slot. */ + for (i = NUMKEYS(mp); i > indx; i--) + mp->mp_ptrs[i] = mp->mp_ptrs[i - 1]; + + /* Adjust free space offsets. */ + ofs = mp->mp_upper - node_size; + mdb_cassert(mc, ofs >= mp->mp_lower + sizeof(indx_t)); + mp->mp_ptrs[indx] = ofs; + mp->mp_upper = ofs; + mp->mp_lower += sizeof(indx_t); + + /* Write the node data. */ + node = NODEPTR(mp, indx); + node->mn_ksize = (key == NULL) ? 0 : key->mv_size; + node->mn_flags = flags; + if (IS_LEAF(mp)) + SETDSZ(node,data->mv_size); + else + SETPGNO(node,pgno); + + if (key) + memcpy(NODEKEY(node), key->mv_data, key->mv_size); + + if (IS_LEAF(mp)) { + mdb_cassert(mc, key); + if (ofp == NULL) { + if (F_ISSET(flags, F_BIGDATA)) + memcpy(node->mn_data + key->mv_size, data->mv_data, + sizeof(pgno_t)); + else if (F_ISSET(flags, MDB_RESERVE)) + data->mv_data = node->mn_data + key->mv_size; + else + memcpy(node->mn_data + key->mv_size, data->mv_data, + data->mv_size); + } else { + memcpy(node->mn_data + key->mv_size, &ofp->mp_pgno, + sizeof(pgno_t)); + if (F_ISSET(flags, MDB_RESERVE)) + data->mv_data = METADATA(ofp); + else + memcpy(METADATA(ofp), data->mv_data, data->mv_size); + } + } + + return MDB_SUCCESS; + +full: + DPRINTF(("not enough room in page %"Z"u, got %u ptrs", + mdb_dbg_pgno(mp), NUMKEYS(mp))); + DPRINTF(("upper-lower = %u - %u = %"Z"d", mp->mp_upper,mp->mp_lower,room)); + DPRINTF(("node size = %"Z"u", node_size)); + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return MDB_PAGE_FULL; +} + +/** Delete the specified node from a page. + * @param[in] mc Cursor pointing to the node to delete. + * @param[in] ksize The size of a node. Only used if the page is + * part of a #MDB_DUPFIXED database. + */ +static void +mdb_node_del(MDB_cursor *mc, int ksize) +{ + MDB_page *mp = mc->mc_pg[mc->mc_top]; + indx_t indx = mc->mc_ki[mc->mc_top]; + unsigned int sz; + indx_t i, j, numkeys, ptr; + MDB_node *node; + char *base; + + DPRINTF(("delete node %u on %s page %"Z"u", indx, + IS_LEAF(mp) ? "leaf" : "branch", mdb_dbg_pgno(mp))); + numkeys = NUMKEYS(mp); + mdb_cassert(mc, indx < numkeys); + + if (IS_LEAF2(mp)) { + int x = numkeys - 1 - indx; + base = LEAF2KEY(mp, indx, ksize); + if (x) + memmove(base, base + ksize, x * ksize); + mp->mp_lower -= sizeof(indx_t); + mp->mp_upper += ksize - sizeof(indx_t); + return; + } + + node = NODEPTR(mp, indx); + sz = NODESIZE + node->mn_ksize; + if (IS_LEAF(mp)) { + if (F_ISSET(node->mn_flags, F_BIGDATA)) + sz += sizeof(pgno_t); + else + sz += NODEDSZ(node); + } + sz = EVEN(sz); + + ptr = mp->mp_ptrs[indx]; + for (i = j = 0; i < numkeys; i++) { + if (i != indx) { + mp->mp_ptrs[j] = mp->mp_ptrs[i]; + if (mp->mp_ptrs[i] < ptr) + mp->mp_ptrs[j] += sz; + j++; + } + } + + base = (char *)mp + mp->mp_upper + PAGEBASE; + memmove(base + sz, base, ptr - mp->mp_upper); + + mp->mp_lower -= sizeof(indx_t); + mp->mp_upper += sz; +} + +/** Compact the main page after deleting a node on a subpage. + * @param[in] mp The main page to operate on. + * @param[in] indx The index of the subpage on the main page. + */ +static void +mdb_node_shrink(MDB_page *mp, indx_t indx) +{ + MDB_node *node; + MDB_page *sp, *xp; + char *base; + int nsize, delta; + indx_t i, numkeys, ptr; + + node = NODEPTR(mp, indx); + sp = (MDB_page *)NODEDATA(node); + delta = SIZELEFT(sp); + xp = (MDB_page *)((char *)sp + delta); + + /* shift subpage upward */ + if (IS_LEAF2(sp)) { + nsize = NUMKEYS(sp) * sp->mp_pad; + if (nsize & 1) + return; /* do not make the node uneven-sized */ + memmove(METADATA(xp), METADATA(sp), nsize); + } else { + int i; + numkeys = NUMKEYS(sp); + for (i=numkeys-1; i>=0; i--) + xp->mp_ptrs[i] = sp->mp_ptrs[i] - delta; + } + xp->mp_upper = sp->mp_lower; + xp->mp_lower = sp->mp_lower; + xp->mp_flags = sp->mp_flags; + xp->mp_pad = sp->mp_pad; + COPY_PGNO(xp->mp_pgno, mp->mp_pgno); + + nsize = NODEDSZ(node) - delta; + SETDSZ(node, nsize); + + /* shift lower nodes upward */ + ptr = mp->mp_ptrs[indx]; + numkeys = NUMKEYS(mp); + for (i = 0; i < numkeys; i++) { + if (mp->mp_ptrs[i] <= ptr) + mp->mp_ptrs[i] += delta; + } + + base = (char *)mp + mp->mp_upper + PAGEBASE; + memmove(base + delta, base, ptr - mp->mp_upper + NODESIZE + NODEKSZ(node)); + mp->mp_upper += delta; +} + +/** Initial setup of a sorted-dups cursor. + * Sorted duplicates are implemented as a sub-database for the given key. + * The duplicate data items are actually keys of the sub-database. + * Operations on the duplicate data items are performed using a sub-cursor + * initialized when the sub-database is first accessed. This function does + * the preliminary setup of the sub-cursor, filling in the fields that + * depend only on the parent DB. + * @param[in] mc The main cursor whose sorted-dups cursor is to be initialized. + */ +static void +mdb_xcursor_init0(MDB_cursor *mc) +{ + MDB_xcursor *mx = mc->mc_xcursor; + + mx->mx_cursor.mc_xcursor = NULL; + mx->mx_cursor.mc_txn = mc->mc_txn; + mx->mx_cursor.mc_db = &mx->mx_db; + mx->mx_cursor.mc_dbx = &mx->mx_dbx; + mx->mx_cursor.mc_dbi = mc->mc_dbi; + mx->mx_cursor.mc_dbflag = &mx->mx_dbflag; + mx->mx_cursor.mc_snum = 0; + mx->mx_cursor.mc_top = 0; + mx->mx_cursor.mc_flags = C_SUB; + mx->mx_dbx.md_name.mv_size = 0; + mx->mx_dbx.md_name.mv_data = NULL; + mx->mx_dbx.md_cmp = mc->mc_dbx->md_dcmp; + mx->mx_dbx.md_dcmp = NULL; + mx->mx_dbx.md_rel = mc->mc_dbx->md_rel; +} + +/** Final setup of a sorted-dups cursor. + * Sets up the fields that depend on the data from the main cursor. + * @param[in] mc The main cursor whose sorted-dups cursor is to be initialized. + * @param[in] node The data containing the #MDB_db record for the + * sorted-dup database. + */ +static void +mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node) +{ + MDB_xcursor *mx = mc->mc_xcursor; + + if (node->mn_flags & F_SUBDATA) { + memcpy(&mx->mx_db, NODEDATA(node), sizeof(MDB_db)); + mx->mx_cursor.mc_pg[0] = 0; + mx->mx_cursor.mc_snum = 0; + mx->mx_cursor.mc_top = 0; + mx->mx_cursor.mc_flags = C_SUB; + } else { + MDB_page *fp = NODEDATA(node); + mx->mx_db.md_pad = mc->mc_pg[mc->mc_top]->mp_pad; + mx->mx_db.md_flags = 0; + mx->mx_db.md_depth = 1; + mx->mx_db.md_branch_pages = 0; + mx->mx_db.md_leaf_pages = 1; + mx->mx_db.md_overflow_pages = 0; + mx->mx_db.md_entries = NUMKEYS(fp); + COPY_PGNO(mx->mx_db.md_root, fp->mp_pgno); + mx->mx_cursor.mc_snum = 1; + mx->mx_cursor.mc_top = 0; + mx->mx_cursor.mc_flags = C_INITIALIZED|C_SUB; + mx->mx_cursor.mc_pg[0] = fp; + mx->mx_cursor.mc_ki[0] = 0; + if (mc->mc_db->md_flags & MDB_DUPFIXED) { + mx->mx_db.md_flags = MDB_DUPFIXED; + mx->mx_db.md_pad = fp->mp_pad; + if (mc->mc_db->md_flags & MDB_INTEGERDUP) + mx->mx_db.md_flags |= MDB_INTEGERKEY; + } + } + DPRINTF(("Sub-db -%u root page %"Z"u", mx->mx_cursor.mc_dbi, + mx->mx_db.md_root)); + mx->mx_dbflag = DB_VALID|DB_DIRTY; /* DB_DIRTY guides mdb_cursor_touch */ +#if UINT_MAX < SIZE_MAX + if (mx->mx_dbx.md_cmp == mdb_cmp_int && mx->mx_db.md_pad == sizeof(size_t)) + mx->mx_dbx.md_cmp = mdb_cmp_clong; +#endif +} + +/** Initialize a cursor for a given transaction and database. */ +static void +mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx) +{ + mc->mc_next = NULL; + mc->mc_backup = NULL; + mc->mc_dbi = dbi; + mc->mc_txn = txn; + mc->mc_db = &txn->mt_dbs[dbi]; + mc->mc_dbx = &txn->mt_dbxs[dbi]; + mc->mc_dbflag = &txn->mt_dbflags[dbi]; + mc->mc_snum = 0; + mc->mc_top = 0; + mc->mc_pg[0] = 0; + mc->mc_flags = 0; + if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) { + mdb_tassert(txn, mx != NULL); + mc->mc_xcursor = mx; + mdb_xcursor_init0(mc); + } else { + mc->mc_xcursor = NULL; + } + if (*mc->mc_dbflag & DB_STALE) { + mdb_page_search(mc, NULL, MDB_PS_ROOTONLY); + } +} + +int +mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **ret) +{ + MDB_cursor *mc; + size_t size = sizeof(MDB_cursor); + + if (!ret || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + + if (txn->mt_flags & MDB_TXN_ERROR) + return MDB_BAD_TXN; + + /* Allow read access to the freelist */ + if (!dbi && !F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) + return EINVAL; + + if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) + size += sizeof(MDB_xcursor); + + if ((mc = malloc(size)) != NULL) { + mdb_cursor_init(mc, txn, dbi, (MDB_xcursor *)(mc + 1)); + if (txn->mt_cursors) { + mc->mc_next = txn->mt_cursors[dbi]; + txn->mt_cursors[dbi] = mc; + mc->mc_flags |= C_UNTRACK; + } + } else { + return ENOMEM; + } + + *ret = mc; + + return MDB_SUCCESS; +} + +int +mdb_cursor_renew(MDB_txn *txn, MDB_cursor *mc) +{ + if (!mc || !TXN_DBI_EXIST(txn, mc->mc_dbi)) + return EINVAL; + + if ((mc->mc_flags & C_UNTRACK) || txn->mt_cursors) + return EINVAL; + + if (txn->mt_flags & MDB_TXN_ERROR) + return MDB_BAD_TXN; + + mdb_cursor_init(mc, txn, mc->mc_dbi, mc->mc_xcursor); + return MDB_SUCCESS; +} + +/* Return the count of duplicate data items for the current key */ +int +mdb_cursor_count(MDB_cursor *mc, size_t *countp) +{ + MDB_node *leaf; + + if (mc == NULL || countp == NULL) + return EINVAL; + + if (mc->mc_xcursor == NULL) + return MDB_INCOMPATIBLE; + + if (mc->mc_txn->mt_flags & MDB_TXN_ERROR) + return MDB_BAD_TXN; + + if (!(mc->mc_flags & C_INITIALIZED)) + return EINVAL; + + if (!mc->mc_snum || (mc->mc_flags & C_EOF)) + return MDB_NOTFOUND; + + leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { + *countp = 1; + } else { + if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) + return EINVAL; + + *countp = mc->mc_xcursor->mx_db.md_entries; + } + return MDB_SUCCESS; +} + +void +mdb_cursor_close(MDB_cursor *mc) +{ + if (mc && !mc->mc_backup) { + /* remove from txn, if tracked */ + if ((mc->mc_flags & C_UNTRACK) && mc->mc_txn->mt_cursors) { + MDB_cursor **prev = &mc->mc_txn->mt_cursors[mc->mc_dbi]; + while (*prev && *prev != mc) prev = &(*prev)->mc_next; + if (*prev == mc) + *prev = mc->mc_next; + } + free(mc); + } +} + +MDB_txn * +mdb_cursor_txn(MDB_cursor *mc) +{ + if (!mc) return NULL; + return mc->mc_txn; +} + +MDB_dbi +mdb_cursor_dbi(MDB_cursor *mc) +{ + return mc->mc_dbi; +} + +/** Replace the key for a branch node with a new key. + * @param[in] mc Cursor pointing to the node to operate on. + * @param[in] key The new key to use. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_update_key(MDB_cursor *mc, MDB_val *key) +{ + MDB_page *mp; + MDB_node *node; + char *base; + size_t len; + int delta, ksize, oksize; + indx_t ptr, i, numkeys, indx; + DKBUF; + + indx = mc->mc_ki[mc->mc_top]; + mp = mc->mc_pg[mc->mc_top]; + node = NODEPTR(mp, indx); + ptr = mp->mp_ptrs[indx]; +#if MDB_DEBUG + { + MDB_val k2; + char kbuf2[DKBUF_MAXKEYSIZE*2+1]; + k2.mv_data = NODEKEY(node); + k2.mv_size = node->mn_ksize; + DPRINTF(("update key %u (ofs %u) [%s] to [%s] on page %"Z"u", + indx, ptr, + mdb_dkey(&k2, kbuf2), + DKEY(key), + mp->mp_pgno)); + } +#endif + + /* Sizes must be 2-byte aligned. */ + ksize = EVEN(key->mv_size); + oksize = EVEN(node->mn_ksize); + delta = ksize - oksize; + + /* Shift node contents if EVEN(key length) changed. */ + if (delta) { + if (delta > 0 && SIZELEFT(mp) < delta) { + pgno_t pgno; + /* not enough space left, do a delete and split */ + DPRINTF(("Not enough room, delta = %d, splitting...", delta)); + pgno = NODEPGNO(node); + mdb_node_del(mc, 0); + return mdb_page_split(mc, key, NULL, pgno, MDB_SPLIT_REPLACE); + } + + numkeys = NUMKEYS(mp); + for (i = 0; i < numkeys; i++) { + if (mp->mp_ptrs[i] <= ptr) + mp->mp_ptrs[i] -= delta; + } + + base = (char *)mp + mp->mp_upper + PAGEBASE; + len = ptr - mp->mp_upper + NODESIZE; + memmove(base - delta, base, len); + mp->mp_upper -= delta; + + node = NODEPTR(mp, indx); + } + + /* But even if no shift was needed, update ksize */ + if (node->mn_ksize != key->mv_size) + node->mn_ksize = key->mv_size; + + if (key->mv_size) + memcpy(NODEKEY(node), key->mv_data, key->mv_size); + + return MDB_SUCCESS; +} + +static void +mdb_cursor_copy(const MDB_cursor *csrc, MDB_cursor *cdst); + +/** Move a node from csrc to cdst. + */ +static int +mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst) +{ + MDB_node *srcnode; + MDB_val key, data; + pgno_t srcpg; + MDB_cursor mn; + int rc; + unsigned short flags; + + DKBUF; + + /* Mark src and dst as dirty. */ + if ((rc = mdb_page_touch(csrc)) || + (rc = mdb_page_touch(cdst))) + return rc; + + if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { + key.mv_size = csrc->mc_db->md_pad; + key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], csrc->mc_ki[csrc->mc_top], key.mv_size); + data.mv_size = 0; + data.mv_data = NULL; + srcpg = 0; + flags = 0; + } else { + srcnode = NODEPTR(csrc->mc_pg[csrc->mc_top], csrc->mc_ki[csrc->mc_top]); + mdb_cassert(csrc, !((size_t)srcnode & 1)); + srcpg = NODEPGNO(srcnode); + flags = srcnode->mn_flags; + if (csrc->mc_ki[csrc->mc_top] == 0 && IS_BRANCH(csrc->mc_pg[csrc->mc_top])) { + unsigned int snum = csrc->mc_snum; + MDB_node *s2; + /* must find the lowest key below src */ + rc = mdb_page_search_lowest(csrc); + if (rc) + return rc; + if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { + key.mv_size = csrc->mc_db->md_pad; + key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], 0, key.mv_size); + } else { + s2 = NODEPTR(csrc->mc_pg[csrc->mc_top], 0); + key.mv_size = NODEKSZ(s2); + key.mv_data = NODEKEY(s2); + } + csrc->mc_snum = snum--; + csrc->mc_top = snum; + } else { + key.mv_size = NODEKSZ(srcnode); + key.mv_data = NODEKEY(srcnode); + } + data.mv_size = NODEDSZ(srcnode); + data.mv_data = NODEDATA(srcnode); + } + if (IS_BRANCH(cdst->mc_pg[cdst->mc_top]) && cdst->mc_ki[cdst->mc_top] == 0) { + unsigned int snum = cdst->mc_snum; + MDB_node *s2; + MDB_val bkey; + /* must find the lowest key below dst */ + mdb_cursor_copy(cdst, &mn); + rc = mdb_page_search_lowest(&mn); + if (rc) + return rc; + if (IS_LEAF2(mn.mc_pg[mn.mc_top])) { + bkey.mv_size = mn.mc_db->md_pad; + bkey.mv_data = LEAF2KEY(mn.mc_pg[mn.mc_top], 0, bkey.mv_size); + } else { + s2 = NODEPTR(mn.mc_pg[mn.mc_top], 0); + bkey.mv_size = NODEKSZ(s2); + bkey.mv_data = NODEKEY(s2); + } + mn.mc_snum = snum--; + mn.mc_top = snum; + mn.mc_ki[snum] = 0; + rc = mdb_update_key(&mn, &bkey); + if (rc) + return rc; + } + + DPRINTF(("moving %s node %u [%s] on page %"Z"u to node %u on page %"Z"u", + IS_LEAF(csrc->mc_pg[csrc->mc_top]) ? "leaf" : "branch", + csrc->mc_ki[csrc->mc_top], + DKEY(&key), + csrc->mc_pg[csrc->mc_top]->mp_pgno, + cdst->mc_ki[cdst->mc_top], cdst->mc_pg[cdst->mc_top]->mp_pgno)); + + /* Add the node to the destination page. + */ + rc = mdb_node_add(cdst, cdst->mc_ki[cdst->mc_top], &key, &data, srcpg, flags); + if (rc != MDB_SUCCESS) + return rc; + + /* Delete the node from the source page. + */ + mdb_node_del(csrc, key.mv_size); + + { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2, *m3; + MDB_dbi dbi = csrc->mc_dbi; + MDB_page *mp = csrc->mc_pg[csrc->mc_top]; + + for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (csrc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3 == csrc) continue; + if (m3->mc_pg[csrc->mc_top] == mp && m3->mc_ki[csrc->mc_top] == + csrc->mc_ki[csrc->mc_top]) { + m3->mc_pg[csrc->mc_top] = cdst->mc_pg[cdst->mc_top]; + m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top]; + } + } + } + + /* Update the parent separators. + */ + if (csrc->mc_ki[csrc->mc_top] == 0) { + if (csrc->mc_ki[csrc->mc_top-1] != 0) { + if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { + key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], 0, key.mv_size); + } else { + srcnode = NODEPTR(csrc->mc_pg[csrc->mc_top], 0); + key.mv_size = NODEKSZ(srcnode); + key.mv_data = NODEKEY(srcnode); + } + DPRINTF(("update separator for source page %"Z"u to [%s]", + csrc->mc_pg[csrc->mc_top]->mp_pgno, DKEY(&key))); + mdb_cursor_copy(csrc, &mn); + mn.mc_snum--; + mn.mc_top--; + if ((rc = mdb_update_key(&mn, &key)) != MDB_SUCCESS) + return rc; + } + if (IS_BRANCH(csrc->mc_pg[csrc->mc_top])) { + MDB_val nullkey; + indx_t ix = csrc->mc_ki[csrc->mc_top]; + nullkey.mv_size = 0; + csrc->mc_ki[csrc->mc_top] = 0; + rc = mdb_update_key(csrc, &nullkey); + csrc->mc_ki[csrc->mc_top] = ix; + mdb_cassert(csrc, rc == MDB_SUCCESS); + } + } + + if (cdst->mc_ki[cdst->mc_top] == 0) { + if (cdst->mc_ki[cdst->mc_top-1] != 0) { + if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { + key.mv_data = LEAF2KEY(cdst->mc_pg[cdst->mc_top], 0, key.mv_size); + } else { + srcnode = NODEPTR(cdst->mc_pg[cdst->mc_top], 0); + key.mv_size = NODEKSZ(srcnode); + key.mv_data = NODEKEY(srcnode); + } + DPRINTF(("update separator for destination page %"Z"u to [%s]", + cdst->mc_pg[cdst->mc_top]->mp_pgno, DKEY(&key))); + mdb_cursor_copy(cdst, &mn); + mn.mc_snum--; + mn.mc_top--; + if ((rc = mdb_update_key(&mn, &key)) != MDB_SUCCESS) + return rc; + } + if (IS_BRANCH(cdst->mc_pg[cdst->mc_top])) { + MDB_val nullkey; + indx_t ix = cdst->mc_ki[cdst->mc_top]; + nullkey.mv_size = 0; + cdst->mc_ki[cdst->mc_top] = 0; + rc = mdb_update_key(cdst, &nullkey); + cdst->mc_ki[cdst->mc_top] = ix; + mdb_cassert(csrc, rc == MDB_SUCCESS); + } + } + + return MDB_SUCCESS; +} + +/** Merge one page into another. + * The nodes from the page pointed to by \b csrc will + * be copied to the page pointed to by \b cdst and then + * the \b csrc page will be freed. + * @param[in] csrc Cursor pointing to the source page. + * @param[in] cdst Cursor pointing to the destination page. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst) +{ + MDB_page *psrc, *pdst; + MDB_node *srcnode; + MDB_val key, data; + unsigned nkeys; + int rc; + indx_t i, j; + + psrc = csrc->mc_pg[csrc->mc_top]; + pdst = cdst->mc_pg[cdst->mc_top]; + + DPRINTF(("merging page %"Z"u into %"Z"u", psrc->mp_pgno, pdst->mp_pgno)); + + mdb_cassert(csrc, csrc->mc_snum > 1); /* can't merge root page */ + mdb_cassert(csrc, cdst->mc_snum > 1); + + /* Mark dst as dirty. */ + if ((rc = mdb_page_touch(cdst))) + return rc; + + /* Move all nodes from src to dst. + */ + j = nkeys = NUMKEYS(pdst); + if (IS_LEAF2(psrc)) { + key.mv_size = csrc->mc_db->md_pad; + key.mv_data = METADATA(psrc); + for (i = 0; i < NUMKEYS(psrc); i++, j++) { + rc = mdb_node_add(cdst, j, &key, NULL, 0, 0); + if (rc != MDB_SUCCESS) + return rc; + key.mv_data = (char *)key.mv_data + key.mv_size; + } + } else { + for (i = 0; i < NUMKEYS(psrc); i++, j++) { + srcnode = NODEPTR(psrc, i); + if (i == 0 && IS_BRANCH(psrc)) { + MDB_cursor mn; + MDB_node *s2; + mdb_cursor_copy(csrc, &mn); + /* must find the lowest key below src */ + rc = mdb_page_search_lowest(&mn); + if (rc) + return rc; + if (IS_LEAF2(mn.mc_pg[mn.mc_top])) { + key.mv_size = mn.mc_db->md_pad; + key.mv_data = LEAF2KEY(mn.mc_pg[mn.mc_top], 0, key.mv_size); + } else { + s2 = NODEPTR(mn.mc_pg[mn.mc_top], 0); + key.mv_size = NODEKSZ(s2); + key.mv_data = NODEKEY(s2); + } + } else { + key.mv_size = srcnode->mn_ksize; + key.mv_data = NODEKEY(srcnode); + } + + data.mv_size = NODEDSZ(srcnode); + data.mv_data = NODEDATA(srcnode); + rc = mdb_node_add(cdst, j, &key, &data, NODEPGNO(srcnode), srcnode->mn_flags); + if (rc != MDB_SUCCESS) + return rc; + } + } + + DPRINTF(("dst page %"Z"u now has %u keys (%.1f%% filled)", + pdst->mp_pgno, NUMKEYS(pdst), + (float)PAGEFILL(cdst->mc_txn->mt_env, pdst) / 10)); + + /* Unlink the src page from parent and add to free list. + */ + csrc->mc_top--; + mdb_node_del(csrc, 0); + if (csrc->mc_ki[csrc->mc_top] == 0) { + key.mv_size = 0; + rc = mdb_update_key(csrc, &key); + if (rc) { + csrc->mc_top++; + return rc; + } + } + csrc->mc_top++; + + psrc = csrc->mc_pg[csrc->mc_top]; + /* If not operating on FreeDB, allow this page to be reused + * in this txn. Otherwise just add to free list. + */ + rc = mdb_page_loose(csrc, psrc); + if (rc) + return rc; + if (IS_LEAF(psrc)) + csrc->mc_db->md_leaf_pages--; + else + csrc->mc_db->md_branch_pages--; + { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2, *m3; + MDB_dbi dbi = csrc->mc_dbi; + + for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (csrc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3 == csrc) continue; + if (m3->mc_snum < csrc->mc_snum) continue; + if (m3->mc_pg[csrc->mc_top] == psrc) { + m3->mc_pg[csrc->mc_top] = pdst; + m3->mc_ki[csrc->mc_top] += nkeys; + } + } + } + { + unsigned int snum = cdst->mc_snum; + uint16_t depth = cdst->mc_db->md_depth; + mdb_cursor_pop(cdst); + rc = mdb_rebalance(cdst); + /* Did the tree shrink? */ + if (depth > cdst->mc_db->md_depth) + snum--; + cdst->mc_snum = snum; + cdst->mc_top = snum-1; + } + return rc; +} + +/** Copy the contents of a cursor. + * @param[in] csrc The cursor to copy from. + * @param[out] cdst The cursor to copy to. + */ +static void +mdb_cursor_copy(const MDB_cursor *csrc, MDB_cursor *cdst) +{ + unsigned int i; + + cdst->mc_txn = csrc->mc_txn; + cdst->mc_dbi = csrc->mc_dbi; + cdst->mc_db = csrc->mc_db; + cdst->mc_dbx = csrc->mc_dbx; + cdst->mc_snum = csrc->mc_snum; + cdst->mc_top = csrc->mc_top; + cdst->mc_flags = csrc->mc_flags; + + for (i=0; i<csrc->mc_snum; i++) { + cdst->mc_pg[i] = csrc->mc_pg[i]; + cdst->mc_ki[i] = csrc->mc_ki[i]; + } +} + +/** Rebalance the tree after a delete operation. + * @param[in] mc Cursor pointing to the page where rebalancing + * should begin. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_rebalance(MDB_cursor *mc) +{ + MDB_node *node; + int rc; + unsigned int ptop, minkeys; + MDB_cursor mn; + indx_t oldki; + + minkeys = 1 + (IS_BRANCH(mc->mc_pg[mc->mc_top])); + DPRINTF(("rebalancing %s page %"Z"u (has %u keys, %.1f%% full)", + IS_LEAF(mc->mc_pg[mc->mc_top]) ? "leaf" : "branch", + mdb_dbg_pgno(mc->mc_pg[mc->mc_top]), NUMKEYS(mc->mc_pg[mc->mc_top]), + (float)PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) / 10)); + + if (PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) >= FILL_THRESHOLD && + NUMKEYS(mc->mc_pg[mc->mc_top]) >= minkeys) { + DPRINTF(("no need to rebalance page %"Z"u, above fill threshold", + mdb_dbg_pgno(mc->mc_pg[mc->mc_top]))); + return MDB_SUCCESS; + } + + if (mc->mc_snum < 2) { + MDB_page *mp = mc->mc_pg[0]; + if (IS_SUBP(mp)) { + DPUTS("Can't rebalance a subpage, ignoring"); + return MDB_SUCCESS; + } + if (NUMKEYS(mp) == 0) { + DPUTS("tree is completely empty"); + mc->mc_db->md_root = P_INVALID; + mc->mc_db->md_depth = 0; + mc->mc_db->md_leaf_pages = 0; + rc = mdb_midl_append(&mc->mc_txn->mt_free_pgs, mp->mp_pgno); + if (rc) + return rc; + /* Adjust cursors pointing to mp */ + mc->mc_snum = 0; + mc->mc_top = 0; + mc->mc_flags &= ~C_INITIALIZED; + { + MDB_cursor *m2, *m3; + MDB_dbi dbi = mc->mc_dbi; + + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (mc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3->mc_snum < mc->mc_snum) continue; + if (m3->mc_pg[0] == mp) { + m3->mc_snum = 0; + m3->mc_top = 0; + m3->mc_flags &= ~C_INITIALIZED; + } + } + } + } else if (IS_BRANCH(mp) && NUMKEYS(mp) == 1) { + int i; + DPUTS("collapsing root page!"); + rc = mdb_midl_append(&mc->mc_txn->mt_free_pgs, mp->mp_pgno); + if (rc) + return rc; + mc->mc_db->md_root = NODEPGNO(NODEPTR(mp, 0)); + rc = mdb_page_get(mc->mc_txn,mc->mc_db->md_root,&mc->mc_pg[0],NULL); + if (rc) + return rc; + mc->mc_db->md_depth--; + mc->mc_db->md_branch_pages--; + mc->mc_ki[0] = mc->mc_ki[1]; + for (i = 1; i<mc->mc_db->md_depth; i++) { + mc->mc_pg[i] = mc->mc_pg[i+1]; + mc->mc_ki[i] = mc->mc_ki[i+1]; + } + { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2, *m3; + MDB_dbi dbi = mc->mc_dbi; + + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (mc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3 == mc || m3->mc_snum < mc->mc_snum) continue; + if (m3->mc_pg[0] == mp) { + m3->mc_snum--; + m3->mc_top--; + for (i=0; i<m3->mc_snum; i++) { + m3->mc_pg[i] = m3->mc_pg[i+1]; + m3->mc_ki[i] = m3->mc_ki[i+1]; + } + } + } + } + } else + DPUTS("root page doesn't need rebalancing"); + return MDB_SUCCESS; + } + + /* The parent (branch page) must have at least 2 pointers, + * otherwise the tree is invalid. + */ + ptop = mc->mc_top-1; + mdb_cassert(mc, NUMKEYS(mc->mc_pg[ptop]) > 1); + + /* Leaf page fill factor is below the threshold. + * Try to move keys from left or right neighbor, or + * merge with a neighbor page. + */ + + /* Find neighbors. + */ + mdb_cursor_copy(mc, &mn); + mn.mc_xcursor = NULL; + + oldki = mc->mc_ki[mc->mc_top]; + if (mc->mc_ki[ptop] == 0) { + /* We're the leftmost leaf in our parent. + */ + DPUTS("reading right neighbor"); + mn.mc_ki[ptop]++; + node = NODEPTR(mc->mc_pg[ptop], mn.mc_ki[ptop]); + rc = mdb_page_get(mc->mc_txn,NODEPGNO(node),&mn.mc_pg[mn.mc_top],NULL); + if (rc) + return rc; + mn.mc_ki[mn.mc_top] = 0; + mc->mc_ki[mc->mc_top] = NUMKEYS(mc->mc_pg[mc->mc_top]); + } else { + /* There is at least one neighbor to the left. + */ + DPUTS("reading left neighbor"); + mn.mc_ki[ptop]--; + node = NODEPTR(mc->mc_pg[ptop], mn.mc_ki[ptop]); + rc = mdb_page_get(mc->mc_txn,NODEPGNO(node),&mn.mc_pg[mn.mc_top],NULL); + if (rc) + return rc; + mn.mc_ki[mn.mc_top] = NUMKEYS(mn.mc_pg[mn.mc_top]) - 1; + mc->mc_ki[mc->mc_top] = 0; + } + + DPRINTF(("found neighbor page %"Z"u (%u keys, %.1f%% full)", + mn.mc_pg[mn.mc_top]->mp_pgno, NUMKEYS(mn.mc_pg[mn.mc_top]), + (float)PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) / 10)); + + /* If the neighbor page is above threshold and has enough keys, + * move one key from it. Otherwise we should try to merge them. + * (A branch page must never have less than 2 keys.) + */ + minkeys = 1 + (IS_BRANCH(mn.mc_pg[mn.mc_top])); + if (PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) >= FILL_THRESHOLD && NUMKEYS(mn.mc_pg[mn.mc_top]) > minkeys) { + rc = mdb_node_move(&mn, mc); + if (mc->mc_ki[ptop]) { + oldki++; + } + } else { + if (mc->mc_ki[ptop] == 0) { + rc = mdb_page_merge(&mn, mc); + } else { + oldki += NUMKEYS(mn.mc_pg[mn.mc_top]); + mn.mc_ki[mn.mc_top] += mc->mc_ki[mn.mc_top] + 1; + rc = mdb_page_merge(mc, &mn); + mdb_cursor_copy(&mn, mc); + } + mc->mc_flags &= ~C_EOF; + } + mc->mc_ki[mc->mc_top] = oldki; + return rc; +} + +/** Complete a delete operation started by #mdb_cursor_del(). */ +static int +mdb_cursor_del0(MDB_cursor *mc) +{ + int rc; + MDB_page *mp; + indx_t ki; + unsigned int nkeys; + + ki = mc->mc_ki[mc->mc_top]; + mdb_node_del(mc, mc->mc_db->md_pad); + mc->mc_db->md_entries--; + rc = mdb_rebalance(mc); + + if (rc == MDB_SUCCESS) { + MDB_cursor *m2, *m3; + MDB_dbi dbi = mc->mc_dbi; + + mp = mc->mc_pg[mc->mc_top]; + nkeys = NUMKEYS(mp); + + /* if mc points past last node in page, find next sibling */ + if (mc->mc_ki[mc->mc_top] >= nkeys) { + rc = mdb_cursor_sibling(mc, 1); + if (rc == MDB_NOTFOUND) { + mc->mc_flags |= C_EOF; + rc = MDB_SUCCESS; + } + } + + /* Adjust other cursors pointing to mp */ + for (m2 = mc->mc_txn->mt_cursors[dbi]; !rc && m2; m2=m2->mc_next) { + m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; + if (! (m2->mc_flags & m3->mc_flags & C_INITIALIZED)) + continue; + if (m3 == mc || m3->mc_snum < mc->mc_snum) + continue; + if (m3->mc_pg[mc->mc_top] == mp) { + if (m3->mc_ki[mc->mc_top] >= ki) { + m3->mc_flags |= C_DEL; + if (m3->mc_ki[mc->mc_top] > ki) + m3->mc_ki[mc->mc_top]--; + else if (mc->mc_db->md_flags & MDB_DUPSORT) + m3->mc_xcursor->mx_cursor.mc_flags |= C_EOF; + } + if (m3->mc_ki[mc->mc_top] >= nkeys) { + rc = mdb_cursor_sibling(m3, 1); + if (rc == MDB_NOTFOUND) { + m3->mc_flags |= C_EOF; + rc = MDB_SUCCESS; + } + } + } + } + mc->mc_flags |= C_DEL; + } + + if (rc) + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return rc; +} + +int +mdb_del(MDB_txn *txn, MDB_dbi dbi, + MDB_val *key, MDB_val *data) +{ + if (!key || dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + + if (txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_ERROR)) + return (txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN; + + if (!F_ISSET(txn->mt_dbs[dbi].md_flags, MDB_DUPSORT)) { + /* must ignore any data */ + data = NULL; + } + + return mdb_del0(txn, dbi, key, data, 0); +} + +static int +mdb_del0(MDB_txn *txn, MDB_dbi dbi, + MDB_val *key, MDB_val *data, unsigned flags) +{ + MDB_cursor mc; + MDB_xcursor mx; + MDB_cursor_op op; + MDB_val rdata, *xdata; + int rc, exact = 0; + DKBUF; + + DPRINTF(("====> delete db %u key [%s]", dbi, DKEY(key))); + + mdb_cursor_init(&mc, txn, dbi, &mx); + + if (data) { + op = MDB_GET_BOTH; + rdata = *data; + xdata = &rdata; + } else { + op = MDB_SET; + xdata = NULL; + flags |= MDB_NODUPDATA; + } + rc = mdb_cursor_set(&mc, key, xdata, op, &exact); + if (rc == 0) { + /* let mdb_page_split know about this cursor if needed: + * delete will trigger a rebalance; if it needs to move + * a node from one page to another, it will have to + * update the parent's separator key(s). If the new sepkey + * is larger than the current one, the parent page may + * run out of space, triggering a split. We need this + * cursor to be consistent until the end of the rebalance. + */ + mc.mc_flags |= C_UNTRACK; + mc.mc_next = txn->mt_cursors[dbi]; + txn->mt_cursors[dbi] = &mc; + rc = mdb_cursor_del(&mc, flags); + txn->mt_cursors[dbi] = mc.mc_next; + } + return rc; +} + +/** Split a page and insert a new node. + * @param[in,out] mc Cursor pointing to the page and desired insertion index. + * The cursor will be updated to point to the actual page and index where + * the node got inserted after the split. + * @param[in] newkey The key for the newly inserted node. + * @param[in] newdata The data for the newly inserted node. + * @param[in] newpgno The page number, if the new node is a branch node. + * @param[in] nflags The #NODE_ADD_FLAGS for the new node. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno, + unsigned int nflags) +{ + unsigned int flags; + int rc = MDB_SUCCESS, new_root = 0, did_split = 0; + indx_t newindx; + pgno_t pgno = 0; + int i, j, split_indx, nkeys, pmax; + MDB_env *env = mc->mc_txn->mt_env; + MDB_node *node; + MDB_val sepkey, rkey, xdata, *rdata = &xdata; + MDB_page *copy = NULL; + MDB_page *mp, *rp, *pp; + int ptop; + MDB_cursor mn; + DKBUF; + + mp = mc->mc_pg[mc->mc_top]; + newindx = mc->mc_ki[mc->mc_top]; + nkeys = NUMKEYS(mp); + + DPRINTF(("-----> splitting %s page %"Z"u and adding [%s] at index %i/%i", + IS_LEAF(mp) ? "leaf" : "branch", mp->mp_pgno, + DKEY(newkey), mc->mc_ki[mc->mc_top], nkeys)); + + /* Create a right sibling. */ + if ((rc = mdb_page_new(mc, mp->mp_flags, 1, &rp))) + return rc; + DPRINTF(("new right sibling: page %"Z"u", rp->mp_pgno)); + + if (mc->mc_snum < 2) { + if ((rc = mdb_page_new(mc, P_BRANCH, 1, &pp))) + goto done; + /* shift current top to make room for new parent */ + mc->mc_pg[1] = mc->mc_pg[0]; + mc->mc_ki[1] = mc->mc_ki[0]; + mc->mc_pg[0] = pp; + mc->mc_ki[0] = 0; + mc->mc_db->md_root = pp->mp_pgno; + DPRINTF(("root split! new root = %"Z"u", pp->mp_pgno)); + mc->mc_db->md_depth++; + new_root = 1; + + /* Add left (implicit) pointer. */ + if ((rc = mdb_node_add(mc, 0, NULL, NULL, mp->mp_pgno, 0)) != MDB_SUCCESS) { + /* undo the pre-push */ + mc->mc_pg[0] = mc->mc_pg[1]; + mc->mc_ki[0] = mc->mc_ki[1]; + mc->mc_db->md_root = mp->mp_pgno; + mc->mc_db->md_depth--; + goto done; + } + mc->mc_snum = 2; + mc->mc_top = 1; + ptop = 0; + } else { + ptop = mc->mc_top-1; + DPRINTF(("parent branch page is %"Z"u", mc->mc_pg[ptop]->mp_pgno)); + } + + mc->mc_flags |= C_SPLITTING; + mdb_cursor_copy(mc, &mn); + mn.mc_pg[mn.mc_top] = rp; + mn.mc_ki[ptop] = mc->mc_ki[ptop]+1; + + if (nflags & MDB_APPEND) { + mn.mc_ki[mn.mc_top] = 0; + sepkey = *newkey; + split_indx = newindx; + nkeys = 0; + } else { + + split_indx = (nkeys+1) / 2; + + if (IS_LEAF2(rp)) { + char *split, *ins; + int x; + unsigned int lsize, rsize, ksize; + /* Move half of the keys to the right sibling */ + x = mc->mc_ki[mc->mc_top] - split_indx; + ksize = mc->mc_db->md_pad; + split = LEAF2KEY(mp, split_indx, ksize); + rsize = (nkeys - split_indx) * ksize; + lsize = (nkeys - split_indx) * sizeof(indx_t); + mp->mp_lower -= lsize; + rp->mp_lower += lsize; + mp->mp_upper += rsize - lsize; + rp->mp_upper -= rsize - lsize; + sepkey.mv_size = ksize; + if (newindx == split_indx) { + sepkey.mv_data = newkey->mv_data; + } else { + sepkey.mv_data = split; + } + if (x<0) { + ins = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], ksize); + memcpy(rp->mp_ptrs, split, rsize); + sepkey.mv_data = rp->mp_ptrs; + memmove(ins+ksize, ins, (split_indx - mc->mc_ki[mc->mc_top]) * ksize); + memcpy(ins, newkey->mv_data, ksize); + mp->mp_lower += sizeof(indx_t); + mp->mp_upper -= ksize - sizeof(indx_t); + } else { + if (x) + memcpy(rp->mp_ptrs, split, x * ksize); + ins = LEAF2KEY(rp, x, ksize); + memcpy(ins, newkey->mv_data, ksize); + memcpy(ins+ksize, split + x * ksize, rsize - x * ksize); + rp->mp_lower += sizeof(indx_t); + rp->mp_upper -= ksize - sizeof(indx_t); + mc->mc_ki[mc->mc_top] = x; + mc->mc_pg[mc->mc_top] = rp; + } + } else { + int psize, nsize, k; + /* Maximum free space in an empty page */ + pmax = env->me_psize - PAGEHDRSZ; + if (IS_LEAF(mp)) + nsize = mdb_leaf_size(env, newkey, newdata); + else + nsize = mdb_branch_size(env, newkey); + nsize = EVEN(nsize); + + /* grab a page to hold a temporary copy */ + copy = mdb_page_malloc(mc->mc_txn, 1); + if (copy == NULL) { + rc = ENOMEM; + goto done; + } + copy->mp_pgno = mp->mp_pgno; + copy->mp_flags = mp->mp_flags; + copy->mp_lower = (PAGEHDRSZ-PAGEBASE); + copy->mp_upper = env->me_psize - PAGEBASE; + + /* prepare to insert */ + for (i=0, j=0; i<nkeys; i++) { + if (i == newindx) { + copy->mp_ptrs[j++] = 0; + } + copy->mp_ptrs[j++] = mp->mp_ptrs[i]; + } + + /* When items are relatively large the split point needs + * to be checked, because being off-by-one will make the + * difference between success or failure in mdb_node_add. + * + * It's also relevant if a page happens to be laid out + * such that one half of its nodes are all "small" and + * the other half of its nodes are "large." If the new + * item is also "large" and falls on the half with + * "large" nodes, it also may not fit. + * + * As a final tweak, if the new item goes on the last + * spot on the page (and thus, onto the new page), bias + * the split so the new page is emptier than the old page. + * This yields better packing during sequential inserts. + */ + if (nkeys < 20 || nsize > pmax/16 || newindx >= nkeys) { + /* Find split point */ + psize = 0; + if (newindx <= split_indx || newindx >= nkeys) { + i = 0; j = 1; + k = newindx >= nkeys ? nkeys : split_indx+2; + } else { + i = nkeys; j = -1; + k = split_indx-1; + } + for (; i!=k; i+=j) { + if (i == newindx) { + psize += nsize; + node = NULL; + } else { + node = (MDB_node *)((char *)mp + copy->mp_ptrs[i] + PAGEBASE); + psize += NODESIZE + NODEKSZ(node) + sizeof(indx_t); + if (IS_LEAF(mp)) { + if (F_ISSET(node->mn_flags, F_BIGDATA)) + psize += sizeof(pgno_t); + else + psize += NODEDSZ(node); + } + psize = EVEN(psize); + } + if (psize > pmax || i == k-j) { + split_indx = i + (j<0); + break; + } + } + } + if (split_indx == newindx) { + sepkey.mv_size = newkey->mv_size; + sepkey.mv_data = newkey->mv_data; + } else { + node = (MDB_node *)((char *)mp + copy->mp_ptrs[split_indx] + PAGEBASE); + sepkey.mv_size = node->mn_ksize; + sepkey.mv_data = NODEKEY(node); + } + } + } + + DPRINTF(("separator is %d [%s]", split_indx, DKEY(&sepkey))); + + /* Copy separator key to the parent. + */ + if (SIZELEFT(mn.mc_pg[ptop]) < mdb_branch_size(env, &sepkey)) { + mn.mc_snum--; + mn.mc_top--; + did_split = 1; + rc = mdb_page_split(&mn, &sepkey, NULL, rp->mp_pgno, 0); + if (rc) + goto done; + + /* root split? */ + if (mn.mc_snum == mc->mc_snum) { + mc->mc_pg[mc->mc_snum] = mc->mc_pg[mc->mc_top]; + mc->mc_ki[mc->mc_snum] = mc->mc_ki[mc->mc_top]; + mc->mc_pg[mc->mc_top] = mc->mc_pg[ptop]; + mc->mc_ki[mc->mc_top] = mc->mc_ki[ptop]; + mc->mc_snum++; + mc->mc_top++; + ptop++; + } + /* Right page might now have changed parent. + * Check if left page also changed parent. + */ + if (mn.mc_pg[ptop] != mc->mc_pg[ptop] && + mc->mc_ki[ptop] >= NUMKEYS(mc->mc_pg[ptop])) { + for (i=0; i<ptop; i++) { + mc->mc_pg[i] = mn.mc_pg[i]; + mc->mc_ki[i] = mn.mc_ki[i]; + } + mc->mc_pg[ptop] = mn.mc_pg[ptop]; + if (mn.mc_ki[ptop]) { + mc->mc_ki[ptop] = mn.mc_ki[ptop] - 1; + } else { + /* find right page's left sibling */ + mc->mc_ki[ptop] = mn.mc_ki[ptop]; + mdb_cursor_sibling(mc, 0); + } + } + } else { + mn.mc_top--; + rc = mdb_node_add(&mn, mn.mc_ki[ptop], &sepkey, NULL, rp->mp_pgno, 0); + mn.mc_top++; + } + mc->mc_flags ^= C_SPLITTING; + if (rc != MDB_SUCCESS) { + goto done; + } + if (nflags & MDB_APPEND) { + mc->mc_pg[mc->mc_top] = rp; + mc->mc_ki[mc->mc_top] = 0; + rc = mdb_node_add(mc, 0, newkey, newdata, newpgno, nflags); + if (rc) + goto done; + for (i=0; i<mc->mc_top; i++) + mc->mc_ki[i] = mn.mc_ki[i]; + } else if (!IS_LEAF2(mp)) { + /* Move nodes */ + mc->mc_pg[mc->mc_top] = rp; + i = split_indx; + j = 0; + do { + if (i == newindx) { + rkey.mv_data = newkey->mv_data; + rkey.mv_size = newkey->mv_size; + if (IS_LEAF(mp)) { + rdata = newdata; + } else + pgno = newpgno; + flags = nflags; + /* Update index for the new key. */ + mc->mc_ki[mc->mc_top] = j; + } else { + node = (MDB_node *)((char *)mp + copy->mp_ptrs[i] + PAGEBASE); + rkey.mv_data = NODEKEY(node); + rkey.mv_size = node->mn_ksize; + if (IS_LEAF(mp)) { + xdata.mv_data = NODEDATA(node); + xdata.mv_size = NODEDSZ(node); + rdata = &xdata; + } else + pgno = NODEPGNO(node); + flags = node->mn_flags; + } + + if (!IS_LEAF(mp) && j == 0) { + /* First branch index doesn't need key data. */ + rkey.mv_size = 0; + } + + rc = mdb_node_add(mc, j, &rkey, rdata, pgno, flags); + if (rc) + goto done; + if (i == nkeys) { + i = 0; + j = 0; + mc->mc_pg[mc->mc_top] = copy; + } else { + i++; + j++; + } + } while (i != split_indx); + + nkeys = NUMKEYS(copy); + for (i=0; i<nkeys; i++) + mp->mp_ptrs[i] = copy->mp_ptrs[i]; + mp->mp_lower = copy->mp_lower; + mp->mp_upper = copy->mp_upper; + memcpy(NODEPTR(mp, nkeys-1), NODEPTR(copy, nkeys-1), + env->me_psize - copy->mp_upper - PAGEBASE); + + /* reset back to original page */ + if (newindx < split_indx) { + mc->mc_pg[mc->mc_top] = mp; + if (nflags & MDB_RESERVE) { + node = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + if (!(node->mn_flags & F_BIGDATA)) + newdata->mv_data = NODEDATA(node); + } + } else { + mc->mc_pg[mc->mc_top] = rp; + mc->mc_ki[ptop]++; + /* Make sure mc_ki is still valid. + */ + if (mn.mc_pg[ptop] != mc->mc_pg[ptop] && + mc->mc_ki[ptop] >= NUMKEYS(mc->mc_pg[ptop])) { + for (i=0; i<=ptop; i++) { + mc->mc_pg[i] = mn.mc_pg[i]; + mc->mc_ki[i] = mn.mc_ki[i]; + } + } + } + } + + { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2, *m3; + MDB_dbi dbi = mc->mc_dbi; + int fixup = NUMKEYS(mp); + + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (mc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3 == mc) + continue; + if (!(m2->mc_flags & m3->mc_flags & C_INITIALIZED)) + continue; + if (m3->mc_flags & C_SPLITTING) + continue; + if (new_root) { + int k; + /* root split */ + for (k=m3->mc_top; k>=0; k--) { + m3->mc_ki[k+1] = m3->mc_ki[k]; + m3->mc_pg[k+1] = m3->mc_pg[k]; + } + if (m3->mc_ki[0] >= split_indx) { + m3->mc_ki[0] = 1; + } else { + m3->mc_ki[0] = 0; + } + m3->mc_pg[0] = mc->mc_pg[0]; + m3->mc_snum++; + m3->mc_top++; + } + if (m3->mc_top >= mc->mc_top && m3->mc_pg[mc->mc_top] == mp) { + if (m3->mc_ki[mc->mc_top] >= newindx && !(nflags & MDB_SPLIT_REPLACE)) + m3->mc_ki[mc->mc_top]++; + if (m3->mc_ki[mc->mc_top] >= fixup) { + m3->mc_pg[mc->mc_top] = rp; + m3->mc_ki[mc->mc_top] -= fixup; + m3->mc_ki[ptop] = mn.mc_ki[ptop]; + } + } else if (!did_split && m3->mc_top >= ptop && m3->mc_pg[ptop] == mc->mc_pg[ptop] && + m3->mc_ki[ptop] >= mc->mc_ki[ptop]) { + m3->mc_ki[ptop]++; + } + } + } + DPRINTF(("mp left: %d, rp left: %d", SIZELEFT(mp), SIZELEFT(rp))); + +done: + if (copy) /* tmp page */ + mdb_page_free(env, copy); + if (rc) + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return rc; +} + +int +mdb_put(MDB_txn *txn, MDB_dbi dbi, + MDB_val *key, MDB_val *data, unsigned int flags) +{ + MDB_cursor mc; + MDB_xcursor mx; + + if (!key || !data || dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + + if ((flags & (MDB_NOOVERWRITE|MDB_NODUPDATA|MDB_RESERVE|MDB_APPEND|MDB_APPENDDUP)) != flags) + return EINVAL; + + mdb_cursor_init(&mc, txn, dbi, &mx); + return mdb_cursor_put(&mc, key, data, flags); +} + +#ifndef MDB_WBUF +#define MDB_WBUF (1024*1024) +#endif + + /** State needed for a compacting copy. */ +typedef struct mdb_copy { + pthread_mutex_t mc_mutex; + pthread_cond_t mc_cond; + char *mc_wbuf[2]; + char *mc_over[2]; + MDB_env *mc_env; + MDB_txn *mc_txn; + int mc_wlen[2]; + int mc_olen[2]; + pgno_t mc_next_pgno; + HANDLE mc_fd; + int mc_status; + volatile int mc_new; + int mc_toggle; + +} mdb_copy; + + /** Dedicated writer thread for compacting copy. */ +static THREAD_RET ESECT +mdb_env_copythr(void *arg) +{ + mdb_copy *my = arg; + char *ptr; + int toggle = 0, wsize, rc; +#ifdef _WIN32 + DWORD len; +#define DO_WRITE(rc, fd, ptr, w2, len) rc = WriteFile(fd, ptr, w2, &len, NULL) +#else + int len; +#define DO_WRITE(rc, fd, ptr, w2, len) len = write(fd, ptr, w2); rc = (len >= 0) +#endif + + pthread_mutex_lock(&my->mc_mutex); + my->mc_new = 0; + pthread_cond_signal(&my->mc_cond); + for(;;) { + while (!my->mc_new) + pthread_cond_wait(&my->mc_cond, &my->mc_mutex); + if (my->mc_new < 0) { + my->mc_new = 0; + break; + } + my->mc_new = 0; + wsize = my->mc_wlen[toggle]; + ptr = my->mc_wbuf[toggle]; +again: + while (wsize > 0) { + DO_WRITE(rc, my->mc_fd, ptr, wsize, len); + if (!rc) { + rc = ErrCode(); + break; + } else if (len > 0) { + rc = MDB_SUCCESS; + ptr += len; + wsize -= len; + continue; + } else { + rc = EIO; + break; + } + } + if (rc) { + my->mc_status = rc; + break; + } + /* If there's an overflow page tail, write it too */ + if (my->mc_olen[toggle]) { + wsize = my->mc_olen[toggle]; + ptr = my->mc_over[toggle]; + my->mc_olen[toggle] = 0; + goto again; + } + my->mc_wlen[toggle] = 0; + toggle ^= 1; + pthread_cond_signal(&my->mc_cond); + } + pthread_cond_signal(&my->mc_cond); + pthread_mutex_unlock(&my->mc_mutex); + return (THREAD_RET)0; +#undef DO_WRITE +} + + /** Tell the writer thread there's a buffer ready to write */ +static int ESECT +mdb_env_cthr_toggle(mdb_copy *my, int st) +{ + int toggle = my->mc_toggle ^ 1; + pthread_mutex_lock(&my->mc_mutex); + if (my->mc_status) { + pthread_mutex_unlock(&my->mc_mutex); + return my->mc_status; + } + while (my->mc_new == 1) + pthread_cond_wait(&my->mc_cond, &my->mc_mutex); + my->mc_new = st; + my->mc_toggle = toggle; + pthread_cond_signal(&my->mc_cond); + pthread_mutex_unlock(&my->mc_mutex); + return 0; +} + + /** Depth-first tree traversal for compacting copy. */ +static int ESECT +mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags) +{ + MDB_cursor mc; + MDB_txn *txn = my->mc_txn; + MDB_node *ni; + MDB_page *mo, *mp, *leaf; + char *buf, *ptr; + int rc, toggle; + unsigned int i; + + /* Empty DB, nothing to do */ + if (*pg == P_INVALID) + return MDB_SUCCESS; + + mc.mc_snum = 1; + mc.mc_top = 0; + mc.mc_txn = txn; + + rc = mdb_page_get(my->mc_txn, *pg, &mc.mc_pg[0], NULL); + if (rc) + return rc; + rc = mdb_page_search_root(&mc, NULL, MDB_PS_FIRST); + if (rc) + return rc; + + /* Make cursor pages writable */ + buf = ptr = malloc(my->mc_env->me_psize * mc.mc_snum); + if (buf == NULL) + return ENOMEM; + + for (i=0; i<mc.mc_top; i++) { + mdb_page_copy((MDB_page *)ptr, mc.mc_pg[i], my->mc_env->me_psize); + mc.mc_pg[i] = (MDB_page *)ptr; + ptr += my->mc_env->me_psize; + } + + /* This is writable space for a leaf page. Usually not needed. */ + leaf = (MDB_page *)ptr; + + toggle = my->mc_toggle; + while (mc.mc_snum > 0) { + unsigned n; + mp = mc.mc_pg[mc.mc_top]; + n = NUMKEYS(mp); + + if (IS_LEAF(mp)) { + if (!IS_LEAF2(mp) && !(flags & F_DUPDATA)) { + for (i=0; i<n; i++) { + ni = NODEPTR(mp, i); + if (ni->mn_flags & F_BIGDATA) { + MDB_page *omp; + pgno_t pg; + + /* Need writable leaf */ + if (mp != leaf) { + mc.mc_pg[mc.mc_top] = leaf; + mdb_page_copy(leaf, mp, my->mc_env->me_psize); + mp = leaf; + ni = NODEPTR(mp, i); + } + + memcpy(&pg, NODEDATA(ni), sizeof(pg)); + rc = mdb_page_get(txn, pg, &omp, NULL); + if (rc) + goto done; + if (my->mc_wlen[toggle] >= MDB_WBUF) { + rc = mdb_env_cthr_toggle(my, 1); + if (rc) + goto done; + toggle = my->mc_toggle; + } + mo = (MDB_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]); + memcpy(mo, omp, my->mc_env->me_psize); + mo->mp_pgno = my->mc_next_pgno; + my->mc_next_pgno += omp->mp_pages; + my->mc_wlen[toggle] += my->mc_env->me_psize; + if (omp->mp_pages > 1) { + my->mc_olen[toggle] = my->mc_env->me_psize * (omp->mp_pages - 1); + my->mc_over[toggle] = (char *)omp + my->mc_env->me_psize; + rc = mdb_env_cthr_toggle(my, 1); + if (rc) + goto done; + toggle = my->mc_toggle; + } + memcpy(NODEDATA(ni), &mo->mp_pgno, sizeof(pgno_t)); + } else if (ni->mn_flags & F_SUBDATA) { + MDB_db db; + + /* Need writable leaf */ + if (mp != leaf) { + mc.mc_pg[mc.mc_top] = leaf; + mdb_page_copy(leaf, mp, my->mc_env->me_psize); + mp = leaf; + ni = NODEPTR(mp, i); + } + + memcpy(&db, NODEDATA(ni), sizeof(db)); + my->mc_toggle = toggle; + rc = mdb_env_cwalk(my, &db.md_root, ni->mn_flags & F_DUPDATA); + if (rc) + goto done; + toggle = my->mc_toggle; + memcpy(NODEDATA(ni), &db, sizeof(db)); + } + } + } + } else { + mc.mc_ki[mc.mc_top]++; + if (mc.mc_ki[mc.mc_top] < n) { + pgno_t pg; +again: + ni = NODEPTR(mp, mc.mc_ki[mc.mc_top]); + pg = NODEPGNO(ni); + rc = mdb_page_get(txn, pg, &mp, NULL); + if (rc) + goto done; + mc.mc_top++; + mc.mc_snum++; + mc.mc_ki[mc.mc_top] = 0; + if (IS_BRANCH(mp)) { + /* Whenever we advance to a sibling branch page, + * we must proceed all the way down to its first leaf. + */ + mdb_page_copy(mc.mc_pg[mc.mc_top], mp, my->mc_env->me_psize); + goto again; + } else + mc.mc_pg[mc.mc_top] = mp; + continue; + } + } + if (my->mc_wlen[toggle] >= MDB_WBUF) { + rc = mdb_env_cthr_toggle(my, 1); + if (rc) + goto done; + toggle = my->mc_toggle; + } + mo = (MDB_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]); + mdb_page_copy(mo, mp, my->mc_env->me_psize); + mo->mp_pgno = my->mc_next_pgno++; + my->mc_wlen[toggle] += my->mc_env->me_psize; + if (mc.mc_top) { + /* Update parent if there is one */ + ni = NODEPTR(mc.mc_pg[mc.mc_top-1], mc.mc_ki[mc.mc_top-1]); + SETPGNO(ni, mo->mp_pgno); + mdb_cursor_pop(&mc); + } else { + /* Otherwise we're done */ + *pg = mo->mp_pgno; + break; + } + } +done: + free(buf); + return rc; +} + + /** Copy environment with compaction. */ +static int ESECT +mdb_env_copyfd1(MDB_env *env, HANDLE fd) +{ + MDB_meta *mm; + MDB_page *mp; + mdb_copy my; + MDB_txn *txn = NULL; + pthread_t thr; + int rc; + +#ifdef _WIN32 + my.mc_mutex = CreateMutex(NULL, FALSE, NULL); + my.mc_cond = CreateEvent(NULL, FALSE, FALSE, NULL); + my.mc_wbuf[0] = _aligned_malloc(MDB_WBUF*2, env->me_os_psize); + if (my.mc_wbuf[0] == NULL) + return errno; +#else + pthread_mutex_init(&my.mc_mutex, NULL); + pthread_cond_init(&my.mc_cond, NULL); +#ifdef HAVE_MEMALIGN + my.mc_wbuf[0] = memalign(env->me_os_psize, MDB_WBUF*2); + if (my.mc_wbuf[0] == NULL) + return errno; +#else + rc = posix_memalign((void **)&my.mc_wbuf[0], env->me_os_psize, MDB_WBUF*2); + if (rc) + return rc; +#endif +#endif + memset(my.mc_wbuf[0], 0, MDB_WBUF*2); + my.mc_wbuf[1] = my.mc_wbuf[0] + MDB_WBUF; + my.mc_wlen[0] = 0; + my.mc_wlen[1] = 0; + my.mc_olen[0] = 0; + my.mc_olen[1] = 0; + my.mc_next_pgno = 2; + my.mc_status = 0; + my.mc_new = 1; + my.mc_toggle = 0; + my.mc_env = env; + my.mc_fd = fd; + THREAD_CREATE(thr, mdb_env_copythr, &my); + + rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + if (rc) + return rc; + + mp = (MDB_page *)my.mc_wbuf[0]; + memset(mp, 0, 2*env->me_psize); + mp->mp_pgno = 0; + mp->mp_flags = P_META; + mm = (MDB_meta *)METADATA(mp); + mdb_env_init_meta0(env, mm); + mm->mm_address = env->me_metas[0]->mm_address; + + mp = (MDB_page *)(my.mc_wbuf[0] + env->me_psize); + mp->mp_pgno = 1; + mp->mp_flags = P_META; + *(MDB_meta *)METADATA(mp) = *mm; + mm = (MDB_meta *)METADATA(mp); + + /* Count the number of free pages, subtract from lastpg to find + * number of active pages + */ + { + MDB_ID freecount = 0; + MDB_cursor mc; + MDB_val key, data; + mdb_cursor_init(&mc, txn, FREE_DBI, NULL); + while ((rc = mdb_cursor_get(&mc, &key, &data, MDB_NEXT)) == 0) + freecount += *(MDB_ID *)data.mv_data; + freecount += txn->mt_dbs[0].md_branch_pages + + txn->mt_dbs[0].md_leaf_pages + + txn->mt_dbs[0].md_overflow_pages; + + /* Set metapage 1 */ + mm->mm_last_pg = txn->mt_next_pgno - freecount - 1; + mm->mm_dbs[1] = txn->mt_dbs[1]; + mm->mm_dbs[1].md_root = mm->mm_last_pg; + mm->mm_txnid = 1; + } + my.mc_wlen[0] = env->me_psize * 2; + my.mc_txn = txn; + pthread_mutex_lock(&my.mc_mutex); + while(my.mc_new) + pthread_cond_wait(&my.mc_cond, &my.mc_mutex); + pthread_mutex_unlock(&my.mc_mutex); + rc = mdb_env_cwalk(&my, &txn->mt_dbs[1].md_root, 0); + if (rc == MDB_SUCCESS && my.mc_wlen[my.mc_toggle]) + rc = mdb_env_cthr_toggle(&my, 1); + mdb_env_cthr_toggle(&my, -1); + pthread_mutex_lock(&my.mc_mutex); + while(my.mc_new) + pthread_cond_wait(&my.mc_cond, &my.mc_mutex); + pthread_mutex_unlock(&my.mc_mutex); + THREAD_FINISH(thr); + + mdb_txn_abort(txn); +#ifdef _WIN32 + CloseHandle(my.mc_cond); + CloseHandle(my.mc_mutex); + _aligned_free(my.mc_wbuf[0]); +#else + pthread_cond_destroy(&my.mc_cond); + pthread_mutex_destroy(&my.mc_mutex); + free(my.mc_wbuf[0]); +#endif + return rc; +} + + /** Copy environment as-is. */ +static int ESECT +mdb_env_copyfd0(MDB_env *env, HANDLE fd) +{ + MDB_txn *txn = NULL; + int rc; + size_t wsize; + char *ptr; +#ifdef _WIN32 + DWORD len, w2; +#define DO_WRITE(rc, fd, ptr, w2, len) rc = WriteFile(fd, ptr, w2, &len, NULL) +#else + ssize_t len; + size_t w2; +#define DO_WRITE(rc, fd, ptr, w2, len) len = write(fd, ptr, w2); rc = (len >= 0) +#endif + + /* Do the lock/unlock of the reader mutex before starting the + * write txn. Otherwise other read txns could block writers. + */ + rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + if (rc) + return rc; + + if (env->me_txns) { + /* We must start the actual read txn after blocking writers */ + mdb_txn_reset0(txn, "reset-stage1"); + + /* Temporarily block writers until we snapshot the meta pages */ + LOCK_MUTEX_W(env); + + rc = mdb_txn_renew0(txn); + if (rc) { + UNLOCK_MUTEX_W(env); + goto leave; + } + } + + wsize = env->me_psize * 2; + ptr = env->me_map; + w2 = wsize; + while (w2 > 0) { + DO_WRITE(rc, fd, ptr, w2, len); + if (!rc) { + rc = ErrCode(); + break; + } else if (len > 0) { + rc = MDB_SUCCESS; + ptr += len; + w2 -= len; + continue; + } else { + /* Non-blocking or async handles are not supported */ + rc = EIO; + break; + } + } + if (env->me_txns) + UNLOCK_MUTEX_W(env); + + if (rc) + goto leave; + + w2 = txn->mt_next_pgno * env->me_psize; +#ifdef WIN32 + { + LARGE_INTEGER fsize; + GetFileSizeEx(env->me_fd, &fsize); + if (w2 > fsize.QuadPart) + w2 = fsize.QuadPart; + } +#else + { + struct stat st; + fstat(env->me_fd, &st); + if (w2 > (size_t)st.st_size) + w2 = st.st_size; + } +#endif + wsize = w2 - wsize; + while (wsize > 0) { + if (wsize > MAX_WRITE) + w2 = MAX_WRITE; + else + w2 = wsize; + DO_WRITE(rc, fd, ptr, w2, len); + if (!rc) { + rc = ErrCode(); + break; + } else if (len > 0) { + rc = MDB_SUCCESS; + ptr += len; + wsize -= len; + continue; + } else { + rc = EIO; + break; + } + } + +leave: + mdb_txn_abort(txn); + return rc; +} + +int ESECT +mdb_env_copyfd2(MDB_env *env, HANDLE fd, unsigned int flags) +{ + if (flags & MDB_CP_COMPACT) + return mdb_env_copyfd1(env, fd); + else + return mdb_env_copyfd0(env, fd); +} + +int ESECT +mdb_env_copyfd(MDB_env *env, HANDLE fd) +{ + return mdb_env_copyfd2(env, fd, 0); +} + +int ESECT +mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags) +{ + int rc, len; + char *lpath; + HANDLE newfd = INVALID_HANDLE_VALUE; + + if (env->me_flags & MDB_NOSUBDIR) { + lpath = (char *)path; + } else { + len = strlen(path); + len += sizeof(DATANAME); + lpath = malloc(len); + if (!lpath) + return ENOMEM; + sprintf(lpath, "%s" DATANAME, path); + } + + /* The destination path must exist, but the destination file must not. + * We don't want the OS to cache the writes, since the source data is + * already in the OS cache. + */ +#ifdef _WIN32 + newfd = CreateFile(lpath, GENERIC_WRITE, 0, NULL, CREATE_NEW, + FILE_FLAG_NO_BUFFERING|FILE_FLAG_WRITE_THROUGH, NULL); +#else + newfd = open(lpath, O_WRONLY|O_CREAT|O_EXCL, 0666); +#endif + if (newfd == INVALID_HANDLE_VALUE) { + rc = ErrCode(); + goto leave; + } + + if (env->me_psize >= env->me_os_psize) { +#ifdef O_DIRECT + /* Set O_DIRECT if the file system supports it */ + if ((rc = fcntl(newfd, F_GETFL)) != -1) + (void) fcntl(newfd, F_SETFL, rc | O_DIRECT); +#endif +#ifdef F_NOCACHE /* __APPLE__ */ + rc = fcntl(newfd, F_NOCACHE, 1); + if (rc) { + rc = ErrCode(); + goto leave; + } +#endif + } + + rc = mdb_env_copyfd2(env, newfd, flags); + +leave: + if (!(env->me_flags & MDB_NOSUBDIR)) + free(lpath); + if (newfd != INVALID_HANDLE_VALUE) + if (close(newfd) < 0 && rc == MDB_SUCCESS) + rc = ErrCode(); + + return rc; +} + +int ESECT +mdb_env_copy(MDB_env *env, const char *path) +{ + return mdb_env_copy2(env, path, 0); +} + +int ESECT +mdb_env_set_flags(MDB_env *env, unsigned int flag, int onoff) +{ + if ((flag & CHANGEABLE) != flag) + return EINVAL; + if (onoff) + env->me_flags |= flag; + else + env->me_flags &= ~flag; + return MDB_SUCCESS; +} + +int ESECT +mdb_env_get_flags(MDB_env *env, unsigned int *arg) +{ + if (!env || !arg) + return EINVAL; + + *arg = env->me_flags; + return MDB_SUCCESS; +} + +int ESECT +mdb_env_set_userctx(MDB_env *env, void *ctx) +{ + if (!env) + return EINVAL; + env->me_userctx = ctx; + return MDB_SUCCESS; +} + +void * ESECT +mdb_env_get_userctx(MDB_env *env) +{ + return env ? env->me_userctx : NULL; +} + +int ESECT +mdb_env_set_assert(MDB_env *env, MDB_assert_func *func) +{ + if (!env) + return EINVAL; +#ifndef NDEBUG + env->me_assert_func = func; +#endif + return MDB_SUCCESS; +} + +int ESECT +mdb_env_get_path(MDB_env *env, const char **arg) +{ + if (!env || !arg) + return EINVAL; + + *arg = env->me_path; + return MDB_SUCCESS; +} + +int ESECT +mdb_env_get_fd(MDB_env *env, mdb_filehandle_t *arg) +{ + if (!env || !arg) + return EINVAL; + + *arg = env->me_fd; + return MDB_SUCCESS; +} + +/** Common code for #mdb_stat() and #mdb_env_stat(). + * @param[in] env the environment to operate in. + * @param[in] db the #MDB_db record containing the stats to return. + * @param[out] arg the address of an #MDB_stat structure to receive the stats. + * @return 0, this function always succeeds. + */ +static int ESECT +mdb_stat0(MDB_env *env, MDB_db *db, MDB_stat *arg) +{ + arg->ms_psize = env->me_psize; + arg->ms_depth = db->md_depth; + arg->ms_branch_pages = db->md_branch_pages; + arg->ms_leaf_pages = db->md_leaf_pages; + arg->ms_overflow_pages = db->md_overflow_pages; + arg->ms_entries = db->md_entries; + + return MDB_SUCCESS; +} + +int ESECT +mdb_env_stat(MDB_env *env, MDB_stat *arg) +{ + int toggle; + + if (env == NULL || arg == NULL) + return EINVAL; + + toggle = mdb_env_pick_meta(env); + + return mdb_stat0(env, &env->me_metas[toggle]->mm_dbs[MAIN_DBI], arg); +} + +int ESECT +mdb_env_info(MDB_env *env, MDB_envinfo *arg) +{ + int toggle; + + if (env == NULL || arg == NULL) + return EINVAL; + + toggle = mdb_env_pick_meta(env); + arg->me_mapaddr = env->me_metas[toggle]->mm_address; + arg->me_mapsize = env->me_mapsize; + arg->me_maxreaders = env->me_maxreaders; + + /* me_numreaders may be zero if this process never used any readers. Use + * the shared numreader count if it exists. + */ + arg->me_numreaders = env->me_txns ? env->me_txns->mti_numreaders : env->me_numreaders; + + arg->me_last_pgno = env->me_metas[toggle]->mm_last_pg; + arg->me_last_txnid = env->me_metas[toggle]->mm_txnid; + return MDB_SUCCESS; +} + +/** Set the default comparison functions for a database. + * Called immediately after a database is opened to set the defaults. + * The user can then override them with #mdb_set_compare() or + * #mdb_set_dupsort(). + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + */ +static void +mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi) +{ + uint16_t f = txn->mt_dbs[dbi].md_flags; + + txn->mt_dbxs[dbi].md_cmp = + (f & MDB_REVERSEKEY) ? mdb_cmp_memnr : + (f & MDB_INTEGERKEY) ? mdb_cmp_cint : mdb_cmp_memn; + + txn->mt_dbxs[dbi].md_dcmp = + !(f & MDB_DUPSORT) ? 0 : + ((f & MDB_INTEGERDUP) + ? ((f & MDB_DUPFIXED) ? mdb_cmp_int : mdb_cmp_cint) + : ((f & MDB_REVERSEDUP) ? mdb_cmp_memnr : mdb_cmp_memn)); +} + +int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *dbi) +{ + MDB_val key, data; + MDB_dbi i; + MDB_cursor mc; + MDB_db dummy; + int rc, dbflag, exact; + unsigned int unused = 0, seq; + size_t len; + + if (txn->mt_dbxs[FREE_DBI].md_cmp == NULL) { + mdb_default_cmp(txn, FREE_DBI); + } + + if ((flags & VALID_FLAGS) != flags) + return EINVAL; + if (txn->mt_flags & MDB_TXN_ERROR) + return MDB_BAD_TXN; + + /* main DB? */ + if (!name) { + *dbi = MAIN_DBI; + if (flags & PERSISTENT_FLAGS) { + uint16_t f2 = flags & PERSISTENT_FLAGS; + /* make sure flag changes get committed */ + if ((txn->mt_dbs[MAIN_DBI].md_flags | f2) != txn->mt_dbs[MAIN_DBI].md_flags) { + txn->mt_dbs[MAIN_DBI].md_flags |= f2; + txn->mt_flags |= MDB_TXN_DIRTY; + } + } + mdb_default_cmp(txn, MAIN_DBI); + return MDB_SUCCESS; + } + + if (txn->mt_dbxs[MAIN_DBI].md_cmp == NULL) { + mdb_default_cmp(txn, MAIN_DBI); + } + + /* Is the DB already open? */ + len = strlen(name); + for (i=2; i<txn->mt_numdbs; i++) { + if (!txn->mt_dbxs[i].md_name.mv_size) { + /* Remember this free slot */ + if (!unused) unused = i; + continue; + } + if (len == txn->mt_dbxs[i].md_name.mv_size && + !strncmp(name, txn->mt_dbxs[i].md_name.mv_data, len)) { + *dbi = i; + return MDB_SUCCESS; + } + } + + /* If no free slot and max hit, fail */ + if (!unused && txn->mt_numdbs >= txn->mt_env->me_maxdbs) + return MDB_DBS_FULL; + + /* Cannot mix named databases with some mainDB flags */ + if (txn->mt_dbs[MAIN_DBI].md_flags & (MDB_DUPSORT|MDB_INTEGERKEY)) + return (flags & MDB_CREATE) ? MDB_INCOMPATIBLE : MDB_NOTFOUND; + + /* Find the DB info */ + dbflag = DB_NEW|DB_VALID; + exact = 0; + key.mv_size = len; + key.mv_data = (void *)name; + mdb_cursor_init(&mc, txn, MAIN_DBI, NULL); + rc = mdb_cursor_set(&mc, &key, &data, MDB_SET, &exact); + if (rc == MDB_SUCCESS) { + /* make sure this is actually a DB */ + MDB_node *node = NODEPTR(mc.mc_pg[mc.mc_top], mc.mc_ki[mc.mc_top]); + if (!(node->mn_flags & F_SUBDATA)) + return MDB_INCOMPATIBLE; + } else if (rc == MDB_NOTFOUND && (flags & MDB_CREATE)) { + /* Create if requested */ + data.mv_size = sizeof(MDB_db); + data.mv_data = &dummy; + memset(&dummy, 0, sizeof(dummy)); + dummy.md_root = P_INVALID; + dummy.md_flags = flags & PERSISTENT_FLAGS; + rc = mdb_cursor_put(&mc, &key, &data, F_SUBDATA); + dbflag |= DB_DIRTY; + } + + /* OK, got info, add to table */ + if (rc == MDB_SUCCESS) { + unsigned int slot = unused ? unused : txn->mt_numdbs; + txn->mt_dbxs[slot].md_name.mv_data = strdup(name); + txn->mt_dbxs[slot].md_name.mv_size = len; + txn->mt_dbxs[slot].md_rel = NULL; + txn->mt_dbflags[slot] = dbflag; + /* txn-> and env-> are the same in read txns, use + * tmp variable to avoid undefined assignment + */ + seq = ++txn->mt_env->me_dbiseqs[slot]; + txn->mt_dbiseqs[slot] = seq; + + memcpy(&txn->mt_dbs[slot], data.mv_data, sizeof(MDB_db)); + *dbi = slot; + mdb_default_cmp(txn, slot); + if (!unused) { + txn->mt_numdbs++; + } + } + + return rc; +} + +int mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *arg) +{ + if (!arg || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + + if (txn->mt_flags & MDB_TXN_ERROR) + return MDB_BAD_TXN; + + if (txn->mt_dbflags[dbi] & DB_STALE) { + MDB_cursor mc; + MDB_xcursor mx; + /* Stale, must read the DB's root. cursor_init does it for us. */ + mdb_cursor_init(&mc, txn, dbi, &mx); + } + return mdb_stat0(txn->mt_env, &txn->mt_dbs[dbi], arg); +} + +void mdb_dbi_close(MDB_env *env, MDB_dbi dbi) +{ + char *ptr; + if (dbi <= MAIN_DBI || dbi >= env->me_maxdbs) + return; + ptr = env->me_dbxs[dbi].md_name.mv_data; + /* If there was no name, this was already closed */ + if (ptr) { + env->me_dbxs[dbi].md_name.mv_data = NULL; + env->me_dbxs[dbi].md_name.mv_size = 0; + env->me_dbflags[dbi] = 0; + env->me_dbiseqs[dbi]++; + free(ptr); + } +} + +int mdb_dbi_flags(MDB_txn *txn, MDB_dbi dbi, unsigned int *flags) +{ + /* We could return the flags for the FREE_DBI too but what's the point? */ + if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + *flags = txn->mt_dbs[dbi].md_flags & PERSISTENT_FLAGS; + return MDB_SUCCESS; +} + +/** Add all the DB's pages to the free list. + * @param[in] mc Cursor on the DB to free. + * @param[in] subs non-Zero to check for sub-DBs in this DB. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_drop0(MDB_cursor *mc, int subs) +{ + int rc; + + rc = mdb_page_search(mc, NULL, MDB_PS_FIRST); + if (rc == MDB_SUCCESS) { + MDB_txn *txn = mc->mc_txn; + MDB_node *ni; + MDB_cursor mx; + unsigned int i; + + /* LEAF2 pages have no nodes, cannot have sub-DBs */ + if (IS_LEAF2(mc->mc_pg[mc->mc_top])) + mdb_cursor_pop(mc); + + mdb_cursor_copy(mc, &mx); + while (mc->mc_snum > 0) { + MDB_page *mp = mc->mc_pg[mc->mc_top]; + unsigned n = NUMKEYS(mp); + if (IS_LEAF(mp)) { + for (i=0; i<n; i++) { + ni = NODEPTR(mp, i); + if (ni->mn_flags & F_BIGDATA) { + MDB_page *omp; + pgno_t pg; + memcpy(&pg, NODEDATA(ni), sizeof(pg)); + rc = mdb_page_get(txn, pg, &omp, NULL); + if (rc != 0) + goto done; + mdb_cassert(mc, IS_OVERFLOW(omp)); + rc = mdb_midl_append_range(&txn->mt_free_pgs, + pg, omp->mp_pages); + if (rc) + goto done; + } else if (subs && (ni->mn_flags & F_SUBDATA)) { + mdb_xcursor_init1(mc, ni); + rc = mdb_drop0(&mc->mc_xcursor->mx_cursor, 0); + if (rc) + goto done; + } + } + } else { + if ((rc = mdb_midl_need(&txn->mt_free_pgs, n)) != 0) + goto done; + for (i=0; i<n; i++) { + pgno_t pg; + ni = NODEPTR(mp, i); + pg = NODEPGNO(ni); + /* free it */ + mdb_midl_xappend(txn->mt_free_pgs, pg); + } + } + if (!mc->mc_top) + break; + mc->mc_ki[mc->mc_top] = i; + rc = mdb_cursor_sibling(mc, 1); + if (rc) { + if (rc != MDB_NOTFOUND) + goto done; + /* no more siblings, go back to beginning + * of previous level. + */ + mdb_cursor_pop(mc); + mc->mc_ki[0] = 0; + for (i=1; i<mc->mc_snum; i++) { + mc->mc_ki[i] = 0; + mc->mc_pg[i] = mx.mc_pg[i]; + } + } + } + /* free it */ + rc = mdb_midl_append(&txn->mt_free_pgs, mc->mc_db->md_root); +done: + if (rc) + txn->mt_flags |= MDB_TXN_ERROR; + } else if (rc == MDB_NOTFOUND) { + rc = MDB_SUCCESS; + } + return rc; +} + +int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del) +{ + MDB_cursor *mc, *m2; + int rc; + + if ((unsigned)del > 1 || dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + + if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) + return EACCES; + + if (dbi > MAIN_DBI && TXN_DBI_CHANGED(txn, dbi)) + return MDB_BAD_DBI; + + rc = mdb_cursor_open(txn, dbi, &mc); + if (rc) + return rc; + + rc = mdb_drop0(mc, mc->mc_db->md_flags & MDB_DUPSORT); + /* Invalidate the dropped DB's cursors */ + for (m2 = txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) + m2->mc_flags &= ~(C_INITIALIZED|C_EOF); + if (rc) + goto leave; + + /* Can't delete the main DB */ + if (del && dbi > MAIN_DBI) { + rc = mdb_del0(txn, MAIN_DBI, &mc->mc_dbx->md_name, NULL, 0); + if (!rc) { + txn->mt_dbflags[dbi] = DB_STALE; + mdb_dbi_close(txn->mt_env, dbi); + } else { + txn->mt_flags |= MDB_TXN_ERROR; + } + } else { + /* reset the DB record, mark it dirty */ + txn->mt_dbflags[dbi] |= DB_DIRTY; + txn->mt_dbs[dbi].md_depth = 0; + txn->mt_dbs[dbi].md_branch_pages = 0; + txn->mt_dbs[dbi].md_leaf_pages = 0; + txn->mt_dbs[dbi].md_overflow_pages = 0; + txn->mt_dbs[dbi].md_entries = 0; + txn->mt_dbs[dbi].md_root = P_INVALID; + + txn->mt_flags |= MDB_TXN_DIRTY; + } +leave: + mdb_cursor_close(mc); + return rc; +} + +int mdb_set_compare(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp) +{ + if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + + txn->mt_dbxs[dbi].md_cmp = cmp; + return MDB_SUCCESS; +} + +int mdb_set_dupsort(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp) +{ + if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + + txn->mt_dbxs[dbi].md_dcmp = cmp; + return MDB_SUCCESS; +} + +int mdb_set_relfunc(MDB_txn *txn, MDB_dbi dbi, MDB_rel_func *rel) +{ + if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + + txn->mt_dbxs[dbi].md_rel = rel; + return MDB_SUCCESS; +} + +int mdb_set_relctx(MDB_txn *txn, MDB_dbi dbi, void *ctx) +{ + if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + + txn->mt_dbxs[dbi].md_relctx = ctx; + return MDB_SUCCESS; +} + +int ESECT +mdb_env_get_maxkeysize(MDB_env *env) +{ + return ENV_MAXKEY(env); +} + +int ESECT +mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx) +{ + unsigned int i, rdrs; + MDB_reader *mr; + char buf[64]; + int rc = 0, first = 1; + + if (!env || !func) + return -1; + if (!env->me_txns) { + return func("(no reader locks)\n", ctx); + } + rdrs = env->me_txns->mti_numreaders; + mr = env->me_txns->mti_readers; + for (i=0; i<rdrs; i++) { + if (mr[i].mr_pid) { + txnid_t txnid = mr[i].mr_txnid; + sprintf(buf, txnid == (txnid_t)-1 ? + "%10d %"Z"x -\n" : "%10d %"Z"x %"Z"u\n", + (int)mr[i].mr_pid, (size_t)mr[i].mr_tid, txnid); + if (first) { + first = 0; + rc = func(" pid thread txnid\n", ctx); + if (rc < 0) + break; + } + rc = func(buf, ctx); + if (rc < 0) + break; + } + } + if (first) { + rc = func("(no active readers)\n", ctx); + } + return rc; +} + +/** Insert pid into list if not already present. + * return -1 if already present. + */ +static int ESECT +mdb_pid_insert(MDB_PID_T *ids, MDB_PID_T pid) +{ + /* binary search of pid in list */ + unsigned base = 0; + unsigned cursor = 1; + int val = 0; + unsigned n = ids[0]; + + while( 0 < n ) { + unsigned pivot = n >> 1; + cursor = base + pivot + 1; + val = pid - ids[cursor]; + + if( val < 0 ) { + n = pivot; + + } else if ( val > 0 ) { + base = cursor; + n -= pivot + 1; + + } else { + /* found, so it's a duplicate */ + return -1; + } + } + + if( val > 0 ) { + ++cursor; + } + ids[0]++; + for (n = ids[0]; n > cursor; n--) + ids[n] = ids[n-1]; + ids[n] = pid; + return 0; +} + +int ESECT +mdb_reader_check(MDB_env *env, int *dead) +{ + unsigned int i, j, rdrs; + MDB_reader *mr; + MDB_PID_T *pids, pid; + int count = 0; + + if (!env) + return EINVAL; + if (dead) + *dead = 0; + if (!env->me_txns) + return MDB_SUCCESS; + rdrs = env->me_txns->mti_numreaders; + pids = malloc((rdrs+1) * sizeof(MDB_PID_T)); + if (!pids) + return ENOMEM; + pids[0] = 0; + mr = env->me_txns->mti_readers; + for (i=0; i<rdrs; i++) { + if (mr[i].mr_pid && mr[i].mr_pid != env->me_pid) { + pid = mr[i].mr_pid; + if (mdb_pid_insert(pids, pid) == 0) { + if (!mdb_reader_pid(env, Pidcheck, pid)) { + LOCK_MUTEX_R(env); + /* Recheck, a new process may have reused pid */ + if (!mdb_reader_pid(env, Pidcheck, pid)) { + for (j=i; j<rdrs; j++) + if (mr[j].mr_pid == pid) { + DPRINTF(("clear stale reader pid %u txn %"Z"d", + (unsigned) pid, mr[j].mr_txnid)); + mr[j].mr_pid = 0; + count++; + } + } + UNLOCK_MUTEX_R(env); + } + } + } + } + free(pids); + if (dead) + *dead = count; + return MDB_SUCCESS; +} +/** @} */ diff --git a/external/db_drivers/liblmdb32/mdb_copy.1 b/external/db_drivers/liblmdb32/mdb_copy.1 new file mode 100644 index 000000000..094b26056 --- /dev/null +++ b/external/db_drivers/liblmdb32/mdb_copy.1 @@ -0,0 +1,54 @@ +.TH MDB_COPY 1 "2014/06/20" "LMDB 0.9.14" +.\" Copyright 2012-2014 Howard Chu, Symas Corp. All Rights Reserved. +.\" Copying restrictions apply. See COPYRIGHT/LICENSE. +.SH NAME +mdb_copy \- LMDB environment copy tool +.SH SYNOPSIS +.B mdb_copy +[\c +.BR \-V ] +[\c +.BR \-c ] +[\c +.BR \-n ] +.B srcpath +[\c +.BR dstpath ] +.SH DESCRIPTION +The +.B mdb_copy +utility copies an LMDB environment. The environment can +be copied regardless of whether it is currently in use. +No lockfile is created, since it gets recreated at need. + +If +.I dstpath +is specified it must be the path of an empty directory +for storing the backup. Otherwise, the backup will be +written to stdout. + +.SH OPTIONS +.TP +.BR \-V +Write the library version number to the standard output, and exit. +.TP +.BR \-c +Compact while copying. Only current data pages will be copied; freed +or unused pages will be omitted from the copy. This option will +slow down the backup process as it is more CPU-intensive. +.TP +.BR \-n +Open LDMB environment(s) which do not use subdirectories. + +.SH DIAGNOSTICS +Exit status is zero if no errors occur. +Errors result in a non-zero exit status and +a diagnostic message being written to standard error. +.SH CAVEATS +This utility can trigger significant file size growth if run +in parallel with write transactions, because pages which they +free during copying cannot be reused until the copy is done. +.SH "SEE ALSO" +.BR mdb_stat (1) +.SH AUTHOR +Howard Chu of Symas Corporation <http://www.symas.com> diff --git a/external/db_drivers/liblmdb32/mdb_copy.c b/external/db_drivers/liblmdb32/mdb_copy.c new file mode 100644 index 000000000..e7f965c03 --- /dev/null +++ b/external/db_drivers/liblmdb32/mdb_copy.c @@ -0,0 +1,82 @@ +/* mdb_copy.c - memory-mapped database backup tool */ +/* + * Copyright 2012 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ +#ifdef _WIN32 +#include <windows.h> +#define MDB_STDOUT GetStdHandle(STD_OUTPUT_HANDLE) +#else +#define MDB_STDOUT 1 +#endif +#include <stdio.h> +#include <stdlib.h> +#include <signal.h> +#include "lmdb.h" + +static void +sighandle(int sig) +{ +} + +int main(int argc,char * argv[]) +{ + int rc; + MDB_env *env; + const char *progname = argv[0], *act; + unsigned flags = MDB_RDONLY; + unsigned cpflags = 0; + + for (; argc > 1 && argv[1][0] == '-'; argc--, argv++) { + if (argv[1][1] == 'n' && argv[1][2] == '\0') + flags |= MDB_NOSUBDIR; + else if (argv[1][1] == 'c' && argv[1][2] == '\0') + cpflags |= MDB_CP_COMPACT; + else if (argv[1][1] == 'V' && argv[1][2] == '\0') { + printf("%s\n", MDB_VERSION_STRING); + exit(0); + } else + argc = 0; + } + + if (argc<2 || argc>3) { + fprintf(stderr, "usage: %s [-V] [-c] [-n] srcpath [dstpath]\n", progname); + exit(EXIT_FAILURE); + } + +#ifdef SIGPIPE + signal(SIGPIPE, sighandle); +#endif +#ifdef SIGHUP + signal(SIGHUP, sighandle); +#endif + signal(SIGINT, sighandle); + signal(SIGTERM, sighandle); + + act = "opening environment"; + rc = mdb_env_create(&env); + if (rc == MDB_SUCCESS) { + rc = mdb_env_open(env, argv[1], flags, 0664); + } + if (rc == MDB_SUCCESS) { + act = "copying"; + if (argc == 2) + rc = mdb_env_copyfd2(env, MDB_STDOUT, cpflags); + else + rc = mdb_env_copy2(env, argv[2], cpflags); + } + if (rc) + fprintf(stderr, "%s: %s failed, error %d (%s)\n", + progname, act, rc, mdb_strerror(rc)); + mdb_env_close(env); + + return rc ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/external/db_drivers/liblmdb32/mdb_dump.1 b/external/db_drivers/liblmdb32/mdb_dump.1 new file mode 100644 index 000000000..6fcc93081 --- /dev/null +++ b/external/db_drivers/liblmdb32/mdb_dump.1 @@ -0,0 +1,75 @@ +.TH MDB_DUMP 1 "2014/06/20" "LMDB 0.9.14" +.\" Copyright 2014 Howard Chu, Symas Corp. All Rights Reserved. +.\" Copying restrictions apply. See COPYRIGHT/LICENSE. +.SH NAME +mdb_dump \- LMDB environment export tool +.SH SYNOPSIS +.B mdb_dump +.BR \ envpath +[\c +.BR \-V ] +[\c +.BI \-f \ file\fR] +[\c +.BR \-l ] +[\c +.BR \-n ] +[\c +.BR \-p ] +[\c +.BR \-a \ | +.BI \-s \ subdb\fR] +.SH DESCRIPTION +The +.B mdb_dump +utility reads a database and writes its contents to the +standard output using a portable flat-text format +understood by the +.BR mdb_load (1) +utility. +.SH OPTIONS +.TP +.BR \-V +Write the library version number to the standard output, and exit. +.TP +.BR \-f \ file +Write to the specified file instead of to the standard output. +.TP +.BR \-l +List the databases stored in the environment. Just the +names will be listed, no data will be output. +.TP +.BR \-n +Dump an LMDB database which does not use subdirectories. +.TP +.BR \-p +If characters in either the key or data items are printing characters (as +defined by isprint(3)), output them directly. This option permits users to +use standard text editors and tools to modify the contents of databases. + +Note: different systems may have different notions about what characters +are considered printing characters, and databases dumped in this manner may +be less portable to external systems. +.TP +.BR \-a +Dump all of the subdatabases in the environment. +.TP +.BR \-s \ subdb +Dump a specific subdatabase. If no database is specified, only the main database is dumped. +.SH DIAGNOSTICS +Exit status is zero if no errors occur. +Errors result in a non-zero exit status and +a diagnostic message being written to standard error. + +Dumping and reloading databases that use user-defined comparison functions +will result in new databases that use the default comparison functions. +\fBIn this case it is quite likely that the reloaded database will be +damaged beyond repair permitting neither record storage nor retrieval.\fP + +The only available workaround is to modify the source for the +.BR mdb_load (1) +utility to load the database using the correct comparison functions. +.SH "SEE ALSO" +.BR mdb_load (1) +.SH AUTHOR +Howard Chu of Symas Corporation <http://www.symas.com> diff --git a/external/db_drivers/liblmdb32/mdb_dump.c b/external/db_drivers/liblmdb32/mdb_dump.c new file mode 100644 index 000000000..0eb85fd20 --- /dev/null +++ b/external/db_drivers/liblmdb32/mdb_dump.c @@ -0,0 +1,317 @@ +/* mdb_dump.c - memory-mapped database dump tool */ +/* + * Copyright 2011-2014 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ +#include <stdio.h> +#include <errno.h> +#include <stdlib.h> +#include <string.h> +#include <ctype.h> +#include <unistd.h> +#include <signal.h> +#include "lmdb.h" + +#ifdef _WIN32 +#define Z "I" +#else +#define Z "z" +#endif + +#define PRINT 1 +static int mode; + +typedef struct flagbit { + int bit; + char *name; +} flagbit; + +flagbit dbflags[] = { + { MDB_REVERSEKEY, "reversekey" }, + { MDB_DUPSORT, "dupsort" }, + { MDB_INTEGERKEY, "integerkey" }, + { MDB_DUPFIXED, "dupfixed" }, + { MDB_INTEGERDUP, "integerdup" }, + { MDB_REVERSEDUP, "reversedup" }, + { 0, NULL } +}; + +static volatile sig_atomic_t gotsig; + +static void dumpsig( int sig ) +{ + gotsig=1; +} + +static const char hexc[] = "0123456789abcdef"; + +static void hex(unsigned char c) +{ + putchar(hexc[c >> 4]); + putchar(hexc[c & 0xf]); +} + +static void text(MDB_val *v) +{ + unsigned char *c, *end; + + putchar(' '); + c = v->mv_data; + end = c + v->mv_size; + while (c < end) { + if (isprint(*c)) { + putchar(*c); + } else { + putchar('\\'); + hex(*c); + } + c++; + } + putchar('\n'); +} + +static void byte(MDB_val *v) +{ + unsigned char *c, *end; + + putchar(' '); + c = v->mv_data; + end = c + v->mv_size; + while (c < end) { + hex(*c++); + } + putchar('\n'); +} + +/* Dump in BDB-compatible format */ +static int dumpit(MDB_txn *txn, MDB_dbi dbi, char *name) +{ + MDB_cursor *mc; + MDB_stat ms; + MDB_val key, data; + MDB_envinfo info; + unsigned int flags; + int rc, i; + + rc = mdb_dbi_flags(txn, dbi, &flags); + if (rc) return rc; + + rc = mdb_stat(txn, dbi, &ms); + if (rc) return rc; + + rc = mdb_env_info(mdb_txn_env(txn), &info); + if (rc) return rc; + + printf("VERSION=3\n"); + printf("format=%s\n", mode & PRINT ? "print" : "bytevalue"); + if (name) + printf("database=%s\n", name); + printf("type=btree\n"); + printf("mapsize=%" Z "u\n", info.me_mapsize); + if (info.me_mapaddr) + printf("mapaddr=%p\n", info.me_mapaddr); + printf("maxreaders=%u\n", info.me_maxreaders); + + if (flags & MDB_DUPSORT) + printf("duplicates=1\n"); + + for (i=0; dbflags[i].bit; i++) + if (flags & dbflags[i].bit) + printf("%s=1\n", dbflags[i].name); + + printf("db_pagesize=%d\n", ms.ms_psize); + printf("HEADER=END\n"); + + rc = mdb_cursor_open(txn, dbi, &mc); + if (rc) return rc; + + while ((rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT) == MDB_SUCCESS)) { + if (gotsig) { + rc = EINTR; + break; + } + if (mode & PRINT) { + text(&key); + text(&data); + } else { + byte(&key); + byte(&data); + } + } + printf("DATA=END\n"); + if (rc == MDB_NOTFOUND) + rc = MDB_SUCCESS; + + return rc; +} + +static void usage(char *prog) +{ + fprintf(stderr, "usage: %s dbpath [-V] [-f output] [-l] [-n] [-p] [-a|-s subdb]\n", prog); + exit(EXIT_FAILURE); +} + +int main(int argc, char *argv[]) +{ + int i, rc; + MDB_env *env; + MDB_txn *txn; + MDB_dbi dbi; + char *prog = argv[0]; + char *envname; + char *subname = NULL; + int alldbs = 0, envflags = 0, list = 0; + + if (argc < 2) { + usage(prog); + } + + /* -a: dump main DB and all subDBs + * -s: dump only the named subDB + * -n: use NOSUBDIR flag on env_open + * -p: use printable characters + * -f: write to file instead of stdout + * -V: print version and exit + * (default) dump only the main DB + */ + while ((i = getopt(argc, argv, "af:lnps:V")) != EOF) { + switch(i) { + case 'V': + printf("%s\n", MDB_VERSION_STRING); + exit(0); + break; + case 'l': + list = 1; + /*FALLTHROUGH*/; + case 'a': + if (subname) + usage(prog); + alldbs++; + break; + case 'f': + if (freopen(optarg, "w", stdout) == NULL) { + fprintf(stderr, "%s: %s: reopen: %s\n", + prog, optarg, strerror(errno)); + exit(EXIT_FAILURE); + } + break; + case 'n': + envflags |= MDB_NOSUBDIR; + break; + case 'p': + mode |= PRINT; + break; + case 's': + if (alldbs) + usage(prog); + subname = optarg; + break; + default: + usage(prog); + } + } + + if (optind != argc - 1) + usage(prog); + +#ifdef SIGPIPE + signal(SIGPIPE, dumpsig); +#endif +#ifdef SIGHUP + signal(SIGHUP, dumpsig); +#endif + signal(SIGINT, dumpsig); + signal(SIGTERM, dumpsig); + + envname = argv[optind]; + rc = mdb_env_create(&env); + if (rc) { + fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc)); + return EXIT_FAILURE; + } + + if (alldbs || subname) { + mdb_env_set_maxdbs(env, 2); + } + + rc = mdb_env_open(env, envname, envflags | MDB_RDONLY, 0664); + if (rc) { + fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto env_close; + } + + rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + if (rc) { + fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc)); + goto env_close; + } + + rc = mdb_open(txn, subname, 0, &dbi); + if (rc) { + fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + + if (alldbs) { + MDB_cursor *cursor; + MDB_val key; + int count = 0; + + rc = mdb_cursor_open(txn, dbi, &cursor); + if (rc) { + fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + while ((rc = mdb_cursor_get(cursor, &key, NULL, MDB_NEXT_NODUP)) == 0) { + char *str; + MDB_dbi db2; + if (memchr(key.mv_data, '\0', key.mv_size)) + continue; + count++; + str = malloc(key.mv_size+1); + memcpy(str, key.mv_data, key.mv_size); + str[key.mv_size] = '\0'; + rc = mdb_open(txn, str, 0, &db2); + if (rc == MDB_SUCCESS) { + if (list) { + printf("%s\n", str); + list++; + } else { + rc = dumpit(txn, db2, str); + if (rc) + break; + } + mdb_close(env, db2); + } + free(str); + if (rc) continue; + } + mdb_cursor_close(cursor); + if (!count) { + fprintf(stderr, "%s: %s does not contain multiple databases\n", prog, envname); + rc = MDB_NOTFOUND; + } else if (rc == MDB_NOTFOUND) { + rc = MDB_SUCCESS; + } + } else { + rc = dumpit(txn, dbi, subname); + } + if (rc && rc != MDB_NOTFOUND) + fprintf(stderr, "%s: %s: %s\n", prog, envname, mdb_strerror(rc)); + + mdb_close(env, dbi); +txn_abort: + mdb_txn_abort(txn); +env_close: + mdb_env_close(env); + + return rc ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/external/db_drivers/liblmdb32/mdb_load.1 b/external/db_drivers/liblmdb32/mdb_load.1 new file mode 100644 index 000000000..728024010 --- /dev/null +++ b/external/db_drivers/liblmdb32/mdb_load.1 @@ -0,0 +1,77 @@ +.TH MDB_LOAD 1 "2014/06/20" "LMDB 0.9.14" +.\" Copyright 2014 Howard Chu, Symas Corp. All Rights Reserved. +.\" Copying restrictions apply. See COPYRIGHT/LICENSE. +.SH NAME +mdb_load \- LMDB environment import tool +.SH SYNOPSIS +.B mdb_load +.BR \ envpath +[\c +.BR \-V ] +[\c +.BI \-f \ file\fR] +[\c +.BR \-n ] +[\c +.BI \-s \ subdb\fR] +[\c +.BR \-N ] +[\c +.BR \-T ] +.SH DESCRIPTION +The +.B mdb_load +utility reads from the standard input and loads it into the +LMDB environment +.BR envpath . + +The input to +.B mdb_load +must be in the output format specified by the +.BR mdb_dump (1) +utility or as specified by the +.B -T +option below. +.SH OPTIONS +.TP +.BR \-V +Write the library version number to the standard output, and exit. +.TP +.BR \-f \ file +Read from the specified file instead of from the standard input. +.TP +.BR \-n +Load an LMDB database which does not use subdirectories. +.TP +.BR \-s \ subdb +Load a specific subdatabase. If no database is specified, data is loaded into the main database. +.TP +.BR \-N +Don't overwrite existing records when loading into an already existing database; just skip them. +.TP +.BR \-T +Load data from simple text files. The input must be paired lines of text, where the first +line of the pair is the key item, and the second line of the pair is its corresponding +data item. + +A simple escape mechanism, where newline and backslash (\\) characters are special, is +applied to the text input. Newline characters are interpreted as record separators. +Backslash characters in the text will be interpreted in one of two ways: If the backslash +character precedes another backslash character, the pair will be interpreted as a literal +backslash. If the backslash character precedes any other character, the two characters +following the backslash will be interpreted as a hexadecimal specification of a single +character; for example, \\0a is a newline character in the ASCII character set. + +For this reason, any backslash or newline characters that naturally occur in the text +input must be escaped to avoid misinterpretation by +.BR mdb_load . + +.SH DIAGNOSTICS +Exit status is zero if no errors occur. +Errors result in a non-zero exit status and +a diagnostic message being written to standard error. + +.SH "SEE ALSO" +.BR mdb_dump (1) +.SH AUTHOR +Howard Chu of Symas Corporation <http://www.symas.com> diff --git a/external/db_drivers/liblmdb32/mdb_load.c b/external/db_drivers/liblmdb32/mdb_load.c new file mode 100644 index 000000000..e0d95e13c --- /dev/null +++ b/external/db_drivers/liblmdb32/mdb_load.c @@ -0,0 +1,452 @@ +/* mdb_load.c - memory-mapped database load tool */ +/* + * Copyright 2011-2014 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ +#include <stdio.h> +#include <stdlib.h> +#include <errno.h> +#include <string.h> +#include <ctype.h> +#include <unistd.h> +#include "lmdb.h" + +#define PRINT 1 +#define NOHDR 2 +static int mode; + +static char *subname = NULL; + +static size_t lineno; +static int version; + +static int flags; + +static char *prog; + +static int Eof; + +static MDB_envinfo info; + +static MDB_val kbuf, dbuf; + +#ifdef _WIN32 +#define Z "I" +#else +#define Z "z" +#endif + +#define STRLENOF(s) (sizeof(s)-1) + +typedef struct flagbit { + int bit; + char *name; + int len; +} flagbit; + +#define S(s) s, STRLENOF(s) + +flagbit dbflags[] = { + { MDB_REVERSEKEY, S("reversekey") }, + { MDB_DUPSORT, S("dupsort") }, + { MDB_INTEGERKEY, S("integerkey") }, + { MDB_DUPFIXED, S("dupfixed") }, + { MDB_INTEGERDUP, S("integerdup") }, + { MDB_REVERSEDUP, S("reversedup") }, + { 0, NULL, 0 } +}; + +static void readhdr(void) +{ + char *ptr; + + while (fgets(dbuf.mv_data, dbuf.mv_size, stdin) != NULL) { + lineno++; + if (!strncmp(dbuf.mv_data, "VERSION=", STRLENOF("VERSION="))) { + version=atoi((char *)dbuf.mv_data+STRLENOF("VERSION=")); + if (version > 3) { + fprintf(stderr, "%s: line %" Z "d: unsupported VERSION %d\n", + prog, lineno, version); + exit(EXIT_FAILURE); + } + } else if (!strncmp(dbuf.mv_data, "HEADER=END", STRLENOF("HEADER=END"))) { + break; + } else if (!strncmp(dbuf.mv_data, "format=", STRLENOF("format="))) { + if (!strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "print", STRLENOF("print"))) + mode |= PRINT; + else if (strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "bytevalue", STRLENOF("bytevalue"))) { + fprintf(stderr, "%s: line %" Z "d: unsupported FORMAT %s\n", + prog, lineno, (char *)dbuf.mv_data+STRLENOF("FORMAT=")); + exit(EXIT_FAILURE); + } + } else if (!strncmp(dbuf.mv_data, "database=", STRLENOF("database="))) { + ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size); + if (ptr) *ptr = '\0'; + if (subname) free(subname); + subname = strdup((char *)dbuf.mv_data+STRLENOF("database=")); + } else if (!strncmp(dbuf.mv_data, "type=", STRLENOF("type="))) { + if (strncmp((char *)dbuf.mv_data+STRLENOF("type="), "btree", STRLENOF("btree"))) { + fprintf(stderr, "%s: line %" Z "d: unsupported type %s\n", + prog, lineno, (char *)dbuf.mv_data+STRLENOF("type=")); + exit(EXIT_FAILURE); + } + } else if (!strncmp(dbuf.mv_data, "mapaddr=", STRLENOF("mapaddr="))) { + int i; + ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size); + if (ptr) *ptr = '\0'; + i = sscanf((char *)dbuf.mv_data+STRLENOF("mapaddr="), "%p", &info.me_mapaddr); + if (i != 1) { + fprintf(stderr, "%s: line %" Z "d: invalid mapaddr %s\n", + prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapaddr=")); + exit(EXIT_FAILURE); + } + } else if (!strncmp(dbuf.mv_data, "mapsize=", STRLENOF("mapsize="))) { + int i; + ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size); + if (ptr) *ptr = '\0'; + i = sscanf((char *)dbuf.mv_data+STRLENOF("mapsize="), "%" Z "u", &info.me_mapsize); + if (i != 1) { + fprintf(stderr, "%s: line %" Z "d: invalid mapsize %s\n", + prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapsize=")); + exit(EXIT_FAILURE); + } + } else if (!strncmp(dbuf.mv_data, "maxreaders=", STRLENOF("maxreaders="))) { + int i; + ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size); + if (ptr) *ptr = '\0'; + i = sscanf((char *)dbuf.mv_data+STRLENOF("maxreaders="), "%u", &info.me_maxreaders); + if (i != 1) { + fprintf(stderr, "%s: line %" Z "d: invalid maxreaders %s\n", + prog, lineno, (char *)dbuf.mv_data+STRLENOF("maxreaders=")); + exit(EXIT_FAILURE); + } + } else { + int i; + for (i=0; dbflags[i].bit; i++) { + if (!strncmp(dbuf.mv_data, dbflags[i].name, dbflags[i].len) && + ((char *)dbuf.mv_data)[dbflags[i].len] == '=') { + flags |= dbflags[i].bit; + break; + } + } + if (!dbflags[i].bit) { + ptr = memchr(dbuf.mv_data, '=', dbuf.mv_size); + if (!ptr) { + fprintf(stderr, "%s: line %" Z "d: unexpected format\n", + prog, lineno); + exit(EXIT_FAILURE); + } else { + *ptr = '\0'; + fprintf(stderr, "%s: line %" Z "d: unrecognized keyword ignored: %s\n", + prog, lineno, (char *)dbuf.mv_data); + } + } + } + } +} + +static void badend(void) +{ + fprintf(stderr, "%s: line %" Z "d: unexpected end of input\n", + prog, lineno); +} + +static int unhex(unsigned char *c2) +{ + int x, c; + x = *c2++ & 0x4f; + if (x & 0x40) + x -= 55; + c = x << 4; + x = *c2 & 0x4f; + if (x & 0x40) + x -= 55; + c |= x; + return c; +} + +static int readline(MDB_val *out, MDB_val *buf) +{ + unsigned char *c1, *c2, *end; + size_t len; + int c; + + if (!(mode & NOHDR)) { + c = fgetc(stdin); + if (c == EOF) { + Eof = 1; + return EOF; + } + if (c != ' ') { + lineno++; + if (fgets(buf->mv_data, buf->mv_size, stdin) == NULL) { +badend: + Eof = 1; + badend(); + return EOF; + } + if (c == 'D' && !strncmp(buf->mv_data, "ATA=END", STRLENOF("ATA=END"))) + return EOF; + goto badend; + } + } + if (fgets(buf->mv_data, buf->mv_size, stdin) == NULL) { + Eof = 1; + return EOF; + } + lineno++; + + c1 = buf->mv_data; + len = strlen((char *)c1); + + /* Is buffer too short? */ + while (c1[len-1] != '\n') { + buf->mv_data = realloc(buf->mv_data, buf->mv_size*2); + if (!buf->mv_data) { + Eof = 1; + fprintf(stderr, "%s: line %" Z "d: out of memory, line too long\n", + prog, lineno); + return EOF; + } + c1 = buf->mv_data; + c1 += buf->mv_size; + if (fgets((char *)c1, buf->mv_size, stdin) == NULL) { + Eof = 1; + badend(); + return EOF; + } + buf->mv_size *= 2; + len = strlen((char *)c1); + } + c1 = c2 = buf->mv_data; + len = strlen((char *)c1); + c1[--len] = '\0'; + end = c1 + len; + + if (mode & PRINT) { + while (c2 < end) { + if (*c2 == '\\') { + if (c2[1] == '\\') { + c1++; c2 += 2; + } else { + if (c2+3 > end || !isxdigit(c2[1]) || !isxdigit(c2[2])) { + Eof = 1; + badend(); + return EOF; + } + *c1++ = unhex(++c2); + c2 += 2; + } + } else { + c1++; c2++; + } + } + } else { + /* odd length not allowed */ + if (len & 1) { + Eof = 1; + badend(); + return EOF; + } + while (c2 < end) { + if (!isxdigit(*c2) || !isxdigit(c2[1])) { + Eof = 1; + badend(); + return EOF; + } + *c1++ = unhex(c2); + c2 += 2; + } + } + c2 = out->mv_data = buf->mv_data; + out->mv_size = c1 - c2; + + return 0; +} + +static void usage(void) +{ + fprintf(stderr, "usage: %s dbpath [-V] [-f input] [-n] [-s name] [-N] [-T]\n", prog); + exit(EXIT_FAILURE); +} + +int main(int argc, char *argv[]) +{ + int i, rc; + MDB_env *env; + MDB_txn *txn; + MDB_cursor *mc; + MDB_dbi dbi; + char *envname; + int envflags = 0, putflags = 0; + int dohdr = 0; + + prog = argv[0]; + + if (argc < 2) { + usage(); + } + + /* -f: load file instead of stdin + * -n: use NOSUBDIR flag on env_open + * -s: load into named subDB + * -N: use NOOVERWRITE on puts + * -T: read plaintext + * -V: print version and exit + */ + while ((i = getopt(argc, argv, "f:ns:NTV")) != EOF) { + switch(i) { + case 'V': + printf("%s\n", MDB_VERSION_STRING); + exit(0); + break; + case 'f': + if (freopen(optarg, "r", stdin) == NULL) { + fprintf(stderr, "%s: %s: reopen: %s\n", + prog, optarg, strerror(errno)); + exit(EXIT_FAILURE); + } + break; + case 'n': + envflags |= MDB_NOSUBDIR; + break; + case 's': + subname = strdup(optarg); + break; + case 'N': + putflags = MDB_NOOVERWRITE|MDB_NODUPDATA; + break; + case 'T': + mode |= NOHDR; + break; + default: + usage(); + } + } + + if (optind != argc - 1) + usage(); + + dbuf.mv_size = 4096; + dbuf.mv_data = malloc(dbuf.mv_size); + + if (!(mode & NOHDR)) + readhdr(); + + envname = argv[optind]; + rc = mdb_env_create(&env); + if (rc) { + fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc)); + return EXIT_FAILURE; + } + + mdb_env_set_maxdbs(env, 2); + + if (info.me_maxreaders) + mdb_env_set_maxreaders(env, info.me_maxreaders); + + if (info.me_mapsize) + mdb_env_set_mapsize(env, info.me_mapsize); + + if (info.me_mapaddr) + envflags |= MDB_FIXEDMAP; + + rc = mdb_env_open(env, envname, envflags, 0664); + if (rc) { + fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto env_close; + } + + kbuf.mv_size = mdb_env_get_maxkeysize(env) * 2 + 2; + kbuf.mv_data = malloc(kbuf.mv_size); + + while(!Eof) { + MDB_val key, data; + int batch = 0; + flags = 0; + + if (!dohdr) { + dohdr = 1; + } else if (!(mode & NOHDR)) + readhdr(); + + rc = mdb_txn_begin(env, NULL, 0, &txn); + if (rc) { + fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc)); + goto env_close; + } + + rc = mdb_open(txn, subname, flags|MDB_CREATE, &dbi); + if (rc) { + fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + + rc = mdb_cursor_open(txn, dbi, &mc); + if (rc) { + fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + + while(1) { + rc = readline(&key, &kbuf); + if (rc == EOF) + break; + if (rc) + goto txn_abort; + + rc = readline(&data, &dbuf); + if (rc) + goto txn_abort; + + rc = mdb_cursor_put(mc, &key, &data, putflags); + if (rc == MDB_KEYEXIST && putflags) + continue; + if (rc) + goto txn_abort; + batch++; + if (batch == 100) { + rc = mdb_txn_commit(txn); + if (rc) { + fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n", + prog, lineno, mdb_strerror(rc)); + goto env_close; + } + rc = mdb_txn_begin(env, NULL, 0, &txn); + if (rc) { + fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc)); + goto env_close; + } + rc = mdb_cursor_open(txn, dbi, &mc); + if (rc) { + fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + batch = 0; + } + } + rc = mdb_txn_commit(txn); + txn = NULL; + if (rc) { + fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n", + prog, lineno, mdb_strerror(rc)); + goto env_close; + } + mdb_dbi_close(env, dbi); + } + +txn_abort: + mdb_txn_abort(txn); +env_close: + mdb_env_close(env); + + return rc ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/external/db_drivers/liblmdb32/mdb_stat.1 b/external/db_drivers/liblmdb32/mdb_stat.1 new file mode 100644 index 000000000..3d8d461d9 --- /dev/null +++ b/external/db_drivers/liblmdb32/mdb_stat.1 @@ -0,0 +1,64 @@ +.TH MDB_STAT 1 "2014/06/20" "LMDB 0.9.14" +.\" Copyright 2012-2014 Howard Chu, Symas Corp. All Rights Reserved. +.\" Copying restrictions apply. See COPYRIGHT/LICENSE. +.SH NAME +mdb_stat \- LMDB environment status tool +.SH SYNOPSIS +.B mdb_stat +.BR \ envpath +[\c +.BR \-V ] +[\c +.BR \-e ] +[\c +.BR \-f [ f [ f ]]] +[\c +.BR \-n ] +[\c +.BR \-r [ r ]] +[\c +.BR \-a \ | +.BI \-s \ subdb\fR] +.SH DESCRIPTION +The +.B mdb_stat +utility displays the status of an LMDB environment. +.SH OPTIONS +.TP +.BR \-V +Write the library version number to the standard output, and exit. +.TP +.BR \-e +Display information about the database environment. +.TP +.BR \-f +Display information about the environment freelist. +If \fB\-ff\fP is given, summarize each freelist entry. +If \fB\-fff\fP is given, display the full list of page IDs in the freelist. +.TP +.BR \-n +Display the status of an LMDB database which does not use subdirectories. +.TP +.BR \-r +Display information about the environment reader table. +Shows the process ID, thread ID, and transaction ID for each active +reader slot. The process ID and transaction ID are in decimal, the +thread ID is in hexadecimal. The transaction ID is displayed as "-" +if the reader does not currently have a read transaction open. +If \fB\-rr\fP is given, check for stale entries in the reader +table and clear them. The reader table will be printed again +after the check is performed. +.TP +.BR \-a +Display the status of all of the subdatabases in the environment. +.TP +.BR \-s \ subdb +Display the status of a specific subdatabase. +.SH DIAGNOSTICS +Exit status is zero if no errors occur. +Errors result in a non-zero exit status and +a diagnostic message being written to standard error. +.SH "SEE ALSO" +.BR mdb_copy (1) +.SH AUTHOR +Howard Chu of Symas Corporation <http://www.symas.com> diff --git a/external/db_drivers/liblmdb32/mdb_stat.c b/external/db_drivers/liblmdb32/mdb_stat.c new file mode 100644 index 000000000..1e9229296 --- /dev/null +++ b/external/db_drivers/liblmdb32/mdb_stat.c @@ -0,0 +1,263 @@ +/* mdb_stat.c - memory-mapped database status tool */ +/* + * Copyright 2011-2013 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include "lmdb.h" + +#ifdef _WIN32 +#define Z "I" +#else +#define Z "z" +#endif + +static void prstat(MDB_stat *ms) +{ +#if 0 + printf(" Page size: %u\n", ms->ms_psize); +#endif + printf(" Tree depth: %u\n", ms->ms_depth); + printf(" Branch pages: %"Z"u\n", ms->ms_branch_pages); + printf(" Leaf pages: %"Z"u\n", ms->ms_leaf_pages); + printf(" Overflow pages: %"Z"u\n", ms->ms_overflow_pages); + printf(" Entries: %"Z"u\n", ms->ms_entries); +} + +static void usage(char *prog) +{ + fprintf(stderr, "usage: %s dbpath [-V] [-n] [-e] [-r[r]] [-f[f[f]]] [-a|-s subdb]\n", prog); + exit(EXIT_FAILURE); +} + +int main(int argc, char *argv[]) +{ + int i, rc; + MDB_env *env; + MDB_txn *txn; + MDB_dbi dbi; + MDB_stat mst; + MDB_envinfo mei; + char *prog = argv[0]; + char *envname; + char *subname = NULL; + int alldbs = 0, envinfo = 0, envflags = 0, freinfo = 0, rdrinfo = 0; + + if (argc < 2) { + usage(prog); + } + + /* -a: print stat of main DB and all subDBs + * -s: print stat of only the named subDB + * -e: print env info + * -f: print freelist info + * -r: print reader info + * -n: use NOSUBDIR flag on env_open + * -V: print version and exit + * (default) print stat of only the main DB + */ + while ((i = getopt(argc, argv, "Vaefnrs:")) != EOF) { + switch(i) { + case 'V': + printf("%s\n", MDB_VERSION_STRING); + exit(0); + break; + case 'a': + if (subname) + usage(prog); + alldbs++; + break; + case 'e': + envinfo++; + break; + case 'f': + freinfo++; + break; + case 'n': + envflags |= MDB_NOSUBDIR; + break; + case 'r': + rdrinfo++; + break; + case 's': + if (alldbs) + usage(prog); + subname = optarg; + break; + default: + usage(prog); + } + } + + if (optind != argc - 1) + usage(prog); + + envname = argv[optind]; + rc = mdb_env_create(&env); + if (rc) { + fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc)); + return EXIT_FAILURE; + } + + if (alldbs || subname) { + mdb_env_set_maxdbs(env, 4); + } + + rc = mdb_env_open(env, envname, envflags | MDB_RDONLY, 0664); + if (rc) { + fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto env_close; + } + + if (envinfo) { + (void)mdb_env_stat(env, &mst); + (void)mdb_env_info(env, &mei); + printf("Environment Info\n"); + printf(" Map address: %p\n", mei.me_mapaddr); + printf(" Map size: %"Z"u\n", mei.me_mapsize); + printf(" Page size: %u\n", mst.ms_psize); + printf(" Max pages: %"Z"u\n", mei.me_mapsize / mst.ms_psize); + printf(" Number of pages used: %"Z"u\n", mei.me_last_pgno+1); + printf(" Last transaction ID: %"Z"u\n", mei.me_last_txnid); + printf(" Max readers: %u\n", mei.me_maxreaders); + printf(" Number of readers used: %u\n", mei.me_numreaders); + } + + if (rdrinfo) { + printf("Reader Table Status\n"); + rc = mdb_reader_list(env, (MDB_msg_func *)fputs, stdout); + if (rdrinfo > 1) { + int dead; + mdb_reader_check(env, &dead); + printf(" %d stale readers cleared.\n", dead); + rc = mdb_reader_list(env, (MDB_msg_func *)fputs, stdout); + } + if (!(subname || alldbs || freinfo)) + goto env_close; + } + + rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + if (rc) { + fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc)); + goto env_close; + } + + if (freinfo) { + MDB_cursor *cursor; + MDB_val key, data; + size_t pages = 0, *iptr; + + printf("Freelist Status\n"); + dbi = 0; + rc = mdb_cursor_open(txn, dbi, &cursor); + if (rc) { + fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + rc = mdb_stat(txn, dbi, &mst); + if (rc) { + fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + prstat(&mst); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + iptr = data.mv_data; + pages += *iptr; + if (freinfo > 1) { + char *bad = ""; + size_t pg, prev; + ssize_t i, j, span = 0; + j = *iptr++; + for (i = j, prev = 1; --i >= 0; ) { + pg = iptr[i]; + if (pg <= prev) + bad = " [bad sequence]"; + prev = pg; + pg += span; + for (; i >= span && iptr[i-span] == pg; span++, pg++) ; + } + printf(" Transaction %"Z"u, %"Z"d pages, maxspan %"Z"d%s\n", + *(size_t *)key.mv_data, j, span, bad); + if (freinfo > 2) { + for (--j; j >= 0; ) { + pg = iptr[j]; + for (span=1; --j >= 0 && iptr[j] == pg+span; span++) ; + printf(span>1 ? " %9"Z"u[%"Z"d]\n" : " %9"Z"u\n", + pg, span); + } + } + } + } + mdb_cursor_close(cursor); + printf(" Free pages: %"Z"u\n", pages); + } + + rc = mdb_open(txn, subname, 0, &dbi); + if (rc) { + fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + + rc = mdb_stat(txn, dbi, &mst); + if (rc) { + fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + printf("Status of %s\n", subname ? subname : "Main DB"); + prstat(&mst); + + if (alldbs) { + MDB_cursor *cursor; + MDB_val key; + + rc = mdb_cursor_open(txn, dbi, &cursor); + if (rc) { + fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + while ((rc = mdb_cursor_get(cursor, &key, NULL, MDB_NEXT_NODUP)) == 0) { + char *str; + MDB_dbi db2; + if (memchr(key.mv_data, '\0', key.mv_size)) + continue; + str = malloc(key.mv_size+1); + memcpy(str, key.mv_data, key.mv_size); + str[key.mv_size] = '\0'; + rc = mdb_open(txn, str, 0, &db2); + if (rc == MDB_SUCCESS) + printf("Status of %s\n", str); + free(str); + if (rc) continue; + rc = mdb_stat(txn, db2, &mst); + if (rc) { + fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + prstat(&mst); + mdb_close(env, db2); + } + mdb_cursor_close(cursor); + } + + if (rc == MDB_NOTFOUND) + rc = MDB_SUCCESS; + + mdb_close(env, dbi); +txn_abort: + mdb_txn_abort(txn); +env_close: + mdb_env_close(env); + + return rc ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/external/db_drivers/liblmdb32/midl.c b/external/db_drivers/liblmdb32/midl.c new file mode 100644 index 000000000..88a3aff10 --- /dev/null +++ b/external/db_drivers/liblmdb32/midl.c @@ -0,0 +1,360 @@ +/** @file midl.c + * @brief ldap bdb back-end ID List functions */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software <http://www.openldap.org/>. + * + * Copyright 2000-2014 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ + +#include <limits.h> +#include <string.h> +#include <stdlib.h> +#include <errno.h> +#include <sys/types.h> +#include "midl.h" + +/** @defgroup internal LMDB Internals + * @{ + */ +/** @defgroup idls ID List Management + * @{ + */ +#define CMP(x,y) ( (x) < (y) ? -1 : (x) > (y) ) + +unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id ) +{ + /* + * binary search of id in ids + * if found, returns position of id + * if not found, returns first position greater than id + */ + unsigned base = 0; + unsigned cursor = 1; + int val = 0; + unsigned n = ids[0]; + + while( 0 < n ) { + unsigned pivot = n >> 1; + cursor = base + pivot + 1; + val = CMP( ids[cursor], id ); + + if( val < 0 ) { + n = pivot; + + } else if ( val > 0 ) { + base = cursor; + n -= pivot + 1; + + } else { + return cursor; + } + } + + if( val > 0 ) { + ++cursor; + } + return cursor; +} + +#if 0 /* superseded by append/sort */ +int mdb_midl_insert( MDB_IDL ids, MDB_ID id ) +{ + unsigned x, i; + + x = mdb_midl_search( ids, id ); + assert( x > 0 ); + + if( x < 1 ) { + /* internal error */ + return -2; + } + + if ( x <= ids[0] && ids[x] == id ) { + /* duplicate */ + assert(0); + return -1; + } + + if ( ++ids[0] >= MDB_IDL_DB_MAX ) { + /* no room */ + --ids[0]; + return -2; + + } else { + /* insert id */ + for (i=ids[0]; i>x; i--) + ids[i] = ids[i-1]; + ids[x] = id; + } + + return 0; +} +#endif + +MDB_IDL mdb_midl_alloc(int num) +{ + MDB_IDL ids = malloc((num+2) * sizeof(MDB_ID)); + if (ids) { + *ids++ = num; + *ids = 0; + } + return ids; +} + +void mdb_midl_free(MDB_IDL ids) +{ + if (ids) + free(ids-1); +} + +int mdb_midl_shrink( MDB_IDL *idp ) +{ + MDB_IDL ids = *idp; + if (*(--ids) > MDB_IDL_UM_MAX && + (ids = realloc(ids, (MDB_IDL_UM_MAX+1) * sizeof(MDB_ID)))) + { + *ids++ = MDB_IDL_UM_MAX; + *idp = ids; + return 1; + } + return 0; +} + +static int mdb_midl_grow( MDB_IDL *idp, int num ) +{ + MDB_IDL idn = *idp-1; + /* grow it */ + idn = realloc(idn, (*idn + num + 2) * sizeof(MDB_ID)); + if (!idn) + return ENOMEM; + *idn++ += num; + *idp = idn; + return 0; +} + +int mdb_midl_need( MDB_IDL *idp, unsigned num ) +{ + MDB_IDL ids = *idp; + num += ids[0]; + if (num > ids[-1]) { + num = (num + num/4 + (256 + 2)) & -256; + if (!(ids = realloc(ids-1, num * sizeof(MDB_ID)))) + return ENOMEM; + *ids++ = num - 2; + *idp = ids; + } + return 0; +} + +int mdb_midl_append( MDB_IDL *idp, MDB_ID id ) +{ + MDB_IDL ids = *idp; + /* Too big? */ + if (ids[0] >= ids[-1]) { + if (mdb_midl_grow(idp, MDB_IDL_UM_MAX)) + return ENOMEM; + ids = *idp; + } + ids[0]++; + ids[ids[0]] = id; + return 0; +} + +int mdb_midl_append_list( MDB_IDL *idp, MDB_IDL app ) +{ + MDB_IDL ids = *idp; + /* Too big? */ + if (ids[0] + app[0] >= ids[-1]) { + if (mdb_midl_grow(idp, app[0])) + return ENOMEM; + ids = *idp; + } + memcpy(&ids[ids[0]+1], &app[1], app[0] * sizeof(MDB_ID)); + ids[0] += app[0]; + return 0; +} + +int mdb_midl_append_range( MDB_IDL *idp, MDB_ID id, unsigned n ) +{ + MDB_ID *ids = *idp, len = ids[0]; + /* Too big? */ + if (len + n > ids[-1]) { + if (mdb_midl_grow(idp, n | MDB_IDL_UM_MAX)) + return ENOMEM; + ids = *idp; + } + ids[0] = len + n; + ids += len; + while (n) + ids[n--] = id++; + return 0; +} + +void mdb_midl_xmerge( MDB_IDL idl, MDB_IDL merge ) +{ + MDB_ID old_id, merge_id, i = merge[0], j = idl[0], k = i+j, total = k; + idl[0] = (MDB_ID)-1; /* delimiter for idl scan below */ + old_id = idl[j]; + while (i) { + merge_id = merge[i--]; + for (; old_id < merge_id; old_id = idl[--j]) + idl[k--] = old_id; + idl[k--] = merge_id; + } + idl[0] = total; +} + +/* Quicksort + Insertion sort for small arrays */ + +#define SMALL 8 +#define MIDL_SWAP(a,b) { itmp=(a); (a)=(b); (b)=itmp; } + +void +mdb_midl_sort( MDB_IDL ids ) +{ + /* Max possible depth of int-indexed tree * 2 items/level */ + int istack[sizeof(int)*CHAR_BIT * 2]; + int i,j,k,l,ir,jstack; + MDB_ID a, itmp; + + ir = (int)ids[0]; + l = 1; + jstack = 0; + for(;;) { + if (ir - l < SMALL) { /* Insertion sort */ + for (j=l+1;j<=ir;j++) { + a = ids[j]; + for (i=j-1;i>=1;i--) { + if (ids[i] >= a) break; + ids[i+1] = ids[i]; + } + ids[i+1] = a; + } + if (jstack == 0) break; + ir = istack[jstack--]; + l = istack[jstack--]; + } else { + k = (l + ir) >> 1; /* Choose median of left, center, right */ + MIDL_SWAP(ids[k], ids[l+1]); + if (ids[l] < ids[ir]) { + MIDL_SWAP(ids[l], ids[ir]); + } + if (ids[l+1] < ids[ir]) { + MIDL_SWAP(ids[l+1], ids[ir]); + } + if (ids[l] < ids[l+1]) { + MIDL_SWAP(ids[l], ids[l+1]); + } + i = l+1; + j = ir; + a = ids[l+1]; + for(;;) { + do i++; while(ids[i] > a); + do j--; while(ids[j] < a); + if (j < i) break; + MIDL_SWAP(ids[i],ids[j]); + } + ids[l+1] = ids[j]; + ids[j] = a; + jstack += 2; + if (ir-i+1 >= j-l) { + istack[jstack] = ir; + istack[jstack-1] = i; + ir = j-1; + } else { + istack[jstack] = j-1; + istack[jstack-1] = l; + l = i; + } + } + } +} + +unsigned mdb_mid2l_search( MDB_ID2L ids, MDB_ID id ) +{ + /* + * binary search of id in ids + * if found, returns position of id + * if not found, returns first position greater than id + */ + unsigned base = 0; + unsigned cursor = 1; + int val = 0; + unsigned n = (unsigned)ids[0].mid; + + while( 0 < n ) { + unsigned pivot = n >> 1; + cursor = base + pivot + 1; + val = CMP( id, ids[cursor].mid ); + + if( val < 0 ) { + n = pivot; + + } else if ( val > 0 ) { + base = cursor; + n -= pivot + 1; + + } else { + return cursor; + } + } + + if( val > 0 ) { + ++cursor; + } + return cursor; +} + +int mdb_mid2l_insert( MDB_ID2L ids, MDB_ID2 *id ) +{ + unsigned x, i; + + x = mdb_mid2l_search( ids, id->mid ); + + if( x < 1 ) { + /* internal error */ + return -2; + } + + if ( x <= ids[0].mid && ids[x].mid == id->mid ) { + /* duplicate */ + return -1; + } + + if ( ids[0].mid >= MDB_IDL_UM_MAX ) { + /* too big */ + return -2; + + } else { + /* insert id */ + ids[0].mid++; + for (i=(unsigned)ids[0].mid; i>x; i--) + ids[i] = ids[i-1]; + ids[x] = *id; + } + + return 0; +} + +int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id ) +{ + /* Too big? */ + if (ids[0].mid >= MDB_IDL_UM_MAX) { + return -2; + } + ids[0].mid++; + ids[ids[0].mid] = *id; + return 0; +} + +/** @} */ +/** @} */ diff --git a/external/db_drivers/liblmdb32/midl.h b/external/db_drivers/liblmdb32/midl.h new file mode 100644 index 000000000..f23d7d3f3 --- /dev/null +++ b/external/db_drivers/liblmdb32/midl.h @@ -0,0 +1,190 @@ +/** @file midl.h + * @brief LMDB ID List header file. + * + * This file was originally part of back-bdb but has been + * modified for use in libmdb. Most of the macros defined + * in this file are unused, just left over from the original. + * + * This file is only used internally in libmdb and its definitions + * are not exposed publicly. + */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software <http://www.openldap.org/>. + * + * Copyright 2000-2014 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ + +#ifndef _MDB_MIDL_H_ +#define _MDB_MIDL_H_ + +#include <stddef.h> + +#ifdef __cplusplus +extern "C" { +#endif + +/** @defgroup internal LMDB Internals + * @{ + */ + +/** @defgroup idls ID List Management + * @{ + */ + /** A generic unsigned ID number. These were entryIDs in back-bdb. + * Preferably it should have the same size as a pointer. + */ +typedef size_t MDB_ID; + + /** An IDL is an ID List, a sorted array of IDs. The first + * element of the array is a counter for how many actual + * IDs are in the list. In the original back-bdb code, IDLs are + * sorted in ascending order. For libmdb IDLs are sorted in + * descending order. + */ +typedef MDB_ID *MDB_IDL; + +/* IDL sizes - likely should be even bigger + * limiting factors: sizeof(ID), thread stack size + */ +#ifdef VL32 +#define MDB_IDL_LOGN 10 /* DB_SIZE is 2^10, UM_SIZE is 2^11 */ +#else +#define MDB_IDL_LOGN 16 /* DB_SIZE is 2^16, UM_SIZE is 2^17 */ +#endif +#define MDB_IDL_DB_SIZE (1<<MDB_IDL_LOGN) +#define MDB_IDL_UM_SIZE (1<<(MDB_IDL_LOGN+1)) + +#define MDB_IDL_DB_MAX (MDB_IDL_DB_SIZE-1) +#define MDB_IDL_UM_MAX (MDB_IDL_UM_SIZE-1) + +#define MDB_IDL_SIZEOF(ids) (((ids)[0]+1) * sizeof(MDB_ID)) +#define MDB_IDL_IS_ZERO(ids) ( (ids)[0] == 0 ) +#define MDB_IDL_CPY( dst, src ) (memcpy( dst, src, MDB_IDL_SIZEOF( src ) )) +#define MDB_IDL_FIRST( ids ) ( (ids)[1] ) +#define MDB_IDL_LAST( ids ) ( (ids)[(ids)[0]] ) + + /** Current max length of an #mdb_midl_alloc()ed IDL */ +#define MDB_IDL_ALLOCLEN( ids ) ( (ids)[-1] ) + + /** Append ID to IDL. The IDL must be big enough. */ +#define mdb_midl_xappend(idl, id) do { \ + MDB_ID *xidl = (idl), xlen = ++(xidl[0]); \ + xidl[xlen] = (id); \ + } while (0) + + /** Search for an ID in an IDL. + * @param[in] ids The IDL to search. + * @param[in] id The ID to search for. + * @return The index of the first ID greater than or equal to \b id. + */ +unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id ); + + /** Allocate an IDL. + * Allocates memory for an IDL of the given size. + * @return IDL on success, NULL on failure. + */ +MDB_IDL mdb_midl_alloc(int num); + + /** Free an IDL. + * @param[in] ids The IDL to free. + */ +void mdb_midl_free(MDB_IDL ids); + + /** Shrink an IDL. + * Return the IDL to the default size if it has grown larger. + * @param[in,out] idp Address of the IDL to shrink. + * @return 0 on no change, non-zero if shrunk. + */ +int mdb_midl_shrink(MDB_IDL *idp); + + /** Make room for num additional elements in an IDL. + * @param[in,out] idp Address of the IDL. + * @param[in] num Number of elements to make room for. + * @return 0 on success, ENOMEM on failure. + */ +int mdb_midl_need(MDB_IDL *idp, unsigned num); + + /** Append an ID onto an IDL. + * @param[in,out] idp Address of the IDL to append to. + * @param[in] id The ID to append. + * @return 0 on success, ENOMEM if the IDL is too large. + */ +int mdb_midl_append( MDB_IDL *idp, MDB_ID id ); + + /** Append an IDL onto an IDL. + * @param[in,out] idp Address of the IDL to append to. + * @param[in] app The IDL to append. + * @return 0 on success, ENOMEM if the IDL is too large. + */ +int mdb_midl_append_list( MDB_IDL *idp, MDB_IDL app ); + + /** Append an ID range onto an IDL. + * @param[in,out] idp Address of the IDL to append to. + * @param[in] id The lowest ID to append. + * @param[in] n Number of IDs to append. + * @return 0 on success, ENOMEM if the IDL is too large. + */ +int mdb_midl_append_range( MDB_IDL *idp, MDB_ID id, unsigned n ); + + /** Merge an IDL onto an IDL. The destination IDL must be big enough. + * @param[in] idl The IDL to merge into. + * @param[in] merge The IDL to merge. + */ +void mdb_midl_xmerge( MDB_IDL idl, MDB_IDL merge ); + + /** Sort an IDL. + * @param[in,out] ids The IDL to sort. + */ +void mdb_midl_sort( MDB_IDL ids ); + + /** An ID2 is an ID/pointer pair. + */ +typedef struct MDB_ID2 { + MDB_ID mid; /**< The ID */ + void *mptr; /**< The pointer */ +} MDB_ID2; + + /** An ID2L is an ID2 List, a sorted array of ID2s. + * The first element's \b mid member is a count of how many actual + * elements are in the array. The \b mptr member of the first element is unused. + * The array is sorted in ascending order by \b mid. + */ +typedef MDB_ID2 *MDB_ID2L; + + /** Search for an ID in an ID2L. + * @param[in] ids The ID2L to search. + * @param[in] id The ID to search for. + * @return The index of the first ID2 whose \b mid member is greater than or equal to \b id. + */ +unsigned mdb_mid2l_search( MDB_ID2L ids, MDB_ID id ); + + + /** Insert an ID2 into a ID2L. + * @param[in,out] ids The ID2L to insert into. + * @param[in] id The ID2 to insert. + * @return 0 on success, -1 if the ID was already present in the ID2L. + */ +int mdb_mid2l_insert( MDB_ID2L ids, MDB_ID2 *id ); + + /** Append an ID2 into a ID2L. + * @param[in,out] ids The ID2L to append into. + * @param[in] id The ID2 to append. + * @return 0 on success, -2 if the ID2L is too big. + */ +int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id ); + +/** @} */ +/** @} */ +#ifdef __cplusplus +} +#endif +#endif /* _MDB_MIDL_H_ */ diff --git a/external/db_drivers/liblmdb32/mtest.c b/external/db_drivers/liblmdb32/mtest.c new file mode 100644 index 000000000..2d4e5e832 --- /dev/null +++ b/external/db_drivers/liblmdb32/mtest.c @@ -0,0 +1,173 @@ +/* mtest.c - memory-mapped database tester/toy */ +/* + * Copyright 2011-2014 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ +#include <stdio.h> +#include <stdlib.h> +#include <time.h> +#include "lmdb.h" + +#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) +#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) +#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \ + "%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort())) + +int main(int argc,char * argv[]) +{ + int i = 0, j = 0, rc; + MDB_env *env; + MDB_dbi dbi; + MDB_val key, data; + MDB_txn *txn; + MDB_stat mst; + MDB_cursor *cursor, *cur2; + MDB_cursor_op op; + int count; + int *values; + char sval[32] = ""; + + srand(time(NULL)); + + count = (rand()%384) + 64; + values = (int *)malloc(count*sizeof(int)); + + for(i = 0;i<count;i++) { + values[i] = rand()%1024; + } + + E(mdb_env_create(&env)); + E(mdb_env_set_mapsize(env, 10485760)); + E(mdb_env_open(env, "./testdb", 0 /* MDB_FIXEDMAP |MDB_NOSYNC*/, 0664)); + E(mdb_txn_begin(env, NULL, 0, &txn)); + E(mdb_open(txn, NULL, 0, &dbi)); + + key.mv_size = sizeof(int); + key.mv_data = sval; + data.mv_size = sizeof(sval); + data.mv_data = sval; + + printf("Adding %d values\n", count); + for (i=0;i<count;i++) { + sprintf(sval, "%03x %d foo bar", values[i], values[i]); + if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) { + j++; + data.mv_size = sizeof(sval); + data.mv_data = sval; + } + } + if (j) printf("%d duplicates skipped\n", j); + E(mdb_txn_commit(txn)); + E(mdb_env_stat(env, &mst)); + + E(mdb_txn_begin(env, NULL, 1, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %p %.*s, data: %p %.*s\n", + key.mv_data, (int) key.mv_size, (char *) key.mv_data, + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + mdb_cursor_close(cursor); + mdb_txn_abort(txn); + + j=0; + key.mv_data = sval; + for (i= count - 1; i > -1; i-= (rand()%5)) { + j++; + txn=NULL; + E(mdb_txn_begin(env, NULL, 0, &txn)); + sprintf(sval, "%03x ", values[i]); + if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) { + j--; + mdb_txn_abort(txn); + } else { + E(mdb_txn_commit(txn)); + } + } + free(values); + printf("Deleted %d values\n", j); + + E(mdb_env_stat(env, &mst)); + E(mdb_txn_begin(env, NULL, 1, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + printf("Cursor next\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + printf("Cursor last\n"); + E(mdb_cursor_get(cursor, &key, &data, MDB_LAST)); + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + printf("Cursor prev\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + printf("Cursor last/prev\n"); + E(mdb_cursor_get(cursor, &key, &data, MDB_LAST)); + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + E(mdb_cursor_get(cursor, &key, &data, MDB_PREV)); + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + + mdb_txn_abort(txn); + + printf("Deleting with cursor\n"); + E(mdb_txn_begin(env, NULL, 0, &txn)); + E(mdb_cursor_open(txn, dbi, &cur2)); + for (i=0; i<50; i++) { + if (RES(MDB_NOTFOUND, mdb_cursor_get(cur2, &key, &data, MDB_NEXT))) + break; + printf("key: %p %.*s, data: %p %.*s\n", + key.mv_data, (int) key.mv_size, (char *) key.mv_data, + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + E(mdb_del(txn, dbi, &key, NULL)); + } + + printf("Restarting cursor in txn\n"); + for (op=MDB_FIRST, i=0; i<=32; op=MDB_NEXT, i++) { + if (RES(MDB_NOTFOUND, mdb_cursor_get(cur2, &key, &data, op))) + break; + printf("key: %p %.*s, data: %p %.*s\n", + key.mv_data, (int) key.mv_size, (char *) key.mv_data, + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + } + mdb_cursor_close(cur2); + E(mdb_txn_commit(txn)); + + printf("Restarting cursor outside txn\n"); + E(mdb_txn_begin(env, NULL, 0, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + for (op=MDB_FIRST, i=0; i<=32; op=MDB_NEXT, i++) { + if (RES(MDB_NOTFOUND, mdb_cursor_get(cursor, &key, &data, op))) + break; + printf("key: %p %.*s, data: %p %.*s\n", + key.mv_data, (int) key.mv_size, (char *) key.mv_data, + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + } + mdb_cursor_close(cursor); + mdb_close(env, dbi); + + mdb_txn_abort(txn); + mdb_env_close(env); + + return 0; +} diff --git a/external/db_drivers/liblmdb32/mtest2.c b/external/db_drivers/liblmdb32/mtest2.c new file mode 100644 index 000000000..f1a3dbd6c --- /dev/null +++ b/external/db_drivers/liblmdb32/mtest2.c @@ -0,0 +1,123 @@ +/* mtest2.c - memory-mapped database tester/toy */ +/* + * Copyright 2011-2014 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ + +/* Just like mtest.c, but using a subDB instead of the main DB */ + +#include <stdio.h> +#include <stdlib.h> +#include <time.h> +#include "lmdb.h" + +#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) +#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) +#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \ + "%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort())) + +int main(int argc,char * argv[]) +{ + int i = 0, j = 0, rc; + MDB_env *env; + MDB_dbi dbi; + MDB_val key, data; + MDB_txn *txn; + MDB_stat mst; + MDB_cursor *cursor; + int count; + int *values; + char sval[32] = ""; + + srand(time(NULL)); + + count = (rand()%384) + 64; + values = (int *)malloc(count*sizeof(int)); + + for(i = 0;i<count;i++) { + values[i] = rand()%1024; + } + + E(mdb_env_create(&env)); + E(mdb_env_set_mapsize(env, 10485760)); + E(mdb_env_set_maxdbs(env, 4)); + E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664)); + E(mdb_txn_begin(env, NULL, 0, &txn)); + E(mdb_open(txn, "id1", MDB_CREATE, &dbi)); + + key.mv_size = sizeof(int); + key.mv_data = sval; + data.mv_size = sizeof(sval); + data.mv_data = sval; + + printf("Adding %d values\n", count); + for (i=0;i<count;i++) { + sprintf(sval, "%03x %d foo bar", values[i], values[i]); + if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) + j++; + } + if (j) printf("%d duplicates skipped\n", j); + E(mdb_txn_commit(txn)); + E(mdb_env_stat(env, &mst)); + + E(mdb_txn_begin(env, NULL, 1, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %p %.*s, data: %p %.*s\n", + key.mv_data, (int) key.mv_size, (char *) key.mv_data, + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + mdb_cursor_close(cursor); + mdb_txn_abort(txn); + + j=0; + key.mv_data = sval; + for (i= count - 1; i > -1; i-= (rand()%5)) { + j++; + txn=NULL; + E(mdb_txn_begin(env, NULL, 0, &txn)); + sprintf(sval, "%03x ", values[i]); + if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) { + j--; + mdb_txn_abort(txn); + } else { + E(mdb_txn_commit(txn)); + } + } + free(values); + printf("Deleted %d values\n", j); + + E(mdb_env_stat(env, &mst)); + E(mdb_txn_begin(env, NULL, 1, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + printf("Cursor next\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + printf("Cursor prev\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + mdb_cursor_close(cursor); + mdb_close(env, dbi); + + mdb_txn_abort(txn); + mdb_env_close(env); + + return 0; +} diff --git a/external/db_drivers/liblmdb32/mtest3.c b/external/db_drivers/liblmdb32/mtest3.c new file mode 100644 index 000000000..f705c52de --- /dev/null +++ b/external/db_drivers/liblmdb32/mtest3.c @@ -0,0 +1,133 @@ +/* mtest3.c - memory-mapped database tester/toy */ +/* + * Copyright 2011-2014 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ + +/* Tests for sorted duplicate DBs */ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include "lmdb.h" + +#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) +#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) +#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \ + "%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort())) + +int main(int argc,char * argv[]) +{ + int i = 0, j = 0, rc; + MDB_env *env; + MDB_dbi dbi; + MDB_val key, data; + MDB_txn *txn; + MDB_stat mst; + MDB_cursor *cursor; + int count; + int *values; + char sval[32]; + char kval[sizeof(int)]; + + srand(time(NULL)); + + memset(sval, 0, sizeof(sval)); + + count = (rand()%384) + 64; + values = (int *)malloc(count*sizeof(int)); + + for(i = 0;i<count;i++) { + values[i] = rand()%1024; + } + + E(mdb_env_create(&env)); + E(mdb_env_set_mapsize(env, 10485760)); + E(mdb_env_set_maxdbs(env, 4)); + E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664)); + E(mdb_txn_begin(env, NULL, 0, &txn)); + E(mdb_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi)); + + key.mv_size = sizeof(int); + key.mv_data = kval; + data.mv_size = sizeof(sval); + data.mv_data = sval; + + printf("Adding %d values\n", count); + for (i=0;i<count;i++) { + if (!(i & 0x0f)) + sprintf(kval, "%03x", values[i]); + sprintf(sval, "%03x %d foo bar", values[i], values[i]); + if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA))) + j++; + } + if (j) printf("%d duplicates skipped\n", j); + E(mdb_txn_commit(txn)); + E(mdb_env_stat(env, &mst)); + + E(mdb_txn_begin(env, NULL, 1, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %p %.*s, data: %p %.*s\n", + key.mv_data, (int) key.mv_size, (char *) key.mv_data, + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + mdb_cursor_close(cursor); + mdb_txn_abort(txn); + + j=0; + + for (i= count - 1; i > -1; i-= (rand()%5)) { + j++; + txn=NULL; + E(mdb_txn_begin(env, NULL, 0, &txn)); + sprintf(kval, "%03x", values[i & ~0x0f]); + sprintf(sval, "%03x %d foo bar", values[i], values[i]); + key.mv_size = sizeof(int); + key.mv_data = kval; + data.mv_size = sizeof(sval); + data.mv_data = sval; + if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) { + j--; + mdb_txn_abort(txn); + } else { + E(mdb_txn_commit(txn)); + } + } + free(values); + printf("Deleted %d values\n", j); + + E(mdb_env_stat(env, &mst)); + E(mdb_txn_begin(env, NULL, 1, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + printf("Cursor next\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + printf("Cursor prev\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + mdb_cursor_close(cursor); + mdb_close(env, dbi); + + mdb_txn_abort(txn); + mdb_env_close(env); + + return 0; +} diff --git a/external/db_drivers/liblmdb32/mtest4.c b/external/db_drivers/liblmdb32/mtest4.c new file mode 100644 index 000000000..da5a95304 --- /dev/null +++ b/external/db_drivers/liblmdb32/mtest4.c @@ -0,0 +1,168 @@ +/* mtest4.c - memory-mapped database tester/toy */ +/* + * Copyright 2011-2014 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ + +/* Tests for sorted duplicate DBs with fixed-size keys */ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include "lmdb.h" + +#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) +#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) +#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \ + "%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort())) + +int main(int argc,char * argv[]) +{ + int i = 0, j = 0, rc; + MDB_env *env; + MDB_dbi dbi; + MDB_val key, data; + MDB_txn *txn; + MDB_stat mst; + MDB_cursor *cursor; + int count; + int *values; + char sval[8]; + char kval[sizeof(int)]; + + memset(sval, 0, sizeof(sval)); + + count = 510; + values = (int *)malloc(count*sizeof(int)); + + for(i = 0;i<count;i++) { + values[i] = i*5; + } + + E(mdb_env_create(&env)); + E(mdb_env_set_mapsize(env, 10485760)); + E(mdb_env_set_maxdbs(env, 4)); + E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664)); + E(mdb_txn_begin(env, NULL, 0, &txn)); + E(mdb_open(txn, "id4", MDB_CREATE|MDB_DUPSORT|MDB_DUPFIXED, &dbi)); + + key.mv_size = sizeof(int); + key.mv_data = kval; + data.mv_size = sizeof(sval); + data.mv_data = sval; + + printf("Adding %d values\n", count); + strcpy(kval, "001"); + for (i=0;i<count;i++) { + sprintf(sval, "%07x", values[i]); + if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA))) + j++; + } + if (j) printf("%d duplicates skipped\n", j); + E(mdb_txn_commit(txn)); + E(mdb_env_stat(env, &mst)); + + /* there should be one full page of dups now. + */ + E(mdb_txn_begin(env, NULL, 1, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %p %.*s, data: %p %.*s\n", + key.mv_data, (int) key.mv_size, (char *) key.mv_data, + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + mdb_cursor_close(cursor); + mdb_txn_abort(txn); + + /* test all 3 branches of split code: + * 1: new key in lower half + * 2: new key at split point + * 3: new key in upper half + */ + + key.mv_size = sizeof(int); + key.mv_data = kval; + data.mv_size = sizeof(sval); + data.mv_data = sval; + + sprintf(sval, "%07x", values[3]+1); + E(mdb_txn_begin(env, NULL, 0, &txn)); + (void)RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA)); + mdb_txn_abort(txn); + + sprintf(sval, "%07x", values[255]+1); + E(mdb_txn_begin(env, NULL, 0, &txn)); + (void)RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA)); + mdb_txn_abort(txn); + + sprintf(sval, "%07x", values[500]+1); + E(mdb_txn_begin(env, NULL, 0, &txn)); + (void)RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA)); + E(mdb_txn_commit(txn)); + + /* Try MDB_NEXT_MULTIPLE */ + E(mdb_txn_begin(env, NULL, 0, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT_MULTIPLE)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + mdb_cursor_close(cursor); + mdb_txn_abort(txn); + j=0; + + for (i= count - 1; i > -1; i-= (rand()%3)) { + j++; + txn=NULL; + E(mdb_txn_begin(env, NULL, 0, &txn)); + sprintf(sval, "%07x", values[i]); + key.mv_size = sizeof(int); + key.mv_data = kval; + data.mv_size = sizeof(sval); + data.mv_data = sval; + if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) { + j--; + mdb_txn_abort(txn); + } else { + E(mdb_txn_commit(txn)); + } + } + free(values); + printf("Deleted %d values\n", j); + + E(mdb_env_stat(env, &mst)); + E(mdb_txn_begin(env, NULL, 1, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + printf("Cursor next\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + printf("Cursor prev\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + mdb_cursor_close(cursor); + mdb_close(env, dbi); + + mdb_txn_abort(txn); + mdb_env_close(env); + + return 0; +} diff --git a/external/db_drivers/liblmdb32/mtest5.c b/external/db_drivers/liblmdb32/mtest5.c new file mode 100644 index 000000000..39a8c728a --- /dev/null +++ b/external/db_drivers/liblmdb32/mtest5.c @@ -0,0 +1,135 @@ +/* mtest5.c - memory-mapped database tester/toy */ +/* + * Copyright 2011-2014 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ + +/* Tests for sorted duplicate DBs using cursor_put */ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include "lmdb.h" + +#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) +#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) +#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \ + "%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort())) + +int main(int argc,char * argv[]) +{ + int i = 0, j = 0, rc; + MDB_env *env; + MDB_dbi dbi; + MDB_val key, data; + MDB_txn *txn; + MDB_stat mst; + MDB_cursor *cursor; + int count; + int *values; + char sval[32]; + char kval[sizeof(int)]; + + srand(time(NULL)); + + memset(sval, 0, sizeof(sval)); + + count = (rand()%384) + 64; + values = (int *)malloc(count*sizeof(int)); + + for(i = 0;i<count;i++) { + values[i] = rand()%1024; + } + + E(mdb_env_create(&env)); + E(mdb_env_set_mapsize(env, 10485760)); + E(mdb_env_set_maxdbs(env, 4)); + E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664)); + E(mdb_txn_begin(env, NULL, 0, &txn)); + E(mdb_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi)); + E(mdb_cursor_open(txn, dbi, &cursor)); + + key.mv_size = sizeof(int); + key.mv_data = kval; + data.mv_size = sizeof(sval); + data.mv_data = sval; + + printf("Adding %d values\n", count); + for (i=0;i<count;i++) { + if (!(i & 0x0f)) + sprintf(kval, "%03x", values[i]); + sprintf(sval, "%03x %d foo bar", values[i], values[i]); + if (RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NODUPDATA))) + j++; + } + if (j) printf("%d duplicates skipped\n", j); + mdb_cursor_close(cursor); + E(mdb_txn_commit(txn)); + E(mdb_env_stat(env, &mst)); + + E(mdb_txn_begin(env, NULL, 1, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %p %.*s, data: %p %.*s\n", + key.mv_data, (int) key.mv_size, (char *) key.mv_data, + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + mdb_cursor_close(cursor); + mdb_txn_abort(txn); + + j=0; + + for (i= count - 1; i > -1; i-= (rand()%5)) { + j++; + txn=NULL; + E(mdb_txn_begin(env, NULL, 0, &txn)); + sprintf(kval, "%03x", values[i & ~0x0f]); + sprintf(sval, "%03x %d foo bar", values[i], values[i]); + key.mv_size = sizeof(int); + key.mv_data = kval; + data.mv_size = sizeof(sval); + data.mv_data = sval; + if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) { + j--; + mdb_txn_abort(txn); + } else { + E(mdb_txn_commit(txn)); + } + } + free(values); + printf("Deleted %d values\n", j); + + E(mdb_env_stat(env, &mst)); + E(mdb_txn_begin(env, NULL, 1, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + printf("Cursor next\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + printf("Cursor prev\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + mdb_cursor_close(cursor); + mdb_close(env, dbi); + + mdb_txn_abort(txn); + mdb_env_close(env); + + return 0; +} diff --git a/external/db_drivers/liblmdb32/mtest6.c b/external/db_drivers/liblmdb32/mtest6.c new file mode 100644 index 000000000..07d575893 --- /dev/null +++ b/external/db_drivers/liblmdb32/mtest6.c @@ -0,0 +1,137 @@ +/* mtest6.c - memory-mapped database tester/toy */ +/* + * Copyright 2011-2014 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ + +/* Tests for DB splits and merges */ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include "lmdb.h" + +#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) +#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) +#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \ + "%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort())) + +char dkbuf[1024]; + +int main(int argc,char * argv[]) +{ + int i = 0, j = 0, rc; + MDB_env *env; + MDB_dbi dbi; + MDB_val key, data; + MDB_txn *txn; + MDB_stat mst; + MDB_cursor *cursor; + int count; + int *values; + long kval; + char *sval; + + srand(time(NULL)); + + E(mdb_env_create(&env)); + E(mdb_env_set_mapsize(env, 10485760)); + E(mdb_env_set_maxdbs(env, 4)); + E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664)); + E(mdb_txn_begin(env, NULL, 0, &txn)); + E(mdb_open(txn, "id6", MDB_CREATE|MDB_INTEGERKEY, &dbi)); + E(mdb_cursor_open(txn, dbi, &cursor)); + E(mdb_stat(txn, dbi, &mst)); + + sval = calloc(1, mst.ms_psize / 4); + key.mv_size = sizeof(long); + key.mv_data = &kval; + data.mv_size = mst.ms_psize / 4 - 30; + data.mv_data = sval; + + printf("Adding 12 values, should yield 3 splits\n"); + for (i=0;i<12;i++) { + kval = i*5; + sprintf(sval, "%08x", kval); + (void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE)); + } + printf("Adding 12 more values, should yield 3 splits\n"); + for (i=0;i<12;i++) { + kval = i*5+4; + sprintf(sval, "%08x", kval); + (void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE)); + } + printf("Adding 12 more values, should yield 3 splits\n"); + for (i=0;i<12;i++) { + kval = i*5+1; + sprintf(sval, "%08x", kval); + (void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE)); + } + E(mdb_cursor_get(cursor, &key, &data, MDB_FIRST)); + + do { + printf("key: %p %s, data: %p %.*s\n", + key.mv_data, mdb_dkey(&key, dkbuf), + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + } while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0); + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + mdb_cursor_close(cursor); + mdb_txn_commit(txn); + +#if 0 + j=0; + + for (i= count - 1; i > -1; i-= (rand()%5)) { + j++; + txn=NULL; + E(mdb_txn_begin(env, NULL, 0, &txn)); + sprintf(kval, "%03x", values[i & ~0x0f]); + sprintf(sval, "%03x %d foo bar", values[i], values[i]); + key.mv_size = sizeof(int); + key.mv_data = kval; + data.mv_size = sizeof(sval); + data.mv_data = sval; + if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) { + j--; + mdb_txn_abort(txn); + } else { + E(mdb_txn_commit(txn)); + } + } + free(values); + printf("Deleted %d values\n", j); + + E(mdb_env_stat(env, &mst)); + E(mdb_txn_begin(env, NULL, 1, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + printf("Cursor next\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + printf("Cursor prev\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + mdb_cursor_close(cursor); + mdb_close(env, dbi); + + mdb_txn_abort(txn); +#endif + mdb_env_close(env); + + return 0; +} diff --git a/external/db_drivers/liblmdb32/sample-bdb.txt b/external/db_drivers/liblmdb32/sample-bdb.txt new file mode 100644 index 000000000..6a959bd68 --- /dev/null +++ b/external/db_drivers/liblmdb32/sample-bdb.txt @@ -0,0 +1,73 @@ +/* sample-bdb.txt - BerkeleyDB toy/sample + * + * Do a line-by-line comparison of this and sample-mdb.txt + */ +/* + * Copyright 2012 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ +#include <stdio.h> +#include <string.h> +#include <db.h> + +int main(int argc,char * argv[]) +{ + int rc; + DB_ENV *env; + DB *dbi; + DBT key, data; + DB_TXN *txn; + DBC *cursor; + char sval[32], kval[32]; + + /* Note: Most error checking omitted for simplicity */ + +#define FLAGS (DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN|DB_INIT_MPOOL|DB_CREATE|DB_THREAD) + rc = db_env_create(&env, 0); + rc = env->open(env, "./testdb", FLAGS, 0664); + rc = db_create(&dbi, env, 0); + rc = env->txn_begin(env, NULL, &txn, 0); + rc = dbi->open(dbi, txn, "test.bdb", NULL, DB_BTREE, DB_CREATE, 0664); + + memset(&key, 0, sizeof(DBT)); + memset(&data, 0, sizeof(DBT)); + key.size = sizeof(int); + key.data = sval; + data.size = sizeof(sval); + data.data = sval; + + sprintf(sval, "%03x %d foo bar", 32, 3141592); + rc = dbi->put(dbi, txn, &key, &data, 0); + rc = txn->commit(txn, 0); + if (rc) { + fprintf(stderr, "txn->commit: (%d) %s\n", rc, db_strerror(rc)); + goto leave; + } + rc = env->txn_begin(env, NULL, &txn, 0); + rc = dbi->cursor(dbi, txn, &cursor, 0); + key.flags = DB_DBT_USERMEM; + key.data = kval; + key.ulen = sizeof(kval); + data.flags = DB_DBT_USERMEM; + data.data = sval; + data.ulen = sizeof(sval); + while ((rc = cursor->c_get(cursor, &key, &data, DB_NEXT)) == 0) { + printf("key: %p %.*s, data: %p %.*s\n", + key.data, (int) key.size, (char *) key.data, + data.data, (int) data.size, (char *) data.data); + } + rc = cursor->c_close(cursor); + rc = txn->abort(txn); +leave: + rc = dbi->close(dbi, 0); + rc = env->close(env, 0); + return rc; +} diff --git a/external/db_drivers/liblmdb32/sample-mdb.txt b/external/db_drivers/liblmdb32/sample-mdb.txt new file mode 100644 index 000000000..a233ec5dd --- /dev/null +++ b/external/db_drivers/liblmdb32/sample-mdb.txt @@ -0,0 +1,62 @@ +/* sample-mdb.txt - MDB toy/sample + * + * Do a line-by-line comparison of this and sample-bdb.txt + */ +/* + * Copyright 2012 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ +#include <stdio.h> +#include "lmdb.h" + +int main(int argc,char * argv[]) +{ + int rc; + MDB_env *env; + MDB_dbi dbi; + MDB_val key, data; + MDB_txn *txn; + MDB_cursor *cursor; + char sval[32]; + + /* Note: Most error checking omitted for simplicity */ + + rc = mdb_env_create(&env); + rc = mdb_env_open(env, "./testdb", 0, 0664); + rc = mdb_txn_begin(env, NULL, 0, &txn); + rc = mdb_open(txn, NULL, 0, &dbi); + + key.mv_size = sizeof(int); + key.mv_data = sval; + data.mv_size = sizeof(sval); + data.mv_data = sval; + + sprintf(sval, "%03x %d foo bar", 32, 3141592); + rc = mdb_put(txn, dbi, &key, &data, 0); + rc = mdb_txn_commit(txn); + if (rc) { + fprintf(stderr, "mdb_txn_commit: (%d) %s\n", rc, mdb_strerror(rc)); + goto leave; + } + rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + rc = mdb_cursor_open(txn, dbi, &cursor); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %p %.*s, data: %p %.*s\n", + key.mv_data, (int) key.mv_size, (char *) key.mv_data, + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + } + mdb_cursor_close(cursor); + mdb_txn_abort(txn); +leave: + mdb_close(env, dbi); + mdb_env_close(env); + return 0; +} diff --git a/external/db_drivers/liblmdb32/tooltag b/external/db_drivers/liblmdb32/tooltag new file mode 100644 index 000000000..229bf16ba --- /dev/null +++ b/external/db_drivers/liblmdb32/tooltag @@ -0,0 +1,22 @@ +<tagfile> + <compound kind="page"> + <name>mdb_copy_1</name> + <title>mdb_copy - environment copy tool</title> + <filename>mdb_copy.1</filename> + </compound> + <compound kind="page"> + <name>mdb_dump_1</name> + <title>mdb_dump - environment export tool</title> + <filename>mdb_dump.1</filename> + </compound> + <compound kind="page"> + <name>mdb_load_1</name> + <title>mdb_load - environment import tool</title> + <filename>mdb_load.1</filename> + </compound> + <compound kind="page"> + <name>mdb_stat_1</name> + <title>mdb_stat - environment status tool</title> + <filename>mdb_stat.1</filename> + </compound> +</tagfile> diff --git a/external/db_drivers/liblmdb64/.gitignore b/external/db_drivers/liblmdb64/.gitignore new file mode 100644 index 000000000..d5102a87c --- /dev/null +++ b/external/db_drivers/liblmdb64/.gitignore @@ -0,0 +1,23 @@ +mtest +mtest[23456] +testdb +mdb_copy +mdb_stat +mdb_dump +mdb_load +*.lo +*.[ao] +*.so +*.exe +*[~#] +*.bak +*.orig +*.rej +*.gcov +*.gcda +*.gcno +core +core.* +valgrind.* +man/ +html/ diff --git a/external/db_drivers/liblmdb64/CHANGES b/external/db_drivers/liblmdb64/CHANGES new file mode 100644 index 000000000..cf12f2eeb --- /dev/null +++ b/external/db_drivers/liblmdb64/CHANGES @@ -0,0 +1,127 @@ +LMDB 0.9 Change Log + +LMDB 0.9.15 Release Engineering + Fix txn init (ITS#7961,#7987) + Fix MDB_PREV_DUP (ITS#7955,#7671) + Fix compact of empty env (ITS#7956) + Fix mdb_load with large values (ITS#8066) + Added workaround for fdatasync bug in ext3fs + Build + Don't use -fPIC for static lib + Update .gitignore (ITS#7952,#7953) + Cleanup for "make test" (ITS#7841) + Misc. Android/Windows cleanup + Documentation + Fix MDB_APPEND doc + Clarify mdb_dbi_open doc + +LMDB 0.9.14 Release (2014/09/20) + Fix to support 64K page size (ITS#7713) + Fix to persist decreased as well as increased mapsizes (ITS#7789) + Fix cursor bug when deleting last node of a DUPSORT key + Fix mdb_env_info to return FIXEDMAP address + Fix ambiguous error code from writing to closed DBI (ITS#7825) + Fix mdb_copy copying past end of file (ITS#7886) + Fix cursor bugs from page_merge/rebalance + Fix to dirty fewer pages in deletes (mdb_page_loose()) + Fix mdb_dbi_open creating subDBs (ITS#7917) + Fix mdb_cursor_get(_DUP) with single value (ITS#7913) + Fix Windows compat issues in mtests (ITS#7879) + Add compacting variant of mdb_copy + Add BigEndian integer key compare code + Add mdb_dump/mdb_load utilities + +LMDB 0.9.13 Release (2014/06/18) + Fix mdb_page_alloc unlimited overflow page search + Documentation + Re-fix MDB_CURRENT doc (ITS#7793) + Fix MDB_GET_MULTIPLE/MDB_NEXT_MULTIPLE doc + +LMDB 0.9.12 Release (2014/06/13) + Fix MDB_GET_BOTH regression (ITS#7875,#7681) + Fix MDB_MULTIPLE writing multiple keys (ITS#7834) + Fix mdb_rebalance (ITS#7829) + Fix mdb_page_split (ITS#7815) + Fix md_entries count (ITS#7861,#7828,#7793) + Fix MDB_CURRENT (ITS#7793) + Fix possible crash on Windows DLL detach + Misc code cleanup + Documentation + mdb_cursor_put: cursor moves on error (ITS#7771) + + +LMDB 0.9.11 Release (2014/01/15) + Add mdb_env_set_assert() (ITS#7775) + Fix: invalidate txn on page allocation errors (ITS#7377) + Fix xcursor tracking in mdb_cursor_del0() (ITS#7771) + Fix corruption from deletes (ITS#7756) + Fix Windows/MSVC build issues + Raise safe limit of max MDB_MAXKEYSIZE + Misc code cleanup + Documentation + Remove spurious note about non-overlapping flags (ITS#7665) + +LMDB 0.9.10 Release (2013/11/12) + Add MDB_NOMEMINIT option + Fix mdb_page_split() again (ITS#7589) + Fix MDB_NORDAHEAD definition (ITS#7734) + Fix mdb_cursor_del() positioning (ITS#7733) + Partial fix for larger page sizes (ITS#7713) + Fix Windows64/MSVC build issues + +LMDB 0.9.9 Release (2013/10/24) + Add mdb_env_get_fd() + Add MDB_NORDAHEAD option + Add MDB_NOLOCK option + Avoid wasting space in mdb_page_split() (ITS#7589) + Fix mdb_page_merge() cursor fixup (ITS#7722) + Fix mdb_cursor_del() on last delete (ITS#7718) + Fix adding WRITEMAP on existing env (ITS#7715) + Fix nested txns (ITS#7515) + Fix mdb_env_copy() O_DIRECT bug (ITS#7682) + Fix mdb_cursor_set(SET_RANGE) return code (ITS#7681) + Fix mdb_rebalance() cursor fixup (ITS#7701) + Misc code cleanup + Documentation + Note that by default, readers need write access + + +LMDB 0.9.8 Release (2013/09/09) + Allow mdb_env_set_mapsize() on an open environment + Fix mdb_dbi_flags() (ITS#7672) + Fix mdb_page_unspill() in nested txns + Fix mdb_cursor_get(CURRENT|NEXT) after a delete + Fix mdb_cursor_get(DUP) to always return key (ITS#7671) + Fix mdb_cursor_del() to always advance to next item (ITS#7670) + Fix mdb_cursor_set(SET_RANGE) for tree with single page (ITS#7681) + Fix mdb_env_copy() retry open if O_DIRECT fails (ITS#7682) + Tweak mdb_page_spill() to be less aggressive + Documentation + Update caveats since mdb_reader_check() added in 0.9.7 + +LMDB 0.9.7 Release (2013/08/17) + Don't leave stale lockfile on failed RDONLY open (ITS#7664) + Fix mdb_page_split() ref beyond cursor depth + Fix read txn data race (ITS#7635) + Fix mdb_rebalance (ITS#7536, #7538) + Fix mdb_drop() (ITS#7561) + Misc DEBUG macro fixes + Add MDB_NOTLS envflag + Add mdb_env_copyfd() + Add mdb_txn_env() (ITS#7660) + Add mdb_dbi_flags() (ITS#7661) + Add mdb_env_get_maxkeysize() + Add mdb_env_reader_list()/mdb_env_reader_check() + Add mdb_page_spill/unspill, remove hard txn size limit + Use shorter names for semaphores (ITS#7615) + Build + Fix install target (ITS#7656) + Documentation + Misc updates for cursors, DB handles, data lifetime + +LMDB 0.9.6 Release (2013/02/25) + Many fixes/enhancements + +LMDB 0.9.5 Release (2012/11/30) + Renamed from libmdb to liblmdb + Many fixes/enhancements diff --git a/external/db_drivers/liblmdb64/CMakeLists.txt b/external/db_drivers/liblmdb64/CMakeLists.txt new file mode 100644 index 000000000..62b800135 --- /dev/null +++ b/external/db_drivers/liblmdb64/CMakeLists.txt @@ -0,0 +1,39 @@ +# Copyright (c) 2014-2015, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set (lmdb_sources +mdb.c +midl.c) + +include_directories("${CMAKE_CURRENT_SOURCE_DIR}") + +add_library(lmdb + ${lmdb_sources}) +target_link_libraries(lmdb + LINK_PRIVATE + ${CMAKE_THREAD_LIBS_INIT}) diff --git a/external/db_drivers/liblmdb64/COPYRIGHT b/external/db_drivers/liblmdb64/COPYRIGHT new file mode 100644 index 000000000..722d1a515 --- /dev/null +++ b/external/db_drivers/liblmdb64/COPYRIGHT @@ -0,0 +1,20 @@ +Copyright 2011-2015 Howard Chu, Symas Corp. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted only as authorized by the OpenLDAP +Public License. + +A copy of this license is available in the file LICENSE in the +top-level directory of the distribution or, alternatively, at +<http://www.OpenLDAP.org/license.html>. + +OpenLDAP is a registered trademark of the OpenLDAP Foundation. + +Individual files and/or contributed packages may be copyright by +other parties and/or subject to additional restrictions. + +This work also contains materials derived from public sources. + +Additional information about OpenLDAP can be obtained at +<http://www.openldap.org/>. diff --git a/external/db_drivers/liblmdb64/Doxyfile b/external/db_drivers/liblmdb64/Doxyfile new file mode 100644 index 000000000..92d17b09e --- /dev/null +++ b/external/db_drivers/liblmdb64/Doxyfile @@ -0,0 +1,1631 @@ +# Doxyfile 1.7.1 + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for a project +# +# All text after a hash (#) is considered a comment and will be ignored +# The format is: +# TAG = value [value, ...] +# For lists items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (" ") + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- + +# This tag specifies the encoding used for all characters in the config file +# that follow. The default is UTF-8 which is also the encoding used for all +# text before the first occurrence of this tag. Doxygen uses libiconv (or the +# iconv built into libc) for the transcoding. See +# http://www.gnu.org/software/libiconv for the list of possible encodings. + +DOXYFILE_ENCODING = UTF-8 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded +# by quotes) that should identify the project. + +PROJECT_NAME = LMDB + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. +# This could be handy for archiving the generated documentation or +# if some version control system is used. + +PROJECT_NUMBER = + +# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) +# base path where the generated documentation will be put. +# If a relative path is entered, it will be relative to the location +# where doxygen was started. If left blank the current directory will be used. + +OUTPUT_DIRECTORY = + +# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create +# 4096 sub-directories (in 2 levels) under the output directory of each output +# format and will distribute the generated files over these directories. +# Enabling this option can be useful when feeding doxygen a huge amount of +# source files, where putting all generated files in the same directory would +# otherwise cause performance problems for the file system. + +CREATE_SUBDIRS = NO + +# The OUTPUT_LANGUAGE tag is used to specify the language in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all constant output in the proper language. +# The default language is English, other supported languages are: +# Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional, +# Croatian, Czech, Danish, Dutch, Esperanto, Farsi, Finnish, French, German, +# Greek, Hungarian, Italian, Japanese, Japanese-en (Japanese with English +# messages), Korean, Korean-en, Lithuanian, Norwegian, Macedonian, Persian, +# Polish, Portuguese, Romanian, Russian, Serbian, Serbian-Cyrilic, Slovak, +# Slovene, Spanish, Swedish, Ukrainian, and Vietnamese. + +OUTPUT_LANGUAGE = English + +# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will +# include brief member descriptions after the members that are listed in +# the file and class documentation (similar to JavaDoc). +# Set to NO to disable this. + +BRIEF_MEMBER_DESC = YES + +# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend +# the brief description of a member or function before the detailed description. +# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the +# brief descriptions will be completely suppressed. + +REPEAT_BRIEF = YES + +# This tag implements a quasi-intelligent brief description abbreviator +# that is used to form the text in various listings. Each string +# in this list, if found as the leading text of the brief description, will be +# stripped from the text and the result after processing the whole list, is +# used as the annotated text. Otherwise, the brief description is used as-is. +# If left blank, the following values are used ("$name" is automatically +# replaced with the name of the entity): "The $name class" "The $name widget" +# "The $name file" "is" "provides" "specifies" "contains" +# "represents" "a" "an" "the" + +ABBREVIATE_BRIEF = + +# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then +# Doxygen will generate a detailed section even if there is only a brief +# description. + +ALWAYS_DETAILED_SEC = NO + +# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all +# inherited members of a class in the documentation of that class as if those +# members were ordinary class members. Constructors, destructors and assignment +# operators of the base classes will not be shown. + +INLINE_INHERITED_MEMB = NO + +# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full +# path before files name in the file list and in the header files. If set +# to NO the shortest path that makes the file name unique will be used. + +FULL_PATH_NAMES = YES + +# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag +# can be used to strip a user-defined part of the path. Stripping is +# only done if one of the specified strings matches the left-hand part of +# the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the +# path to strip. + +STRIP_FROM_PATH = + +# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of +# the path mentioned in the documentation of a class, which tells +# the reader which header file to include in order to use a class. +# If left blank only the name of the header file containing the class +# definition is used. Otherwise one should specify the include paths that +# are normally passed to the compiler using the -I flag. + +STRIP_FROM_INC_PATH = + +# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter +# (but less readable) file names. This can be useful is your file systems +# doesn't support long names like on DOS, Mac, or CD-ROM. + +SHORT_NAMES = NO + +# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen +# will interpret the first line (until the first dot) of a JavaDoc-style +# comment as the brief description. If set to NO, the JavaDoc +# comments will behave just like regular Qt-style comments +# (thus requiring an explicit @brief command for a brief description.) + +JAVADOC_AUTOBRIEF = NO + +# If the QT_AUTOBRIEF tag is set to YES then Doxygen will +# interpret the first line (until the first dot) of a Qt-style +# comment as the brief description. If set to NO, the comments +# will behave just like regular Qt-style comments (thus requiring +# an explicit \brief command for a brief description.) + +QT_AUTOBRIEF = NO + +# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen +# treat a multi-line C++ special comment block (i.e. a block of //! or /// +# comments) as a brief description. This used to be the default behaviour. +# The new default is to treat a multi-line C++ comment block as a detailed +# description. Set this tag to YES if you prefer the old behaviour instead. + +MULTILINE_CPP_IS_BRIEF = NO + +# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented +# member inherits the documentation from any documented member that it +# re-implements. + +INHERIT_DOCS = YES + +# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce +# a new page for each member. If set to NO, the documentation of a member will +# be part of the file/class/namespace that contains it. + +SEPARATE_MEMBER_PAGES = NO + +# The TAB_SIZE tag can be used to set the number of spaces in a tab. +# Doxygen uses this value to replace tabs by spaces in code fragments. + +TAB_SIZE = 4 + +# This tag can be used to specify a number of aliases that acts +# as commands in the documentation. An alias has the form "name=value". +# For example adding "sideeffect=\par Side Effects:\n" will allow you to +# put the command \sideeffect (or @sideeffect) in the documentation, which +# will result in a user-defined paragraph with heading "Side Effects:". +# You can put \n's in the value part of an alias to insert newlines. + +ALIASES = + +# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C +# sources only. Doxygen will then generate output that is more tailored for C. +# For instance, some of the names that are used will be different. The list +# of all members will be omitted, etc. + +OPTIMIZE_OUTPUT_FOR_C = YES + +# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java +# sources only. Doxygen will then generate output that is more tailored for +# Java. For instance, namespaces will be presented as packages, qualified +# scopes will look different, etc. + +OPTIMIZE_OUTPUT_JAVA = NO + +# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran +# sources only. Doxygen will then generate output that is more tailored for +# Fortran. + +OPTIMIZE_FOR_FORTRAN = NO + +# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL +# sources. Doxygen will then generate output that is tailored for +# VHDL. + +OPTIMIZE_OUTPUT_VHDL = NO + +# Doxygen selects the parser to use depending on the extension of the files it +# parses. With this tag you can assign which parser to use for a given extension. +# Doxygen has a built-in mapping, but you can override or extend it using this +# tag. The format is ext=language, where ext is a file extension, and language +# is one of the parsers supported by doxygen: IDL, Java, Javascript, CSharp, C, +# C++, D, PHP, Objective-C, Python, Fortran, VHDL, C, C++. For instance to make +# doxygen treat .inc files as Fortran files (default is PHP), and .f files as C +# (default is Fortran), use: inc=Fortran f=C. Note that for custom extensions +# you also need to set FILE_PATTERNS otherwise the files are not read by doxygen. + +EXTENSION_MAPPING = + +# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want +# to include (a tag file for) the STL sources as input, then you should +# set this tag to YES in order to let doxygen match functions declarations and +# definitions whose arguments contain STL classes (e.g. func(std::string); v.s. +# func(std::string) {}). This also make the inheritance and collaboration +# diagrams that involve STL classes more complete and accurate. + +BUILTIN_STL_SUPPORT = NO + +# If you use Microsoft's C++/CLI language, you should set this option to YES to +# enable parsing support. + +CPP_CLI_SUPPORT = NO + +# Set the SIP_SUPPORT tag to YES if your project consists of sip sources only. +# Doxygen will parse them like normal C++ but will assume all classes use public +# instead of private inheritance when no explicit protection keyword is present. + +SIP_SUPPORT = NO + +# For Microsoft's IDL there are propget and propput attributes to indicate getter +# and setter methods for a property. Setting this option to YES (the default) +# will make doxygen to replace the get and set methods by a property in the +# documentation. This will only work if the methods are indeed getting or +# setting a simple type. If this is not the case, or you want to show the +# methods anyway, you should set this option to NO. + +IDL_PROPERTY_SUPPORT = YES + +# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC +# tag is set to YES, then doxygen will reuse the documentation of the first +# member in the group (if any) for the other members of the group. By default +# all members of a group must be documented explicitly. + +DISTRIBUTE_GROUP_DOC = NO + +# Set the SUBGROUPING tag to YES (the default) to allow class member groups of +# the same type (for instance a group of public functions) to be put as a +# subgroup of that type (e.g. under the Public Functions section). Set it to +# NO to prevent subgrouping. Alternatively, this can be done per class using +# the \nosubgrouping command. + +SUBGROUPING = YES + +INLINE_GROUPED_CLASSES = YES +# When TYPEDEF_HIDES_STRUCT is enabled, a typedef of a struct, union, or enum +# is documented as struct, union, or enum with the name of the typedef. So +# typedef struct TypeS {} TypeT, will appear in the documentation as a struct +# with name TypeT. When disabled the typedef will appear as a member of a file, +# namespace, or class. And the struct will be named TypeS. This can typically +# be useful for C code in case the coding convention dictates that all compound +# types are typedef'ed and only the typedef is referenced, never the tag name. + +TYPEDEF_HIDES_STRUCT = YES + +# The SYMBOL_CACHE_SIZE determines the size of the internal cache use to +# determine which symbols to keep in memory and which to flush to disk. +# When the cache is full, less often used symbols will be written to disk. +# For small to medium size projects (<1000 input files) the default value is +# probably good enough. For larger projects a too small cache size can cause +# doxygen to be busy swapping symbols to and from disk most of the time +# causing a significant performance penality. +# If the system has enough physical memory increasing the cache will improve the +# performance by keeping more symbols in memory. Note that the value works on +# a logarithmic scale so increasing the size by one will rougly double the +# memory usage. The cache size is given by this formula: +# 2^(16+SYMBOL_CACHE_SIZE). The valid range is 0..9, the default is 0, +# corresponding to a cache size of 2^16 = 65536 symbols + +SYMBOL_CACHE_SIZE = 0 + +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- + +# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in +# documentation are documented, even if no documentation was available. +# Private class members and static file members will be hidden unless +# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES + +EXTRACT_ALL = NO + +# If the EXTRACT_PRIVATE tag is set to YES all private members of a class +# will be included in the documentation. + +EXTRACT_PRIVATE = NO + +# If the EXTRACT_STATIC tag is set to YES all static members of a file +# will be included in the documentation. + +EXTRACT_STATIC = YES + +# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) +# defined locally in source files will be included in the documentation. +# If set to NO only classes defined in header files are included. + +EXTRACT_LOCAL_CLASSES = YES + +# This flag is only useful for Objective-C code. When set to YES local +# methods, which are defined in the implementation section but not in +# the interface are included in the documentation. +# If set to NO (the default) only methods in the interface are included. + +EXTRACT_LOCAL_METHODS = NO + +# If this flag is set to YES, the members of anonymous namespaces will be +# extracted and appear in the documentation as a namespace called +# 'anonymous_namespace{file}', where file will be replaced with the base +# name of the file that contains the anonymous namespace. By default +# anonymous namespace are hidden. + +EXTRACT_ANON_NSPACES = NO + +# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all +# undocumented members of documented classes, files or namespaces. +# If set to NO (the default) these members will be included in the +# various overviews, but no documentation section is generated. +# This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_MEMBERS = NO + +# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all +# undocumented classes that are normally visible in the class hierarchy. +# If set to NO (the default) these classes will be included in the various +# overviews. This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_CLASSES = NO + +# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all +# friend (class|struct|union) declarations. +# If set to NO (the default) these declarations will be included in the +# documentation. + +HIDE_FRIEND_COMPOUNDS = NO + +# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any +# documentation blocks found inside the body of a function. +# If set to NO (the default) these blocks will be appended to the +# function's detailed documentation block. + +HIDE_IN_BODY_DOCS = NO + +# The INTERNAL_DOCS tag determines if documentation +# that is typed after a \internal command is included. If the tag is set +# to NO (the default) then the documentation will be excluded. +# Set it to YES to include the internal documentation. + +INTERNAL_DOCS = NO + +# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate +# file names in lower-case letters. If set to YES upper-case letters are also +# allowed. This is useful if you have classes or files whose names only differ +# in case and if your file system supports case sensitive file names. Windows +# and Mac users are advised to set this option to NO. + +CASE_SENSE_NAMES = YES + +# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen +# will show members with their full class and namespace scopes in the +# documentation. If set to YES the scope will be hidden. + +HIDE_SCOPE_NAMES = NO + +# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen +# will put a list of the files that are included by a file in the documentation +# of that file. + +SHOW_INCLUDE_FILES = YES + +# If the FORCE_LOCAL_INCLUDES tag is set to YES then Doxygen +# will list include files with double quotes in the documentation +# rather than with sharp brackets. + +FORCE_LOCAL_INCLUDES = NO + +# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] +# is inserted in the documentation for inline members. + +INLINE_INFO = YES + +# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen +# will sort the (detailed) documentation of file and class members +# alphabetically by member name. If set to NO the members will appear in +# declaration order. + +SORT_MEMBER_DOCS = NO + +# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the +# brief documentation of file, namespace and class members alphabetically +# by member name. If set to NO (the default) the members will appear in +# declaration order. + +SORT_BRIEF_DOCS = NO + +# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen +# will sort the (brief and detailed) documentation of class members so that +# constructors and destructors are listed first. If set to NO (the default) +# the constructors will appear in the respective orders defined by +# SORT_MEMBER_DOCS and SORT_BRIEF_DOCS. +# This tag will be ignored for brief docs if SORT_BRIEF_DOCS is set to NO +# and ignored for detailed docs if SORT_MEMBER_DOCS is set to NO. + +SORT_MEMBERS_CTORS_1ST = NO + +# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the +# hierarchy of group names into alphabetical order. If set to NO (the default) +# the group names will appear in their defined order. + +SORT_GROUP_NAMES = NO + +# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be +# sorted by fully-qualified names, including namespaces. If set to +# NO (the default), the class list will be sorted only by class name, +# not including the namespace part. +# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. +# Note: This option applies only to the class list, not to the +# alphabetical list. + +SORT_BY_SCOPE_NAME = NO + +# The GENERATE_TODOLIST tag can be used to enable (YES) or +# disable (NO) the todo list. This list is created by putting \todo +# commands in the documentation. + +GENERATE_TODOLIST = YES + +# The GENERATE_TESTLIST tag can be used to enable (YES) or +# disable (NO) the test list. This list is created by putting \test +# commands in the documentation. + +GENERATE_TESTLIST = YES + +# The GENERATE_BUGLIST tag can be used to enable (YES) or +# disable (NO) the bug list. This list is created by putting \bug +# commands in the documentation. + +GENERATE_BUGLIST = YES + +# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or +# disable (NO) the deprecated list. This list is created by putting +# \deprecated commands in the documentation. + +GENERATE_DEPRECATEDLIST= YES + +# The ENABLED_SECTIONS tag can be used to enable conditional +# documentation sections, marked by \if sectionname ... \endif. + +ENABLED_SECTIONS = + +# The MAX_INITIALIZER_LINES tag determines the maximum number of lines +# the initial value of a variable or define consists of for it to appear in +# the documentation. If the initializer consists of more lines than specified +# here it will be hidden. Use a value of 0 to hide initializers completely. +# The appearance of the initializer of individual variables and defines in the +# documentation can be controlled using \showinitializer or \hideinitializer +# command in the documentation regardless of this setting. + +MAX_INITIALIZER_LINES = 30 + +# Set the SHOW_USED_FILES tag to NO to disable the list of files generated +# at the bottom of the documentation of classes and structs. If set to YES the +# list will mention the files that were used to generate the documentation. + +SHOW_USED_FILES = YES + +# If the sources in your project are distributed over multiple directories +# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy +# in the documentation. The default is NO. + +SHOW_DIRECTORIES = NO + +# Set the SHOW_FILES tag to NO to disable the generation of the Files page. +# This will remove the Files entry from the Quick Index and from the +# Folder Tree View (if specified). The default is YES. + +SHOW_FILES = YES + +# Set the SHOW_NAMESPACES tag to NO to disable the generation of the +# Namespaces page. +# This will remove the Namespaces entry from the Quick Index +# and from the Folder Tree View (if specified). The default is YES. + +SHOW_NAMESPACES = YES + +# The FILE_VERSION_FILTER tag can be used to specify a program or script that +# doxygen should invoke to get the current version for each file (typically from +# the version control system). Doxygen will invoke the program by executing (via +# popen()) the command <command> <input-file>, where <command> is the value of +# the FILE_VERSION_FILTER tag, and <input-file> is the name of an input file +# provided by doxygen. Whatever the program writes to standard output +# is used as the file version. See the manual for examples. + +FILE_VERSION_FILTER = + +# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed +# by doxygen. The layout file controls the global structure of the generated +# output files in an output format independent way. The create the layout file +# that represents doxygen's defaults, run doxygen with the -l option. +# You can optionally specify a file name after the option, if omitted +# DoxygenLayout.xml will be used as the name of the layout file. + +LAYOUT_FILE = + +#--------------------------------------------------------------------------- +# configuration options related to warning and progress messages +#--------------------------------------------------------------------------- + +# The QUIET tag can be used to turn on/off the messages that are generated +# by doxygen. Possible values are YES and NO. If left blank NO is used. + +QUIET = NO + +# The WARNINGS tag can be used to turn on/off the warning messages that are +# generated by doxygen. Possible values are YES and NO. If left blank +# NO is used. + +WARNINGS = YES + +# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings +# for undocumented members. If EXTRACT_ALL is set to YES then this flag will +# automatically be disabled. + +WARN_IF_UNDOCUMENTED = YES + +# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for +# potential errors in the documentation, such as not documenting some +# parameters in a documented function, or documenting parameters that +# don't exist or using markup commands wrongly. + +WARN_IF_DOC_ERROR = YES + +# This WARN_NO_PARAMDOC option can be abled to get warnings for +# functions that are documented, but have no documentation for their parameters +# or return value. If set to NO (the default) doxygen will only warn about +# wrong or incomplete parameter documentation, but not about the absence of +# documentation. + +WARN_NO_PARAMDOC = NO + +# The WARN_FORMAT tag determines the format of the warning messages that +# doxygen can produce. The string should contain the $file, $line, and $text +# tags, which will be replaced by the file and line number from which the +# warning originated and the warning text. Optionally the format may contain +# $version, which will be replaced by the version of the file (if it could +# be obtained via FILE_VERSION_FILTER) + +WARN_FORMAT = "$file:$line: $text" + +# The WARN_LOGFILE tag can be used to specify a file to which warning +# and error messages should be written. If left blank the output is written +# to stderr. + +WARN_LOGFILE = + +#--------------------------------------------------------------------------- +# configuration options related to the input files +#--------------------------------------------------------------------------- + +# The INPUT tag can be used to specify the files and/or directories that contain +# documented source files. You may enter file names like "myfile.cpp" or +# directories like "/usr/src/myproject". Separate the files or directories +# with spaces. + +INPUT = lmdb.h midl.h mdb.c midl.c + +# This tag can be used to specify the character encoding of the source files +# that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is +# also the default input encoding. Doxygen uses libiconv (or the iconv built +# into libc) for the transcoding. See http://www.gnu.org/software/libiconv for +# the list of possible encodings. + +INPUT_ENCODING = UTF-8 + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank the following patterns are tested: +# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx +# *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py *.f90 + +FILE_PATTERNS = + +# The RECURSIVE tag can be used to turn specify whether or not subdirectories +# should be searched for input files as well. Possible values are YES and NO. +# If left blank NO is used. + +RECURSIVE = NO + +# The EXCLUDE tag can be used to specify files and/or directories that should +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. + +EXCLUDE = + +# The EXCLUDE_SYMLINKS tag can be used select whether or not files or +# directories that are symbolic links (a Unix filesystem feature) are excluded +# from the input. + +EXCLUDE_SYMLINKS = NO + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. Note that the wildcards are matched +# against the file with absolute path, so to exclude all test directories +# for example use the pattern */test/* + +EXCLUDE_PATTERNS = + +# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names +# (namespaces, classes, functions, etc.) that should be excluded from the +# output. The symbol name can be a fully qualified name, a word, or if the +# wildcard * is used, a substring. Examples: ANamespace, AClass, +# AClass::ANamespace, ANamespace::*Test + +EXCLUDE_SYMBOLS = + +# The EXAMPLE_PATH tag can be used to specify one or more files or +# directories that contain example code fragments that are included (see +# the \include command). + +EXAMPLE_PATH = + +# If the value of the EXAMPLE_PATH tag contains directories, you can use the +# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank all files are included. + +EXAMPLE_PATTERNS = + +# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be +# searched for input files to be used with the \include or \dontinclude +# commands irrespective of the value of the RECURSIVE tag. +# Possible values are YES and NO. If left blank NO is used. + +EXAMPLE_RECURSIVE = NO + +# The IMAGE_PATH tag can be used to specify one or more files or +# directories that contain image that are included in the documentation (see +# the \image command). + +IMAGE_PATH = + +# The INPUT_FILTER tag can be used to specify a program that doxygen should +# invoke to filter for each input file. Doxygen will invoke the filter program +# by executing (via popen()) the command <filter> <input-file>, where <filter> +# is the value of the INPUT_FILTER tag, and <input-file> is the name of an +# input file. Doxygen will then use the output that the filter program writes +# to standard output. +# If FILTER_PATTERNS is specified, this tag will be +# ignored. + +INPUT_FILTER = + +# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern +# basis. +# Doxygen will compare the file name with each pattern and apply the +# filter if there is a match. +# The filters are a list of the form: +# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further +# info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER +# is applied to all files. + +FILTER_PATTERNS = + +# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using +# INPUT_FILTER) will be used to filter the input files when producing source +# files to browse (i.e. when SOURCE_BROWSER is set to YES). + +FILTER_SOURCE_FILES = NO + +#--------------------------------------------------------------------------- +# configuration options related to source browsing +#--------------------------------------------------------------------------- + +# If the SOURCE_BROWSER tag is set to YES then a list of source files will +# be generated. Documented entities will be cross-referenced with these sources. +# Note: To get rid of all source code in the generated output, make sure also +# VERBATIM_HEADERS is set to NO. + +SOURCE_BROWSER = NO + +# Setting the INLINE_SOURCES tag to YES will include the body +# of functions and classes directly in the documentation. + +INLINE_SOURCES = NO + +# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct +# doxygen to hide any special comment blocks from generated source code +# fragments. Normal C and C++ comments will always remain visible. + +STRIP_CODE_COMMENTS = YES + +# If the REFERENCED_BY_RELATION tag is set to YES +# then for each documented function all documented +# functions referencing it will be listed. + +REFERENCED_BY_RELATION = NO + +# If the REFERENCES_RELATION tag is set to YES +# then for each documented function all documented entities +# called/used by that function will be listed. + +REFERENCES_RELATION = NO + +# If the REFERENCES_LINK_SOURCE tag is set to YES (the default) +# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from +# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will +# link to the source code. +# Otherwise they will link to the documentation. + +REFERENCES_LINK_SOURCE = YES + +# If the USE_HTAGS tag is set to YES then the references to source code +# will point to the HTML generated by the htags(1) tool instead of doxygen +# built-in source browser. The htags tool is part of GNU's global source +# tagging system (see http://www.gnu.org/software/global/global.html). You +# will need version 4.8.6 or higher. + +USE_HTAGS = NO + +# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen +# will generate a verbatim copy of the header file for each class for +# which an include is specified. Set to NO to disable this. + +VERBATIM_HEADERS = YES + +#--------------------------------------------------------------------------- +# configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- + +# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index +# of all compounds will be generated. Enable this if the project +# contains a lot of classes, structs, unions or interfaces. + +ALPHABETICAL_INDEX = YES + +# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then +# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns +# in which this list will be split (can be a number in the range [1..20]) + +COLS_IN_ALPHA_INDEX = 5 + +# In case all classes in a project start with a common prefix, all +# classes will be put under the same header in the alphabetical index. +# The IGNORE_PREFIX tag can be used to specify one or more prefixes that +# should be ignored while generating the index headers. + +IGNORE_PREFIX = + +#--------------------------------------------------------------------------- +# configuration options related to the HTML output +#--------------------------------------------------------------------------- + +# If the GENERATE_HTML tag is set to YES (the default) Doxygen will +# generate HTML output. + +GENERATE_HTML = YES + +# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `html' will be used as the default path. + +HTML_OUTPUT = html + +# The HTML_FILE_EXTENSION tag can be used to specify the file extension for +# each generated HTML page (for example: .htm,.php,.asp). If it is left blank +# doxygen will generate files with .html extension. + +HTML_FILE_EXTENSION = .html + +# The HTML_HEADER tag can be used to specify a personal HTML header for +# each generated HTML page. If it is left blank doxygen will generate a +# standard header. + +HTML_HEADER = + +# The HTML_FOOTER tag can be used to specify a personal HTML footer for +# each generated HTML page. If it is left blank doxygen will generate a +# standard footer. + +HTML_FOOTER = + +# The HTML_STYLESHEET tag can be used to specify a user-defined cascading +# style sheet that is used by each HTML page. It can be used to +# fine-tune the look of the HTML output. If the tag is left blank doxygen +# will generate a default style sheet. Note that doxygen will try to copy +# the style sheet file to the HTML output directory, so don't put your own +# stylesheet in the HTML output directory as well, or it will be erased! + +HTML_STYLESHEET = + +# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. +# Doxygen will adjust the colors in the stylesheet and background images +# according to this color. Hue is specified as an angle on a colorwheel, +# see http://en.wikipedia.org/wiki/Hue for more information. +# For instance the value 0 represents red, 60 is yellow, 120 is green, +# 180 is cyan, 240 is blue, 300 purple, and 360 is red again. +# The allowed range is 0 to 359. + +HTML_COLORSTYLE_HUE = 220 + +# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of +# the colors in the HTML output. For a value of 0 the output will use +# grayscales only. A value of 255 will produce the most vivid colors. + +HTML_COLORSTYLE_SAT = 100 + +# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to +# the luminance component of the colors in the HTML output. Values below +# 100 gradually make the output lighter, whereas values above 100 make +# the output darker. The value divided by 100 is the actual gamma applied, +# so 80 represents a gamma of 0.8, The value 220 represents a gamma of 2.2, +# and 100 does not change the gamma. + +HTML_COLORSTYLE_GAMMA = 80 + +# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML +# page will contain the date and time when the page was generated. Setting +# this to NO can help when comparing the output of multiple runs. + +HTML_TIMESTAMP = YES + +# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, +# files or namespaces will be aligned in HTML using tables. If set to +# NO a bullet list will be used. + +HTML_ALIGN_MEMBERS = YES + +# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML +# documentation will contain sections that can be hidden and shown after the +# page has loaded. For this to work a browser that supports +# JavaScript and DHTML is required (for instance Mozilla 1.0+, Firefox +# Netscape 6.0+, Internet explorer 5.0+, Konqueror, or Safari). + +HTML_DYNAMIC_SECTIONS = NO + +# If the GENERATE_DOCSET tag is set to YES, additional index files +# will be generated that can be used as input for Apple's Xcode 3 +# integrated development environment, introduced with OSX 10.5 (Leopard). +# To create a documentation set, doxygen will generate a Makefile in the +# HTML output directory. Running make will produce the docset in that +# directory and running "make install" will install the docset in +# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find +# it at startup. +# See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html +# for more information. + +GENERATE_DOCSET = NO + +# When GENERATE_DOCSET tag is set to YES, this tag determines the name of the +# feed. A documentation feed provides an umbrella under which multiple +# documentation sets from a single provider (such as a company or product suite) +# can be grouped. + +DOCSET_FEEDNAME = "Doxygen generated docs" + +# When GENERATE_DOCSET tag is set to YES, this tag specifies a string that +# should uniquely identify the documentation set bundle. This should be a +# reverse domain-name style string, e.g. com.mycompany.MyDocSet. Doxygen +# will append .docset to the name. + +DOCSET_BUNDLE_ID = org.doxygen.Project + +# When GENERATE_PUBLISHER_ID tag specifies a string that should uniquely identify +# the documentation publisher. This should be a reverse domain-name style +# string, e.g. com.mycompany.MyDocSet.documentation. + +DOCSET_PUBLISHER_ID = org.doxygen.Publisher + +# The GENERATE_PUBLISHER_NAME tag identifies the documentation publisher. + +DOCSET_PUBLISHER_NAME = Publisher + +# If the GENERATE_HTMLHELP tag is set to YES, additional index files +# will be generated that can be used as input for tools like the +# Microsoft HTML help workshop to generate a compiled HTML help file (.chm) +# of the generated HTML documentation. + +GENERATE_HTMLHELP = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can +# be used to specify the file name of the resulting .chm file. You +# can add a path in front of the file if the result should not be +# written to the html output directory. + +CHM_FILE = + +# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can +# be used to specify the location (absolute path including file name) of +# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run +# the HTML help compiler on the generated index.hhp. + +HHC_LOCATION = + +# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag +# controls if a separate .chi index file is generated (YES) or that +# it should be included in the master .chm file (NO). + +GENERATE_CHI = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the CHM_INDEX_ENCODING +# is used to encode HtmlHelp index (hhk), content (hhc) and project file +# content. + +CHM_INDEX_ENCODING = + +# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag +# controls whether a binary table of contents is generated (YES) or a +# normal table of contents (NO) in the .chm file. + +BINARY_TOC = NO + +# The TOC_EXPAND flag can be set to YES to add extra items for group members +# to the contents of the HTML help documentation and to the tree view. + +TOC_EXPAND = NO + +# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and +# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated +# that can be used as input for Qt's qhelpgenerator to generate a +# Qt Compressed Help (.qch) of the generated HTML documentation. + +GENERATE_QHP = NO + +# If the QHG_LOCATION tag is specified, the QCH_FILE tag can +# be used to specify the file name of the resulting .qch file. +# The path specified is relative to the HTML output folder. + +QCH_FILE = + +# The QHP_NAMESPACE tag specifies the namespace to use when generating +# Qt Help Project output. For more information please see +# http://doc.trolltech.com/qthelpproject.html#namespace + +QHP_NAMESPACE = org.doxygen.Project + +# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating +# Qt Help Project output. For more information please see +# http://doc.trolltech.com/qthelpproject.html#virtual-folders + +QHP_VIRTUAL_FOLDER = doc + +# If QHP_CUST_FILTER_NAME is set, it specifies the name of a custom filter to +# add. For more information please see +# http://doc.trolltech.com/qthelpproject.html#custom-filters + +QHP_CUST_FILTER_NAME = + +# The QHP_CUST_FILT_ATTRS tag specifies the list of the attributes of the +# custom filter to add. For more information please see +# <a href="http://doc.trolltech.com/qthelpproject.html#custom-filters"> +# Qt Help Project / Custom Filters</a>. + +QHP_CUST_FILTER_ATTRS = + +# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this +# project's +# filter section matches. +# <a href="http://doc.trolltech.com/qthelpproject.html#filter-attributes"> +# Qt Help Project / Filter Attributes</a>. + +QHP_SECT_FILTER_ATTRS = + +# If the GENERATE_QHP tag is set to YES, the QHG_LOCATION tag can +# be used to specify the location of Qt's qhelpgenerator. +# If non-empty doxygen will try to run qhelpgenerator on the generated +# .qhp file. + +QHG_LOCATION = + +# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files +# will be generated, which together with the HTML files, form an Eclipse help +# plugin. To install this plugin and make it available under the help contents +# menu in Eclipse, the contents of the directory containing the HTML and XML +# files needs to be copied into the plugins directory of eclipse. The name of +# the directory within the plugins directory should be the same as +# the ECLIPSE_DOC_ID value. After copying Eclipse needs to be restarted before +# the help appears. + +GENERATE_ECLIPSEHELP = NO + +# A unique identifier for the eclipse help plugin. When installing the plugin +# the directory name containing the HTML and XML files should also have +# this name. + +ECLIPSE_DOC_ID = org.doxygen.Project + +# The DISABLE_INDEX tag can be used to turn on/off the condensed index at +# top of each HTML page. The value NO (the default) enables the index and +# the value YES disables it. + +DISABLE_INDEX = NO + +# This tag can be used to set the number of enum values (range [1..20]) +# that doxygen will group on one line in the generated HTML documentation. + +ENUM_VALUES_PER_LINE = 4 + +# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index +# structure should be generated to display hierarchical information. +# If the tag value is set to YES, a side panel will be generated +# containing a tree-like index structure (just like the one that +# is generated for HTML Help). For this to work a browser that supports +# JavaScript, DHTML, CSS and frames is required (i.e. any modern browser). +# Windows users are probably better off using the HTML help feature. + +GENERATE_TREEVIEW = NO + +# By enabling USE_INLINE_TREES, doxygen will generate the Groups, Directories, +# and Class Hierarchy pages using a tree view instead of an ordered list. + +USE_INLINE_TREES = NO + +# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be +# used to set the initial width (in pixels) of the frame in which the tree +# is shown. + +TREEVIEW_WIDTH = 250 + +# When the EXT_LINKS_IN_WINDOW option is set to YES doxygen will open +# links to external symbols imported via tag files in a separate window. + +EXT_LINKS_IN_WINDOW = NO + +# Use this tag to change the font size of Latex formulas included +# as images in the HTML documentation. The default is 10. Note that +# when you change the font size after a successful doxygen run you need +# to manually remove any form_*.png images from the HTML output directory +# to force them to be regenerated. + +FORMULA_FONTSIZE = 10 + +# Use the FORMULA_TRANPARENT tag to determine whether or not the images +# generated for formulas are transparent PNGs. Transparent PNGs are +# not supported properly for IE 6.0, but are supported on all modern browsers. +# Note that when changing this option you need to delete any form_*.png files +# in the HTML output before the changes have effect. + +FORMULA_TRANSPARENT = YES + +# When the SEARCHENGINE tag is enabled doxygen will generate a search box +# for the HTML output. The underlying search engine uses javascript +# and DHTML and should work on any modern browser. Note that when using +# HTML help (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets +# (GENERATE_DOCSET) there is already a search function so this one should +# typically be disabled. For large projects the javascript based search engine +# can be slow, then enabling SERVER_BASED_SEARCH may provide a better solution. + +SEARCHENGINE = YES + +# When the SERVER_BASED_SEARCH tag is enabled the search engine will be +# implemented using a PHP enabled web server instead of at the web client +# using Javascript. Doxygen will generate the search PHP script and index +# file to put on the web server. The advantage of the server +# based approach is that it scales better to large projects and allows +# full text search. The disadvances is that it is more difficult to setup +# and does not have live searching capabilities. + +SERVER_BASED_SEARCH = NO + +#--------------------------------------------------------------------------- +# configuration options related to the LaTeX output +#--------------------------------------------------------------------------- + +# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will +# generate Latex output. + +GENERATE_LATEX = NO + +# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `latex' will be used as the default path. + +LATEX_OUTPUT = latex + +# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be +# invoked. If left blank `latex' will be used as the default command name. +# Note that when enabling USE_PDFLATEX this option is only used for +# generating bitmaps for formulas in the HTML output, but not in the +# Makefile that is written to the output directory. + +LATEX_CMD_NAME = latex + +# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to +# generate index for LaTeX. If left blank `makeindex' will be used as the +# default command name. + +MAKEINDEX_CMD_NAME = makeindex + +# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact +# LaTeX documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_LATEX = NO + +# The PAPER_TYPE tag can be used to set the paper type that is used +# by the printer. Possible values are: a4, a4wide, letter, legal and +# executive. If left blank a4wide will be used. + +PAPER_TYPE = a4wide + +# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX +# packages that should be included in the LaTeX output. + +EXTRA_PACKAGES = + +# The LATEX_HEADER tag can be used to specify a personal LaTeX header for +# the generated latex document. The header should contain everything until +# the first chapter. If it is left blank doxygen will generate a +# standard header. Notice: only use this tag if you know what you are doing! + +LATEX_HEADER = + +# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated +# is prepared for conversion to pdf (using ps2pdf). The pdf file will +# contain links (just like the HTML output) instead of page references +# This makes the output suitable for online browsing using a pdf viewer. + +PDF_HYPERLINKS = YES + +# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of +# plain latex in the generated Makefile. Set this option to YES to get a +# higher quality PDF documentation. + +USE_PDFLATEX = YES + +# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. +# command to the generated LaTeX files. This will instruct LaTeX to keep +# running if errors occur, instead of asking the user for help. +# This option is also used when generating formulas in HTML. + +LATEX_BATCHMODE = NO + +# If LATEX_HIDE_INDICES is set to YES then doxygen will not +# include the index chapters (such as File Index, Compound Index, etc.) +# in the output. + +LATEX_HIDE_INDICES = NO + +# If LATEX_SOURCE_CODE is set to YES then doxygen will include +# source code with syntax highlighting in the LaTeX output. +# Note that which sources are shown also depends on other settings +# such as SOURCE_BROWSER. + +LATEX_SOURCE_CODE = NO + +#--------------------------------------------------------------------------- +# configuration options related to the RTF output +#--------------------------------------------------------------------------- + +# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output +# The RTF output is optimized for Word 97 and may not look very pretty with +# other RTF readers or editors. + +GENERATE_RTF = NO + +# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `rtf' will be used as the default path. + +RTF_OUTPUT = rtf + +# If the COMPACT_RTF tag is set to YES Doxygen generates more compact +# RTF documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_RTF = NO + +# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated +# will contain hyperlink fields. The RTF file will +# contain links (just like the HTML output) instead of page references. +# This makes the output suitable for online browsing using WORD or other +# programs which support those fields. +# Note: wordpad (write) and others do not support links. + +RTF_HYPERLINKS = NO + +# Load stylesheet definitions from file. Syntax is similar to doxygen's +# config file, i.e. a series of assignments. You only have to provide +# replacements, missing definitions are set to their default value. + +RTF_STYLESHEET_FILE = + +# Set optional variables used in the generation of an rtf document. +# Syntax is similar to doxygen's config file. + +RTF_EXTENSIONS_FILE = + +#--------------------------------------------------------------------------- +# configuration options related to the man page output +#--------------------------------------------------------------------------- + +# If the GENERATE_MAN tag is set to YES (the default) Doxygen will +# generate man pages + +GENERATE_MAN = YES + +# The MAN_OUTPUT tag is used to specify where the man pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `man' will be used as the default path. + +MAN_OUTPUT = man + +# The MAN_EXTENSION tag determines the extension that is added to +# the generated man pages (default is the subroutine's section .3) + +MAN_EXTENSION = .3 + +# If the MAN_LINKS tag is set to YES and Doxygen generates man output, +# then it will generate one additional man file for each entity +# documented in the real man page(s). These additional files +# only source the real man page, but without them the man command +# would be unable to find the correct page. The default is NO. + +MAN_LINKS = NO + +#--------------------------------------------------------------------------- +# configuration options related to the XML output +#--------------------------------------------------------------------------- + +# If the GENERATE_XML tag is set to YES Doxygen will +# generate an XML file that captures the structure of +# the code including all documentation. + +GENERATE_XML = NO + +# The XML_OUTPUT tag is used to specify where the XML pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `xml' will be used as the default path. + +XML_OUTPUT = xml + +# The XML_SCHEMA tag can be used to specify an XML schema, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_SCHEMA = + +# The XML_DTD tag can be used to specify an XML DTD, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_DTD = + +# If the XML_PROGRAMLISTING tag is set to YES Doxygen will +# dump the program listings (including syntax highlighting +# and cross-referencing information) to the XML output. Note that +# enabling this will significantly increase the size of the XML output. + +XML_PROGRAMLISTING = YES + +#--------------------------------------------------------------------------- +# configuration options for the AutoGen Definitions output +#--------------------------------------------------------------------------- + +# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will +# generate an AutoGen Definitions (see autogen.sf.net) file +# that captures the structure of the code including all +# documentation. Note that this feature is still experimental +# and incomplete at the moment. + +GENERATE_AUTOGEN_DEF = NO + +#--------------------------------------------------------------------------- +# configuration options related to the Perl module output +#--------------------------------------------------------------------------- + +# If the GENERATE_PERLMOD tag is set to YES Doxygen will +# generate a Perl module file that captures the structure of +# the code including all documentation. Note that this +# feature is still experimental and incomplete at the +# moment. + +GENERATE_PERLMOD = NO + +# If the PERLMOD_LATEX tag is set to YES Doxygen will generate +# the necessary Makefile rules, Perl scripts and LaTeX code to be able +# to generate PDF and DVI output from the Perl module output. + +PERLMOD_LATEX = NO + +# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be +# nicely formatted so it can be parsed by a human reader. +# This is useful +# if you want to understand what is going on. +# On the other hand, if this +# tag is set to NO the size of the Perl module output will be much smaller +# and Perl will parse it just the same. + +PERLMOD_PRETTY = YES + +# The names of the make variables in the generated doxyrules.make file +# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. +# This is useful so different doxyrules.make files included by the same +# Makefile don't overwrite each other's variables. + +PERLMOD_MAKEVAR_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the preprocessor +#--------------------------------------------------------------------------- + +# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will +# evaluate all C-preprocessor directives found in the sources and include +# files. + +ENABLE_PREPROCESSING = YES + +# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro +# names in the source code. If set to NO (the default) only conditional +# compilation will be performed. Macro expansion can be done in a controlled +# way by setting EXPAND_ONLY_PREDEF to YES. + +MACRO_EXPANSION = NO + +# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES +# then the macro expansion is limited to the macros specified with the +# PREDEFINED and EXPAND_AS_DEFINED tags. + +EXPAND_ONLY_PREDEF = NO + +# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files +# in the INCLUDE_PATH (see below) will be search if a #include is found. + +SEARCH_INCLUDES = YES + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by +# the preprocessor. + +INCLUDE_PATH = + +# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard +# patterns (like *.h and *.hpp) to filter out the header-files in the +# directories. If left blank, the patterns specified with FILE_PATTERNS will +# be used. + +INCLUDE_FILE_PATTERNS = + +# The PREDEFINED tag can be used to specify one or more macro names that +# are defined before the preprocessor is started (similar to the -D option of +# gcc). The argument of the tag is a list of macros of the form: name +# or name=definition (no spaces). If the definition and the = are +# omitted =1 is assumed. To prevent a macro definition from being +# undefined via #undef or recursively expanded use the := operator +# instead of the = operator. + +PREDEFINED = DEBUG=2 __GNUC__=1 + +# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then +# this tag can be used to specify a list of macro names that should be expanded. +# The macro definition that is found in the sources will be used. +# Use the PREDEFINED tag if you want to use a different macro definition. + +EXPAND_AS_DEFINED = + +# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then +# doxygen's preprocessor will remove all function-like macros that are alone +# on a line, have an all uppercase name, and do not end with a semicolon. Such +# function macros are typically used for boiler-plate code, and will confuse +# the parser if not removed. + +SKIP_FUNCTION_MACROS = YES + +#--------------------------------------------------------------------------- +# Configuration::additions related to external references +#--------------------------------------------------------------------------- + +# The TAGFILES option can be used to specify one or more tagfiles. +# Optionally an initial location of the external documentation +# can be added for each tagfile. The format of a tag file without +# this location is as follows: +# +# TAGFILES = file1 file2 ... +# Adding location for the tag files is done as follows: +# +# TAGFILES = file1=loc1 "file2 = loc2" ... +# where "loc1" and "loc2" can be relative or absolute paths or +# URLs. If a location is present for each tag, the installdox tool +# does not have to be run to correct the links. +# Note that each tag file must have a unique name +# (where the name does NOT include the path) +# If a tag file is not located in the directory in which doxygen +# is run, you must also specify the path to the tagfile here. + +TAGFILES = tooltag=./man1 + +# When a file name is specified after GENERATE_TAGFILE, doxygen will create +# a tag file that is based on the input files it reads. + +GENERATE_TAGFILE = + +# If the ALLEXTERNALS tag is set to YES all external classes will be listed +# in the class index. If set to NO only the inherited external classes +# will be listed. + +ALLEXTERNALS = NO + +# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed +# in the modules index. If set to NO, only the current project's groups will +# be listed. + +EXTERNAL_GROUPS = YES + +# The PERL_PATH should be the absolute path and name of the perl script +# interpreter (i.e. the result of `which perl'). + +PERL_PATH = /usr/bin/perl + +#--------------------------------------------------------------------------- +# Configuration options related to the dot tool +#--------------------------------------------------------------------------- + +# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will +# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base +# or super classes. Setting the tag to NO turns the diagrams off. Note that +# this option is superseded by the HAVE_DOT option below. This is only a +# fallback. It is recommended to install and use dot, since it yields more +# powerful graphs. + +CLASS_DIAGRAMS = YES + +# You can define message sequence charts within doxygen comments using the \msc +# command. Doxygen will then run the mscgen tool (see +# http://www.mcternan.me.uk/mscgen/) to produce the chart and insert it in the +# documentation. The MSCGEN_PATH tag allows you to specify the directory where +# the mscgen tool resides. If left empty the tool is assumed to be found in the +# default search path. + +MSCGEN_PATH = + +# If set to YES, the inheritance and collaboration graphs will hide +# inheritance and usage relations if the target is undocumented +# or is not a class. + +HIDE_UNDOC_RELATIONS = YES + +# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is +# available from the path. This tool is part of Graphviz, a graph visualization +# toolkit from AT&T and Lucent Bell Labs. The other options in this section +# have no effect if this option is set to NO (the default) + +HAVE_DOT = NO + +# The DOT_NUM_THREADS specifies the number of dot invocations doxygen is +# allowed to run in parallel. When set to 0 (the default) doxygen will +# base this on the number of processors available in the system. You can set it +# explicitly to a value larger than 0 to get control over the balance +# between CPU load and processing speed. + +DOT_NUM_THREADS = 0 + +# By default doxygen will write a font called FreeSans.ttf to the output +# directory and reference it in all dot files that doxygen generates. This +# font does not include all possible unicode characters however, so when you need +# these (or just want a differently looking font) you can specify the font name +# using DOT_FONTNAME. You need need to make sure dot is able to find the font, +# which can be done by putting it in a standard location or by setting the +# DOTFONTPATH environment variable or by setting DOT_FONTPATH to the directory +# containing the font. + +DOT_FONTNAME = FreeSans.ttf + +# The DOT_FONTSIZE tag can be used to set the size of the font of dot graphs. +# The default size is 10pt. + +DOT_FONTSIZE = 10 + +# By default doxygen will tell dot to use the output directory to look for the +# FreeSans.ttf font (which doxygen will put there itself). If you specify a +# different font using DOT_FONTNAME you can set the path where dot +# can find it using this tag. + +DOT_FONTPATH = + +# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect inheritance relations. Setting this tag to YES will force the +# the CLASS_DIAGRAMS tag to NO. + +CLASS_GRAPH = YES + +# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect implementation dependencies (inheritance, containment, and +# class references variables) of the class with other documented classes. + +COLLABORATION_GRAPH = YES + +# If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for groups, showing the direct groups dependencies + +GROUP_GRAPHS = YES + +# If the UML_LOOK tag is set to YES doxygen will generate inheritance and +# collaboration diagrams in a style similar to the OMG's Unified Modeling +# Language. + +UML_LOOK = NO + +# If set to YES, the inheritance and collaboration graphs will show the +# relations between templates and their instances. + +TEMPLATE_RELATIONS = NO + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT +# tags are set to YES then doxygen will generate a graph for each documented +# file showing the direct and indirect include dependencies of the file with +# other documented files. + +INCLUDE_GRAPH = YES + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and +# HAVE_DOT tags are set to YES then doxygen will generate a graph for each +# documented header file showing the documented files that directly or +# indirectly include this file. + +INCLUDED_BY_GRAPH = YES + +# If the CALL_GRAPH and HAVE_DOT options are set to YES then +# doxygen will generate a call dependency graph for every global function +# or class method. Note that enabling this option will significantly increase +# the time of a run. So in most cases it will be better to enable call graphs +# for selected functions only using the \callgraph command. + +CALL_GRAPH = NO + +# If the CALLER_GRAPH and HAVE_DOT tags are set to YES then +# doxygen will generate a caller dependency graph for every global function +# or class method. Note that enabling this option will significantly increase +# the time of a run. So in most cases it will be better to enable caller +# graphs for selected functions only using the \callergraph command. + +CALLER_GRAPH = NO + +# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen +# will graphical hierarchy of all classes instead of a textual one. + +GRAPHICAL_HIERARCHY = YES + +# If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES +# then doxygen will show the dependencies a directory has on other directories +# in a graphical way. The dependency relations are determined by the #include +# relations between the files in the directories. + +DIRECTORY_GRAPH = YES + +# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images +# generated by dot. Possible values are png, jpg, or gif +# If left blank png will be used. + +DOT_IMAGE_FORMAT = png + +# The tag DOT_PATH can be used to specify the path where the dot tool can be +# found. If left blank, it is assumed the dot tool can be found in the path. + +DOT_PATH = + +# The DOTFILE_DIRS tag can be used to specify one or more directories that +# contain dot files that are included in the documentation (see the +# \dotfile command). + +DOTFILE_DIRS = + +# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of +# nodes that will be shown in the graph. If the number of nodes in a graph +# becomes larger than this value, doxygen will truncate the graph, which is +# visualized by representing a node as a red box. Note that doxygen if the +# number of direct children of the root node in a graph is already larger than +# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note +# that the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH. + +DOT_GRAPH_MAX_NODES = 50 + +# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the +# graphs generated by dot. A depth value of 3 means that only nodes reachable +# from the root by following a path via at most 3 edges will be shown. Nodes +# that lay further from the root node will be omitted. Note that setting this +# option to 1 or 2 may greatly reduce the computation time needed for large +# code bases. Also note that the size of a graph can be further restricted by +# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction. + +MAX_DOT_GRAPH_DEPTH = 0 + +# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent +# background. This is disabled by default, because dot on Windows does not +# seem to support this out of the box. Warning: Depending on the platform used, +# enabling this option may lead to badly anti-aliased labels on the edges of +# a graph (i.e. they become hard to read). + +DOT_TRANSPARENT = NO + +# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output +# files in one run (i.e. multiple -o and -T options on the command line). This +# makes dot run faster, but since only newer versions of dot (>1.8.10) +# support this, this feature is disabled by default. + +DOT_MULTI_TARGETS = YES + +# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will +# generate a legend page explaining the meaning of the various boxes and +# arrows in the dot generated graphs. + +GENERATE_LEGEND = YES + +# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will +# remove the intermediate dot files that are used to generate +# the various graphs. + +DOT_CLEANUP = YES diff --git a/external/db_drivers/liblmdb64/LICENSE b/external/db_drivers/liblmdb64/LICENSE new file mode 100644 index 000000000..05ad7571e --- /dev/null +++ b/external/db_drivers/liblmdb64/LICENSE @@ -0,0 +1,47 @@ +The OpenLDAP Public License + Version 2.8, 17 August 2003 + +Redistribution and use of this software and associated documentation +("Software"), with or without modification, are permitted provided +that the following conditions are met: + +1. Redistributions in source form must retain copyright statements + and notices, + +2. Redistributions in binary form must reproduce applicable copyright + statements and notices, this list of conditions, and the following + disclaimer in the documentation and/or other materials provided + with the distribution, and + +3. Redistributions must contain a verbatim copy of this document. + +The OpenLDAP Foundation may revise this license from time to time. +Each revision is distinguished by a version number. You may use +this Software under terms of this license revision or under the +terms of any subsequent revision of the license. + +THIS SOFTWARE IS PROVIDED BY THE OPENLDAP FOUNDATION AND ITS +CONTRIBUTORS ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT +SHALL THE OPENLDAP FOUNDATION, ITS CONTRIBUTORS, OR THE AUTHOR(S) +OR OWNER(S) OF THE SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +The names of the authors and copyright holders must not be used in +advertising or otherwise to promote the sale, use or other dealing +in this Software without specific, written prior permission. Title +to copyright in this Software shall at all times remain with copyright +holders. + +OpenLDAP is a registered trademark of the OpenLDAP Foundation. + +Copyright 1999-2003 The OpenLDAP Foundation, Redwood City, +California, USA. All Rights Reserved. Permission to copy and +distribute verbatim copies of this document is granted. diff --git a/external/db_drivers/liblmdb64/Makefile b/external/db_drivers/liblmdb64/Makefile new file mode 100644 index 000000000..2d0983eff --- /dev/null +++ b/external/db_drivers/liblmdb64/Makefile @@ -0,0 +1,104 @@ +# Makefile for liblmdb (Lightning memory-mapped database library). + +######################################################################## +# Configuration. The compiler options must enable threaded compilation. +# +# Preprocessor macros (for CPPFLAGS) of interest... +# Note that the defaults should already be correct for most +# platforms; you should not need to change any of these. +# Read their descriptions in mdb.c if you do: +# +# - MDB_USE_POSIX_SEM +# - MDB_DSYNC +# - MDB_FDATASYNC +# - MDB_FDATASYNC_WORKS +# - MDB_USE_PWRITEV +# +# There may be other macros in mdb.c of interest. You should +# read mdb.c before changing any of them. +# +CC = gcc +W = -W -Wall -Wno-unused-parameter -Wbad-function-cast -Wuninitialized +THREADS = -pthread +OPT = -O2 -g +CFLAGS = $(THREADS) $(OPT) $(W) $(XCFLAGS) +LDLIBS = +SOLIBS = +prefix = /usr/local + +######################################################################## + +IHDRS = lmdb.h +ILIBS = liblmdb.a liblmdb.so +IPROGS = mdb_stat mdb_copy mdb_dump mdb_load +IDOCS = mdb_stat.1 mdb_copy.1 mdb_dump.1 mdb_load.1 +PROGS = $(IPROGS) mtest mtest2 mtest3 mtest4 mtest5 +all: $(ILIBS) $(PROGS) + +install: $(ILIBS) $(IPROGS) $(IHDRS) + for f in $(IPROGS); do cp $$f $(DESTDIR)$(prefix)/bin; done + for f in $(ILIBS); do cp $$f $(DESTDIR)$(prefix)/lib; done + for f in $(IHDRS); do cp $$f $(DESTDIR)$(prefix)/include; done + for f in $(IDOCS); do cp $$f $(DESTDIR)$(prefix)/man/man1; done + +clean: + rm -rf $(PROGS) *.[ao] *.[ls]o *~ testdb + +test: all + rm -rf testdb && mkdir testdb + ./mtest && ./mdb_stat testdb + +liblmdb.a: mdb.o midl.o + ar rs $@ mdb.o midl.o + +liblmdb.so: mdb.lo midl.lo +# $(CC) $(LDFLAGS) -pthread -shared -Wl,-Bsymbolic -o $@ mdb.o midl.o $(SOLIBS) + $(CC) $(LDFLAGS) -pthread -shared -o $@ mdb.lo midl.lo $(SOLIBS) + +mdb_stat: mdb_stat.o liblmdb.a +mdb_copy: mdb_copy.o liblmdb.a +mdb_dump: mdb_dump.o liblmdb.a +mdb_load: mdb_load.o liblmdb.a +mtest: mtest.o liblmdb.a +mtest2: mtest2.o liblmdb.a +mtest3: mtest3.o liblmdb.a +mtest4: mtest4.o liblmdb.a +mtest5: mtest5.o liblmdb.a +mtest6: mtest6.o liblmdb.a + +mdb.o: mdb.c lmdb.h midl.h + $(CC) $(CFLAGS) $(CPPFLAGS) -c mdb.c + +midl.o: midl.c midl.h + $(CC) $(CFLAGS) $(CPPFLAGS) -c midl.c + +mdb.lo: mdb.c lmdb.h midl.h + $(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -c mdb.c -o $@ + +midl.lo: midl.c midl.h + $(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -c midl.c -o $@ + +%: %.o + $(CC) $(CFLAGS) $(LDFLAGS) $^ $(LDLIBS) -o $@ + +%.o: %.c lmdb.h + $(CC) $(CFLAGS) $(CPPFLAGS) -c $< + +COV_FLAGS=-fprofile-arcs -ftest-coverage +COV_OBJS=xmdb.o xmidl.o + +coverage: xmtest + for i in mtest*.c [0-9]*.c; do j=`basename \$$i .c`; $(MAKE) $$j.o; \ + gcc -o x$$j $$j.o $(COV_OBJS) -pthread $(COV_FLAGS); \ + rm -rf testdb; mkdir testdb; ./x$$j; done + gcov xmdb.c + gcov xmidl.c + +xmtest: mtest.o xmdb.o xmidl.o + gcc -o xmtest mtest.o xmdb.o xmidl.o -pthread $(COV_FLAGS) + +xmdb.o: mdb.c lmdb.h midl.h + $(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -O0 $(COV_FLAGS) -c mdb.c -o $@ + +xmidl.o: midl.c midl.h + $(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -O0 $(COV_FLAGS) -c midl.c -o $@ diff --git a/external/db_drivers/liblmdb64/lmdb.h b/external/db_drivers/liblmdb64/lmdb.h new file mode 100644 index 000000000..117f16ea3 --- /dev/null +++ b/external/db_drivers/liblmdb64/lmdb.h @@ -0,0 +1,1554 @@ +/** @file lmdb.h + * @brief Lightning memory-mapped database library + * + * @mainpage Lightning Memory-Mapped Database Manager (LMDB) + * + * @section intro_sec Introduction + * LMDB is a Btree-based database management library modeled loosely on the + * BerkeleyDB API, but much simplified. The entire database is exposed + * in a memory map, and all data fetches return data directly + * from the mapped memory, so no malloc's or memcpy's occur during + * data fetches. As such, the library is extremely simple because it + * requires no page caching layer of its own, and it is extremely high + * performance and memory-efficient. It is also fully transactional with + * full ACID semantics, and when the memory map is read-only, the + * database integrity cannot be corrupted by stray pointer writes from + * application code. + * + * The library is fully thread-aware and supports concurrent read/write + * access from multiple processes and threads. Data pages use a copy-on- + * write strategy so no active data pages are ever overwritten, which + * also provides resistance to corruption and eliminates the need of any + * special recovery procedures after a system crash. Writes are fully + * serialized; only one write transaction may be active at a time, which + * guarantees that writers can never deadlock. The database structure is + * multi-versioned so readers run with no locks; writers cannot block + * readers, and readers don't block writers. + * + * Unlike other well-known database mechanisms which use either write-ahead + * transaction logs or append-only data writes, LMDB requires no maintenance + * during operation. Both write-ahead loggers and append-only databases + * require periodic checkpointing and/or compaction of their log or database + * files otherwise they grow without bound. LMDB tracks free pages within + * the database and re-uses them for new write operations, so the database + * size does not grow without bound in normal use. + * + * The memory map can be used as a read-only or read-write map. It is + * read-only by default as this provides total immunity to corruption. + * Using read-write mode offers much higher write performance, but adds + * the possibility for stray application writes thru pointers to silently + * corrupt the database. Of course if your application code is known to + * be bug-free (...) then this is not an issue. + * + * @section caveats_sec Caveats + * Troubleshooting the lock file, plus semaphores on BSD systems: + * + * - A broken lockfile can cause sync issues. + * Stale reader transactions left behind by an aborted program + * cause further writes to grow the database quickly, and + * stale locks can block further operation. + * + * Fix: Check for stale readers periodically, using the + * #mdb_reader_check function or the \ref mdb_stat_1 "mdb_stat" tool. Or just + * make all programs using the database close it; the lockfile + * is always reset on first open of the environment. + * + * - On BSD systems or others configured with MDB_USE_POSIX_SEM, + * startup can fail due to semaphores owned by another userid. + * + * Fix: Open and close the database as the user which owns the + * semaphores (likely last user) or as root, while no other + * process is using the database. + * + * Restrictions/caveats (in addition to those listed for some functions): + * + * - Only the database owner should normally use the database on + * BSD systems or when otherwise configured with MDB_USE_POSIX_SEM. + * Multiple users can cause startup to fail later, as noted above. + * + * - There is normally no pure read-only mode, since readers need write + * access to locks and lock file. Exceptions: On read-only filesystems + * or with the #MDB_NOLOCK flag described under #mdb_env_open(). + * + * - By default, in versions before 0.9.10, unused portions of the data + * file might receive garbage data from memory freed by other code. + * (This does not happen when using the #MDB_WRITEMAP flag.) As of + * 0.9.10 the default behavior is to initialize such memory before + * writing to the data file. Since there may be a slight performance + * cost due to this initialization, applications may disable it using + * the #MDB_NOMEMINIT flag. Applications handling sensitive data + * which must not be written should not use this flag. This flag is + * irrelevant when using #MDB_WRITEMAP. + * + * - A thread can only use one transaction at a time, plus any child + * transactions. Each transaction belongs to one thread. See below. + * The #MDB_NOTLS flag changes this for read-only transactions. + * + * - Use an MDB_env* in the process which opened it, without fork()ing. + * + * - Do not have open an LMDB database twice in the same process at + * the same time. Not even from a plain open() call - close()ing it + * breaks flock() advisory locking. + * + * - Avoid long-lived transactions. Read transactions prevent + * reuse of pages freed by newer write transactions, thus the + * database can grow quickly. Write transactions prevent + * other write transactions, since writes are serialized. + * + * - Avoid suspending a process with active transactions. These + * would then be "long-lived" as above. Also read transactions + * suspended when writers commit could sometimes see wrong data. + * + * ...when several processes can use a database concurrently: + * + * - Avoid aborting a process with an active transaction. + * The transaction becomes "long-lived" as above until a check + * for stale readers is performed or the lockfile is reset, + * since the process may not remove it from the lockfile. + * + * - If you do that anyway, do a periodic check for stale readers. Or + * close the environment once in a while, so the lockfile can get reset. + * + * - Do not use LMDB databases on remote filesystems, even between + * processes on the same host. This breaks flock() on some OSes, + * possibly memory map sync, and certainly sync between programs + * on different hosts. + * + * - Opening a database can fail if another process is opening or + * closing it at exactly the same time. + * + * @author Howard Chu, Symas Corporation. + * + * @copyright Copyright 2011-2015 Howard Chu, Symas Corp. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + * + * @par Derived From: + * This code is derived from btree.c written by Martin Hedenfalk. + * + * Copyright (c) 2009, 2010 Martin Hedenfalk <martin@bzero.se> + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#ifndef _LMDB_H_ +#define _LMDB_H_ + +#include <sys/types.h> + +#ifdef __cplusplus +extern "C" { +#endif + +/** Unix permissions for creating files, or dummy definition for Windows */ +#ifdef _MSC_VER +typedef int mdb_mode_t; +#else +typedef mode_t mdb_mode_t; +#endif + +/** An abstraction for a file handle. + * On POSIX systems file handles are small integers. On Windows + * they're opaque pointers. + */ +#ifdef _WIN32 +typedef void *mdb_filehandle_t; +#else +typedef int mdb_filehandle_t; +#endif + +/** @defgroup mdb LMDB API + * @{ + * @brief OpenLDAP Lightning Memory-Mapped Database Manager + */ +/** @defgroup Version Version Macros + * @{ + */ +/** Library major version */ +#define MDB_VERSION_MAJOR 0 +/** Library minor version */ +#define MDB_VERSION_MINOR 9 +/** Library patch version */ +#define MDB_VERSION_PATCH 14 + +/** Combine args a,b,c into a single integer for easy version comparisons */ +#define MDB_VERINT(a,b,c) (((a) << 24) | ((b) << 16) | (c)) + +/** The full library version as a single integer */ +#define MDB_VERSION_FULL \ + MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH) + +/** The release date of this library version */ +#define MDB_VERSION_DATE "September 20, 2014" + +/** A stringifier for the version info */ +#define MDB_VERSTR(a,b,c,d) "LMDB " #a "." #b "." #c ": (" d ")" + +/** A helper for the stringifier macro */ +#define MDB_VERFOO(a,b,c,d) MDB_VERSTR(a,b,c,d) + +/** The full library version as a C string */ +#define MDB_VERSION_STRING \ + MDB_VERFOO(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH,MDB_VERSION_DATE) +/** @} */ + +/** @brief Opaque structure for a database environment. + * + * A DB environment supports multiple databases, all residing in the same + * shared-memory map. + */ +typedef struct MDB_env MDB_env; + +/** @brief Opaque structure for a transaction handle. + * + * All database operations require a transaction handle. Transactions may be + * read-only or read-write. + */ +typedef struct MDB_txn MDB_txn; + +/** @brief A handle for an individual database in the DB environment. */ +typedef unsigned int MDB_dbi; + +/** @brief Opaque structure for navigating through a database */ +typedef struct MDB_cursor MDB_cursor; + +/** @brief Generic structure used for passing keys and data in and out + * of the database. + * + * Values returned from the database are valid only until a subsequent + * update operation, or the end of the transaction. Do not modify or + * free them, they commonly point into the database itself. + * + * Key sizes must be between 1 and #mdb_env_get_maxkeysize() inclusive. + * The same applies to data sizes in databases with the #MDB_DUPSORT flag. + * Other data items can in theory be from 0 to 0xffffffff bytes long. + */ +typedef struct MDB_val { + size_t mv_size; /**< size of the data item */ + void *mv_data; /**< address of the data item */ +} MDB_val; + +/** @brief A callback function used to compare two keys in a database */ +typedef int (MDB_cmp_func)(const MDB_val *a, const MDB_val *b); + +/** @brief A callback function used to relocate a position-dependent data item + * in a fixed-address database. + * + * The \b newptr gives the item's desired address in + * the memory map, and \b oldptr gives its previous address. The item's actual + * data resides at the address in \b item. This callback is expected to walk + * through the fields of the record in \b item and modify any + * values based at the \b oldptr address to be relative to the \b newptr address. + * @param[in,out] item The item that is to be relocated. + * @param[in] oldptr The previous address. + * @param[in] newptr The new address to relocate to. + * @param[in] relctx An application-provided context, set by #mdb_set_relctx(). + * @todo This feature is currently unimplemented. + */ +typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *relctx); + +/** @defgroup mdb_env Environment Flags + * @{ + */ + /** mmap at a fixed address (experimental) */ +#define MDB_FIXEDMAP 0x01 + /** no environment directory */ +#define MDB_NOSUBDIR 0x4000 + /** don't fsync after commit */ +#define MDB_NOSYNC 0x10000 + /** read only */ +#define MDB_RDONLY 0x20000 + /** don't fsync metapage after commit */ +#define MDB_NOMETASYNC 0x40000 + /** use writable mmap */ +#define MDB_WRITEMAP 0x80000 + /** use asynchronous msync when #MDB_WRITEMAP is used */ +#define MDB_MAPASYNC 0x100000 + /** tie reader locktable slots to #MDB_txn objects instead of to threads */ +#define MDB_NOTLS 0x200000 + /** don't do any locking, caller must manage their own locks */ +#define MDB_NOLOCK 0x400000 + /** don't do readahead (no effect on Windows) */ +#define MDB_NORDAHEAD 0x800000 + /** don't initialize malloc'd memory before writing to datafile */ +#define MDB_NOMEMINIT 0x1000000 +/** @} */ + +/** @defgroup mdb_dbi_open Database Flags + * @{ + */ + /** use reverse string keys */ +#define MDB_REVERSEKEY 0x02 + /** use sorted duplicates */ +#define MDB_DUPSORT 0x04 + /** numeric keys in native byte order. + * The keys must all be of the same size. */ +#define MDB_INTEGERKEY 0x08 + /** with #MDB_DUPSORT, sorted dup items have fixed size */ +#define MDB_DUPFIXED 0x10 + /** with #MDB_DUPSORT, dups are numeric in native byte order */ +#define MDB_INTEGERDUP 0x20 + /** with #MDB_DUPSORT, use reverse string dups */ +#define MDB_REVERSEDUP 0x40 + /** create DB if not already existing */ +#define MDB_CREATE 0x40000 +/** @} */ + +/** @defgroup mdb_put Write Flags + * @{ + */ +/** For put: Don't write if the key already exists. */ +#define MDB_NOOVERWRITE 0x10 +/** Only for #MDB_DUPSORT<br> + * For put: don't write if the key and data pair already exist.<br> + * For mdb_cursor_del: remove all duplicate data items. + */ +#define MDB_NODUPDATA 0x20 +/** For mdb_cursor_put: overwrite the current key/data pair */ +#define MDB_CURRENT 0x40 +/** For put: Just reserve space for data, don't copy it. Return a + * pointer to the reserved space. + */ +#define MDB_RESERVE 0x10000 +/** Data is being appended, don't split full pages. */ +#define MDB_APPEND 0x20000 +/** Duplicate data is being appended, don't split full pages. */ +#define MDB_APPENDDUP 0x40000 +/** Store multiple data items in one call. Only for #MDB_DUPFIXED. */ +#define MDB_MULTIPLE 0x80000 +/* @} */ + +/** @defgroup mdb_copy Copy Flags + * @{ + */ +/** Compacting copy: Omit free space from copy, and renumber all + * pages sequentially. + */ +#define MDB_CP_COMPACT 0x01 +/* @} */ + +/** @brief Cursor Get operations. + * + * This is the set of all operations for retrieving data + * using a cursor. + */ +typedef enum MDB_cursor_op { + MDB_FIRST, /**< Position at first key/data item */ + MDB_FIRST_DUP, /**< Position at first data item of current key. + Only for #MDB_DUPSORT */ + MDB_GET_BOTH, /**< Position at key/data pair. Only for #MDB_DUPSORT */ + MDB_GET_BOTH_RANGE, /**< position at key, nearest data. Only for #MDB_DUPSORT */ + MDB_GET_CURRENT, /**< Return key/data at current cursor position */ + MDB_GET_MULTIPLE, /**< Return key and up to a page of duplicate data items + from current cursor position. Move cursor to prepare + for #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED */ + MDB_LAST, /**< Position at last key/data item */ + MDB_LAST_DUP, /**< Position at last data item of current key. + Only for #MDB_DUPSORT */ + MDB_NEXT, /**< Position at next data item */ + MDB_NEXT_DUP, /**< Position at next data item of current key. + Only for #MDB_DUPSORT */ + MDB_NEXT_MULTIPLE, /**< Return key and up to a page of duplicate data items + from next cursor position. Move cursor to prepare + for #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED */ + MDB_NEXT_NODUP, /**< Position at first data item of next key */ + MDB_PREV, /**< Position at previous data item */ + MDB_PREV_DUP, /**< Position at previous data item of current key. + Only for #MDB_DUPSORT */ + MDB_PREV_NODUP, /**< Position at last data item of previous key */ + MDB_SET, /**< Position at specified key */ + MDB_SET_KEY, /**< Position at specified key, return key + data */ + MDB_SET_RANGE /**< Position at first key greater than or equal to specified key. */ +} MDB_cursor_op; + +/** @defgroup errors Return Codes + * + * BerkeleyDB uses -30800 to -30999, we'll go under them + * @{ + */ + /** Successful result */ +#define MDB_SUCCESS 0 + /** key/data pair already exists */ +#define MDB_KEYEXIST (-30799) + /** key/data pair not found (EOF) */ +#define MDB_NOTFOUND (-30798) + /** Requested page not found - this usually indicates corruption */ +#define MDB_PAGE_NOTFOUND (-30797) + /** Located page was wrong type */ +#define MDB_CORRUPTED (-30796) + /** Update of meta page failed, probably I/O error */ +#define MDB_PANIC (-30795) + /** Environment version mismatch */ +#define MDB_VERSION_MISMATCH (-30794) + /** File is not a valid LMDB file */ +#define MDB_INVALID (-30793) + /** Environment mapsize reached */ +#define MDB_MAP_FULL (-30792) + /** Environment maxdbs reached */ +#define MDB_DBS_FULL (-30791) + /** Environment maxreaders reached */ +#define MDB_READERS_FULL (-30790) + /** Too many TLS keys in use - Windows only */ +#define MDB_TLS_FULL (-30789) + /** Txn has too many dirty pages */ +#define MDB_TXN_FULL (-30788) + /** Cursor stack too deep - internal error */ +#define MDB_CURSOR_FULL (-30787) + /** Page has not enough space - internal error */ +#define MDB_PAGE_FULL (-30786) + /** Database contents grew beyond environment mapsize */ +#define MDB_MAP_RESIZED (-30785) + /** MDB_INCOMPATIBLE: Operation and DB incompatible, or DB flags changed */ +#define MDB_INCOMPATIBLE (-30784) + /** Invalid reuse of reader locktable slot */ +#define MDB_BAD_RSLOT (-30783) + /** Transaction cannot recover - it must be aborted */ +#define MDB_BAD_TXN (-30782) + /** Unsupported size of key/DB name/data, or wrong DUPFIXED size */ +#define MDB_BAD_VALSIZE (-30781) + /** The specified DBI was changed unexpectedly */ +#define MDB_BAD_DBI (-30780) + /** The last defined error code */ +#define MDB_LAST_ERRCODE MDB_BAD_DBI +/** @} */ + +/** @brief Statistics for a database in the environment */ +typedef struct MDB_stat { + unsigned int ms_psize; /**< Size of a database page. + This is currently the same for all databases. */ + unsigned int ms_depth; /**< Depth (height) of the B-tree */ + size_t ms_branch_pages; /**< Number of internal (non-leaf) pages */ + size_t ms_leaf_pages; /**< Number of leaf pages */ + size_t ms_overflow_pages; /**< Number of overflow pages */ + size_t ms_entries; /**< Number of data items */ +} MDB_stat; + +/** @brief Information about the environment */ +typedef struct MDB_envinfo { + void *me_mapaddr; /**< Address of map, if fixed */ + size_t me_mapsize; /**< Size of the data memory map */ + size_t me_last_pgno; /**< ID of the last used page */ + size_t me_last_txnid; /**< ID of the last committed transaction */ + unsigned int me_maxreaders; /**< max reader slots in the environment */ + unsigned int me_numreaders; /**< max reader slots used in the environment */ +} MDB_envinfo; + + /** @brief Return the LMDB library version information. + * + * @param[out] major if non-NULL, the library major version number is copied here + * @param[out] minor if non-NULL, the library minor version number is copied here + * @param[out] patch if non-NULL, the library patch version number is copied here + * @retval "version string" The library version as a string + */ +char *mdb_version(int *major, int *minor, int *patch); + + /** @brief Return a string describing a given error code. + * + * This function is a superset of the ANSI C X3.159-1989 (ANSI C) strerror(3) + * function. If the error code is greater than or equal to 0, then the string + * returned by the system function strerror(3) is returned. If the error code + * is less than 0, an error string corresponding to the LMDB library error is + * returned. See @ref errors for a list of LMDB-specific error codes. + * @param[in] err The error code + * @retval "error message" The description of the error + */ +char *mdb_strerror(int err); + + /** @brief Create an LMDB environment handle. + * + * This function allocates memory for a #MDB_env structure. To release + * the allocated memory and discard the handle, call #mdb_env_close(). + * Before the handle may be used, it must be opened using #mdb_env_open(). + * Various other options may also need to be set before opening the handle, + * e.g. #mdb_env_set_mapsize(), #mdb_env_set_maxreaders(), #mdb_env_set_maxdbs(), + * depending on usage requirements. + * @param[out] env The address where the new handle will be stored + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_create(MDB_env **env); + + /** @brief Open an environment handle. + * + * If this function fails, #mdb_env_close() must be called to discard the #MDB_env handle. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] path The directory in which the database files reside. This + * directory must already exist and be writable. + * @param[in] flags Special options for this environment. This parameter + * must be set to 0 or by bitwise OR'ing together one or more of the + * values described here. + * Flags set by mdb_env_set_flags() are also used. + * <ul> + * <li>#MDB_FIXEDMAP + * use a fixed address for the mmap region. This flag must be specified + * when creating the environment, and is stored persistently in the environment. + * If successful, the memory map will always reside at the same virtual address + * and pointers used to reference data items in the database will be constant + * across multiple invocations. This option may not always work, depending on + * how the operating system has allocated memory to shared libraries and other uses. + * The feature is highly experimental. + * <li>#MDB_NOSUBDIR + * By default, LMDB creates its environment in a directory whose + * pathname is given in \b path, and creates its data and lock files + * under that directory. With this option, \b path is used as-is for + * the database main data file. The database lock file is the \b path + * with "-lock" appended. + * <li>#MDB_RDONLY + * Open the environment in read-only mode. No write operations will be + * allowed. LMDB will still modify the lock file - except on read-only + * filesystems, where LMDB does not use locks. + * <li>#MDB_WRITEMAP + * Use a writeable memory map unless MDB_RDONLY is set. This is faster + * and uses fewer mallocs, but loses protection from application bugs + * like wild pointer writes and other bad updates into the database. + * Incompatible with nested transactions. + * Do not mix processes with and without MDB_WRITEMAP on the same + * environment. This can defeat durability (#mdb_env_sync etc). + * <li>#MDB_NOMETASYNC + * Flush system buffers to disk only once per transaction, omit the + * metadata flush. Defer that until the system flushes files to disk, + * or next non-MDB_RDONLY commit or #mdb_env_sync(). This optimization + * maintains database integrity, but a system crash may undo the last + * committed transaction. I.e. it preserves the ACI (atomicity, + * consistency, isolation) but not D (durability) database property. + * This flag may be changed at any time using #mdb_env_set_flags(). + * <li>#MDB_NOSYNC + * Don't flush system buffers to disk when committing a transaction. + * This optimization means a system crash can corrupt the database or + * lose the last transactions if buffers are not yet flushed to disk. + * The risk is governed by how often the system flushes dirty buffers + * to disk and how often #mdb_env_sync() is called. However, if the + * filesystem preserves write order and the #MDB_WRITEMAP flag is not + * used, transactions exhibit ACI (atomicity, consistency, isolation) + * properties and only lose D (durability). I.e. database integrity + * is maintained, but a system crash may undo the final transactions. + * Note that (#MDB_NOSYNC | #MDB_WRITEMAP) leaves the system with no + * hint for when to write transactions to disk, unless #mdb_env_sync() + * is called. (#MDB_MAPASYNC | #MDB_WRITEMAP) may be preferable. + * This flag may be changed at any time using #mdb_env_set_flags(). + * <li>#MDB_MAPASYNC + * When using #MDB_WRITEMAP, use asynchronous flushes to disk. + * As with #MDB_NOSYNC, a system crash can then corrupt the + * database or lose the last transactions. Calling #mdb_env_sync() + * ensures on-disk database integrity until next commit. + * This flag may be changed at any time using #mdb_env_set_flags(). + * <li>#MDB_NOTLS + * Don't use Thread-Local Storage. Tie reader locktable slots to + * #MDB_txn objects instead of to threads. I.e. #mdb_txn_reset() keeps + * the slot reseved for the #MDB_txn object. A thread may use parallel + * read-only transactions. A read-only transaction may span threads if + * the user synchronizes its use. Applications that multiplex many + * user threads over individual OS threads need this option. Such an + * application must also serialize the write transactions in an OS + * thread, since LMDB's write locking is unaware of the user threads. + * <li>#MDB_NOLOCK + * Don't do any locking. If concurrent access is anticipated, the + * caller must manage all concurrency itself. For proper operation + * the caller must enforce single-writer semantics, and must ensure + * that no readers are using old transactions while a writer is + * active. The simplest approach is to use an exclusive lock so that + * no readers may be active at all when a writer begins. + * <li>#MDB_NORDAHEAD + * Turn off readahead. Most operating systems perform readahead on + * read requests by default. This option turns it off if the OS + * supports it. Turning it off may help random read performance + * when the DB is larger than RAM and system RAM is full. + * The option is not implemented on Windows. + * <li>#MDB_NOMEMINIT + * Don't initialize malloc'd memory before writing to unused spaces + * in the data file. By default, memory for pages written to the data + * file is obtained using malloc. While these pages may be reused in + * subsequent transactions, freshly malloc'd pages will be initialized + * to zeroes before use. This avoids persisting leftover data from other + * code (that used the heap and subsequently freed the memory) into the + * data file. Note that many other system libraries may allocate + * and free memory from the heap for arbitrary uses. E.g., stdio may + * use the heap for file I/O buffers. This initialization step has a + * modest performance cost so some applications may want to disable + * it using this flag. This option can be a problem for applications + * which handle sensitive data like passwords, and it makes memory + * checkers like Valgrind noisy. This flag is not needed with #MDB_WRITEMAP, + * which writes directly to the mmap instead of using malloc for pages. The + * initialization is also skipped if #MDB_RESERVE is used; the + * caller is expected to overwrite all of the memory that was + * reserved in that case. + * This flag may be changed at any time using #mdb_env_set_flags(). + * </ul> + * @param[in] mode The UNIX permissions to set on created files. This parameter + * is ignored on Windows. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>#MDB_VERSION_MISMATCH - the version of the LMDB library doesn't match the + * version that created the database environment. + * <li>#MDB_INVALID - the environment file headers are corrupted. + * <li>ENOENT - the directory specified by the path parameter doesn't exist. + * <li>EACCES - the user didn't have permission to access the environment files. + * <li>EAGAIN - the environment was locked by another process. + * </ul> + */ +int mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode); + + /** @brief Copy an LMDB environment to the specified path. + * + * This function may be used to make a backup of an existing environment. + * No lockfile is created, since it gets recreated at need. + * @note This call can trigger significant file size growth if run in + * parallel with write transactions, because it employs a read-only + * transaction. See long-lived transactions under @ref caveats_sec. + * @param[in] env An environment handle returned by #mdb_env_create(). It + * must have already been opened successfully. + * @param[in] path The directory in which the copy will reside. This + * directory must already exist and be writable but must otherwise be + * empty. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_copy(MDB_env *env, const char *path); + + /** @brief Copy an LMDB environment to the specified file descriptor. + * + * This function may be used to make a backup of an existing environment. + * No lockfile is created, since it gets recreated at need. + * @note This call can trigger significant file size growth if run in + * parallel with write transactions, because it employs a read-only + * transaction. See long-lived transactions under @ref caveats_sec. + * @param[in] env An environment handle returned by #mdb_env_create(). It + * must have already been opened successfully. + * @param[in] fd The filedescriptor to write the copy to. It must + * have already been opened for Write access. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_copyfd(MDB_env *env, mdb_filehandle_t fd); + + /** @brief Copy an LMDB environment to the specified path, with options. + * + * This function may be used to make a backup of an existing environment. + * No lockfile is created, since it gets recreated at need. + * @note This call can trigger significant file size growth if run in + * parallel with write transactions, because it employs a read-only + * transaction. See long-lived transactions under @ref caveats_sec. + * @param[in] env An environment handle returned by #mdb_env_create(). It + * must have already been opened successfully. + * @param[in] path The directory in which the copy will reside. This + * directory must already exist and be writable but must otherwise be + * empty. + * @param[in] flags Special options for this operation. This parameter + * must be set to 0 or by bitwise OR'ing together one or more of the + * values described here. + * <ul> + * <li>#MDB_CP_COMPACT - Perform compaction while copying: omit free + * pages and sequentially renumber all pages in output. This option + * consumes more CPU and runs more slowly than the default. + * </ul> + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags); + + /** @brief Copy an LMDB environment to the specified file descriptor, + * with options. + * + * This function may be used to make a backup of an existing environment. + * No lockfile is created, since it gets recreated at need. See + * #mdb_env_copy2() for further details. + * @note This call can trigger significant file size growth if run in + * parallel with write transactions, because it employs a read-only + * transaction. See long-lived transactions under @ref caveats_sec. + * @param[in] env An environment handle returned by #mdb_env_create(). It + * must have already been opened successfully. + * @param[in] fd The filedescriptor to write the copy to. It must + * have already been opened for Write access. + * @param[in] flags Special options for this operation. + * See #mdb_env_copy2() for options. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_copyfd2(MDB_env *env, mdb_filehandle_t fd, unsigned int flags); + + /** @brief Return statistics about the LMDB environment. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] stat The address of an #MDB_stat structure + * where the statistics will be copied + */ +int mdb_env_stat(MDB_env *env, MDB_stat *stat); + + /** @brief Return information about the LMDB environment. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] stat The address of an #MDB_envinfo structure + * where the information will be copied + */ +int mdb_env_info(MDB_env *env, MDB_envinfo *stat); + + /** @brief Flush the data buffers to disk. + * + * Data is always written to disk when #mdb_txn_commit() is called, + * but the operating system may keep it buffered. LMDB always flushes + * the OS buffers upon commit as well, unless the environment was + * opened with #MDB_NOSYNC or in part #MDB_NOMETASYNC. This call is + * not valid if the environment was opened with #MDB_RDONLY. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] force If non-zero, force a synchronous flush. Otherwise + * if the environment has the #MDB_NOSYNC flag set the flushes + * will be omitted, and with #MDB_MAPASYNC they will be asynchronous. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EACCES - the environment is read-only. + * <li>EINVAL - an invalid parameter was specified. + * <li>EIO - an error occurred during synchronization. + * </ul> + */ +int mdb_env_sync(MDB_env *env, int force); + + /** @brief Close the environment and release the memory map. + * + * Only a single thread may call this function. All transactions, databases, + * and cursors must already be closed before calling this function. Attempts to + * use any such handles after calling this function will cause a SIGSEGV. + * The environment handle will be freed and must not be used again after this call. + * @param[in] env An environment handle returned by #mdb_env_create() + */ +void mdb_env_close(MDB_env *env); + + /** @brief Set environment flags. + * + * This may be used to set some flags in addition to those from + * #mdb_env_open(), or to unset these flags. If several threads + * change the flags at the same time, the result is undefined. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] flags The flags to change, bitwise OR'ed together + * @param[in] onoff A non-zero value sets the flags, zero clears them. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_env_set_flags(MDB_env *env, unsigned int flags, int onoff); + + /** @brief Get environment flags. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] flags The address of an integer to store the flags + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_env_get_flags(MDB_env *env, unsigned int *flags); + + /** @brief Return the path that was used in #mdb_env_open(). + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] path Address of a string pointer to contain the path. This + * is the actual string in the environment, not a copy. It should not be + * altered in any way. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_env_get_path(MDB_env *env, const char **path); + + /** @brief Return the filedescriptor for the given environment. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] fd Address of a mdb_filehandle_t to contain the descriptor. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_env_get_fd(MDB_env *env, mdb_filehandle_t *fd); + + /** @brief Set the size of the memory map to use for this environment. + * + * The size should be a multiple of the OS page size. The default is + * 10485760 bytes. The size of the memory map is also the maximum size + * of the database. The value should be chosen as large as possible, + * to accommodate future growth of the database. + * This function should be called after #mdb_env_create() and before #mdb_env_open(). + * It may be called at later times if no transactions are active in + * this process. Note that the library does not check for this condition, + * the caller must ensure it explicitly. + * + * The new size takes effect immediately for the current process but + * will not be persisted to any others until a write transaction has been + * committed by the current process. Also, only mapsize increases are + * persisted into the environment. + * + * If the mapsize is increased by another process, and data has grown + * beyond the range of the current mapsize, #mdb_txn_begin() will + * return #MDB_MAP_RESIZED. This function may be called with a size + * of zero to adopt the new size. + * + * Any attempt to set a size smaller than the space already consumed + * by the environment will be silently changed to the current size of the used space. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] size The size in bytes + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified, or the environment has + * an active write transaction. + * </ul> + */ +int mdb_env_set_mapsize(MDB_env *env, size_t size); + + /** @brief Set the maximum number of threads/reader slots for the environment. + * + * This defines the number of slots in the lock table that is used to track readers in the + * the environment. The default is 126. + * Starting a read-only transaction normally ties a lock table slot to the + * current thread until the environment closes or the thread exits. If + * MDB_NOTLS is in use, #mdb_txn_begin() instead ties the slot to the + * MDB_txn object until it or the #MDB_env object is destroyed. + * This function may only be called after #mdb_env_create() and before #mdb_env_open(). + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] readers The maximum number of reader lock table slots + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified, or the environment is already open. + * </ul> + */ +int mdb_env_set_maxreaders(MDB_env *env, unsigned int readers); + + /** @brief Get the maximum number of threads/reader slots for the environment. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] readers Address of an integer to store the number of readers + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_env_get_maxreaders(MDB_env *env, unsigned int *readers); + + /** @brief Set the maximum number of named databases for the environment. + * + * This function is only needed if multiple databases will be used in the + * environment. Simpler applications that use the environment as a single + * unnamed database can ignore this option. + * This function may only be called after #mdb_env_create() and before #mdb_env_open(). + * + * Currently a moderate number of slots are cheap but a huge number gets + * expensive: 7-120 words per transaction, and every #mdb_dbi_open() + * does a linear search of the opened slots. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] dbs The maximum number of databases + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified, or the environment is already open. + * </ul> + */ +int mdb_env_set_maxdbs(MDB_env *env, MDB_dbi dbs); + + /** @brief Get the maximum size of keys and #MDB_DUPSORT data we can write. + * + * Depends on the compile-time constant #MDB_MAXKEYSIZE. Default 511. + * See @ref MDB_val. + * @param[in] env An environment handle returned by #mdb_env_create() + * @return The maximum size of a key we can write + */ +int mdb_env_get_maxkeysize(MDB_env *env); + + /** @brief Set application information associated with the #MDB_env. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] ctx An arbitrary pointer for whatever the application needs. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_set_userctx(MDB_env *env, void *ctx); + + /** @brief Get the application information associated with the #MDB_env. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @return The pointer set by #mdb_env_set_userctx(). + */ +void *mdb_env_get_userctx(MDB_env *env); + + /** @brief A callback function for most LMDB assert() failures, + * called before printing the message and aborting. + * + * @param[in] env An environment handle returned by #mdb_env_create(). + * @param[in] msg The assertion message, not including newline. + */ +typedef void MDB_assert_func(MDB_env *env, const char *msg); + + /** Set or reset the assert() callback of the environment. + * Disabled if liblmdb is buillt with NDEBUG. + * @note This hack should become obsolete as lmdb's error handling matures. + * @param[in] env An environment handle returned by #mdb_env_create(). + * @param[in] func An #MDB_assert_func function, or 0. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_env_set_assert(MDB_env *env, MDB_assert_func *func); + + /** @brief Create a transaction for use with the environment. + * + * The transaction handle may be discarded using #mdb_txn_abort() or #mdb_txn_commit(). + * @note A transaction and its cursors must only be used by a single + * thread, and a thread may only have a single transaction at a time. + * If #MDB_NOTLS is in use, this does not apply to read-only transactions. + * @note Cursors may not span transactions. + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] parent If this parameter is non-NULL, the new transaction + * will be a nested transaction, with the transaction indicated by \b parent + * as its parent. Transactions may be nested to any level. A parent + * transaction and its cursors may not issue any other operations than + * mdb_txn_commit and mdb_txn_abort while it has active child transactions. + * @param[in] flags Special options for this transaction. This parameter + * must be set to 0 or by bitwise OR'ing together one or more of the + * values described here. + * <ul> + * <li>#MDB_RDONLY + * This transaction will not perform any write operations. + * </ul> + * @param[out] txn Address where the new #MDB_txn handle will be stored + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>#MDB_PANIC - a fatal error occurred earlier and the environment + * must be shut down. + * <li>#MDB_MAP_RESIZED - another process wrote data beyond this MDB_env's + * mapsize and this environment's map must be resized as well. + * See #mdb_env_set_mapsize(). + * <li>#MDB_READERS_FULL - a read-only transaction was requested and + * the reader lock table is full. See #mdb_env_set_maxreaders(). + * <li>ENOMEM - out of memory. + * </ul> + */ +int mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **txn); + + /** @brief Returns the transaction's #MDB_env + * + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + */ +MDB_env *mdb_txn_env(MDB_txn *txn); + + /** @brief Commit all the operations of a transaction into the database. + * + * The transaction handle is freed. It and its cursors must not be used + * again after this call, except with #mdb_cursor_renew(). + * @note Earlier documentation incorrectly said all cursors would be freed. + * Only write-transactions free cursors. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified. + * <li>ENOSPC - no more disk space. + * <li>EIO - a low-level I/O error occurred while writing. + * <li>ENOMEM - out of memory. + * </ul> + */ +int mdb_txn_commit(MDB_txn *txn); + + /** @brief Abandon all the operations of the transaction instead of saving them. + * + * The transaction handle is freed. It and its cursors must not be used + * again after this call, except with #mdb_cursor_renew(). + * @note Earlier documentation incorrectly said all cursors would be freed. + * Only write-transactions free cursors. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + */ +void mdb_txn_abort(MDB_txn *txn); + + /** @brief Reset a read-only transaction. + * + * Abort the transaction like #mdb_txn_abort(), but keep the transaction + * handle. #mdb_txn_renew() may reuse the handle. This saves allocation + * overhead if the process will start a new read-only transaction soon, + * and also locking overhead if #MDB_NOTLS is in use. The reader table + * lock is released, but the table slot stays tied to its thread or + * #MDB_txn. Use mdb_txn_abort() to discard a reset handle, and to free + * its lock table slot if MDB_NOTLS is in use. + * Cursors opened within the transaction must not be used + * again after this call, except with #mdb_cursor_renew(). + * Reader locks generally don't interfere with writers, but they keep old + * versions of database pages allocated. Thus they prevent the old pages + * from being reused when writers commit new data, and so under heavy load + * the database size may grow much more rapidly than otherwise. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + */ +void mdb_txn_reset(MDB_txn *txn); + + /** @brief Renew a read-only transaction. + * + * This acquires a new reader lock for a transaction handle that had been + * released by #mdb_txn_reset(). It must be called before a reset transaction + * may be used again. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>#MDB_PANIC - a fatal error occurred earlier and the environment + * must be shut down. + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_txn_renew(MDB_txn *txn); + +/** Compat with version <= 0.9.4, avoid clash with libmdb from MDB Tools project */ +#define mdb_open(txn,name,flags,dbi) mdb_dbi_open(txn,name,flags,dbi) +/** Compat with version <= 0.9.4, avoid clash with libmdb from MDB Tools project */ +#define mdb_close(env,dbi) mdb_dbi_close(env,dbi) + + /** @brief Open a database in the environment. + * + * A database handle denotes the name and parameters of a database, + * independently of whether such a database exists. + * The database handle may be discarded by calling #mdb_dbi_close(). + * The old database handle is returned if the database was already open. + * The handle may only be closed once. + * The database handle will be private to the current transaction until + * the transaction is successfully committed. If the transaction is + * aborted the handle will be closed automatically. + * After a successful commit the + * handle will reside in the shared environment, and may be used + * by other transactions. This function must not be called from + * multiple concurrent transactions in the same process. A transaction + * that uses this function must finish (either commit or abort) before + * any other transaction in the process may use this function. + * + * To use named databases (with name != NULL), #mdb_env_set_maxdbs() + * must be called before opening the environment. Database names + * are kept as keys in the unnamed database. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] name The name of the database to open. If only a single + * database is needed in the environment, this value may be NULL. + * @param[in] flags Special options for this database. This parameter + * must be set to 0 or by bitwise OR'ing together one or more of the + * values described here. + * <ul> + * <li>#MDB_REVERSEKEY + * Keys are strings to be compared in reverse order, from the end + * of the strings to the beginning. By default, Keys are treated as strings and + * compared from beginning to end. + * <li>#MDB_DUPSORT + * Duplicate keys may be used in the database. (Or, from another perspective, + * keys may have multiple data items, stored in sorted order.) By default + * keys must be unique and may have only a single data item. + * <li>#MDB_INTEGERKEY + * Keys are binary integers in native byte order. Setting this option + * requires all keys to be the same size, typically sizeof(int) + * or sizeof(size_t). + * <li>#MDB_DUPFIXED + * This flag may only be used in combination with #MDB_DUPSORT. This option + * tells the library that the data items for this database are all the same + * size, which allows further optimizations in storage and retrieval. When + * all data items are the same size, the #MDB_GET_MULTIPLE and #MDB_NEXT_MULTIPLE + * cursor operations may be used to retrieve multiple items at once. + * <li>#MDB_INTEGERDUP + * This option specifies that duplicate data items are also integers, and + * should be sorted as such. + * <li>#MDB_REVERSEDUP + * This option specifies that duplicate data items should be compared as + * strings in reverse order. + * <li>#MDB_CREATE + * Create the named database if it doesn't exist. This option is not + * allowed in a read-only transaction or a read-only environment. + * </ul> + * @param[out] dbi Address where the new #MDB_dbi handle will be stored + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>#MDB_NOTFOUND - the specified database doesn't exist in the environment + * and #MDB_CREATE was not specified. + * <li>#MDB_DBS_FULL - too many databases have been opened. See #mdb_env_set_maxdbs(). + * </ul> + */ +int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *dbi); + + /** @brief Retrieve statistics for a database. + * + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[out] stat The address of an #MDB_stat structure + * where the statistics will be copied + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *stat); + + /** @brief Retrieve the DB flags for a database handle. + * + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[out] flags Address where the flags will be returned. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_dbi_flags(MDB_txn *txn, MDB_dbi dbi, unsigned int *flags); + + /** @brief Close a database handle. Normally unnecessary. Use with care: + * + * This call is not mutex protected. Handles should only be closed by + * a single thread, and only if no other threads are going to reference + * the database handle or one of its cursors any further. Do not close + * a handle if an existing transaction has modified its database. + * Doing so can cause misbehavior from database corruption to errors + * like MDB_BAD_VALSIZE (since the DB name is gone). + * + * Closing a database handle is not necessary, but lets #mdb_dbi_open() + * reuse the handle value. Usually it's better to set a bigger + * #mdb_env_set_maxdbs(), unless that value would be large. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + */ +void mdb_dbi_close(MDB_env *env, MDB_dbi dbi); + + /** @brief Empty or delete+close a database. + * + * See #mdb_dbi_close() for restrictions about closing the DB handle. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] del 0 to empty the DB, 1 to delete it from the + * environment and close the DB handle. + * @return A non-zero error value on failure and 0 on success. + */ +int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del); + + /** @brief Set a custom key comparison function for a database. + * + * The comparison function is called whenever it is necessary to compare a + * key specified by the application with a key currently stored in the database. + * If no comparison function is specified, and no special key flags were specified + * with #mdb_dbi_open(), the keys are compared lexically, with shorter keys collating + * before longer keys. + * @warning This function must be called before any data access functions are used, + * otherwise data corruption may occur. The same comparison function must be used by every + * program accessing the database, every time the database is used. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] cmp A #MDB_cmp_func function + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_set_compare(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp); + + /** @brief Set a custom data comparison function for a #MDB_DUPSORT database. + * + * This comparison function is called whenever it is necessary to compare a data + * item specified by the application with a data item currently stored in the database. + * This function only takes effect if the database was opened with the #MDB_DUPSORT + * flag. + * If no comparison function is specified, and no special key flags were specified + * with #mdb_dbi_open(), the data items are compared lexically, with shorter items collating + * before longer items. + * @warning This function must be called before any data access functions are used, + * otherwise data corruption may occur. The same comparison function must be used by every + * program accessing the database, every time the database is used. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] cmp A #MDB_cmp_func function + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_set_dupsort(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp); + + /** @brief Set a relocation function for a #MDB_FIXEDMAP database. + * + * @todo The relocation function is called whenever it is necessary to move the data + * of an item to a different position in the database (e.g. through tree + * balancing operations, shifts as a result of adds or deletes, etc.). It is + * intended to allow address/position-dependent data items to be stored in + * a database in an environment opened with the #MDB_FIXEDMAP option. + * Currently the relocation feature is unimplemented and setting + * this function has no effect. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] rel A #MDB_rel_func function + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_set_relfunc(MDB_txn *txn, MDB_dbi dbi, MDB_rel_func *rel); + + /** @brief Set a context pointer for a #MDB_FIXEDMAP database's relocation function. + * + * See #mdb_set_relfunc and #MDB_rel_func for more details. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] ctx An arbitrary pointer for whatever the application needs. + * It will be passed to the callback function set by #mdb_set_relfunc + * as its \b relctx parameter whenever the callback is invoked. + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_set_relctx(MDB_txn *txn, MDB_dbi dbi, void *ctx); + + /** @brief Get items from a database. + * + * This function retrieves key/data pairs from the database. The address + * and length of the data associated with the specified \b key are returned + * in the structure to which \b data refers. + * If the database supports duplicate keys (#MDB_DUPSORT) then the + * first data item for the key will be returned. Retrieval of other + * items requires the use of #mdb_cursor_get(). + * + * @note The memory pointed to by the returned values is owned by the + * database. The caller need not dispose of the memory, and may not + * modify it in any way. For values returned in a read-only transaction + * any modification attempts will cause a SIGSEGV. + * @note Values returned from the database are valid only until a + * subsequent update operation, or the end of the transaction. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] key The key to search for in the database + * @param[out] data The data corresponding to the key + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>#MDB_NOTFOUND - the key was not in the database. + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_get(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data); + + /** @brief Store items into a database. + * + * This function stores key/data pairs in the database. The default behavior + * is to enter the new key/data pair, replacing any previously existing key + * if duplicates are disallowed, or adding a duplicate data item if + * duplicates are allowed (#MDB_DUPSORT). + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] key The key to store in the database + * @param[in,out] data The data to store + * @param[in] flags Special options for this operation. This parameter + * must be set to 0 or by bitwise OR'ing together one or more of the + * values described here. + * <ul> + * <li>#MDB_NODUPDATA - enter the new key/data pair only if it does not + * already appear in the database. This flag may only be specified + * if the database was opened with #MDB_DUPSORT. The function will + * return #MDB_KEYEXIST if the key/data pair already appears in the + * database. + * <li>#MDB_NOOVERWRITE - enter the new key/data pair only if the key + * does not already appear in the database. The function will return + * #MDB_KEYEXIST if the key already appears in the database, even if + * the database supports duplicates (#MDB_DUPSORT). The \b data + * parameter will be set to point to the existing item. + * <li>#MDB_RESERVE - reserve space for data of the given size, but + * don't copy the given data. Instead, return a pointer to the + * reserved space, which the caller can fill in later - before + * the next update operation or the transaction ends. This saves + * an extra memcpy if the data is being generated later. + * LMDB does nothing else with this memory, the caller is expected + * to modify all of the space requested. + * <li>#MDB_APPEND - append the given key/data pair to the end of the + * database. This option allows fast bulk loading when keys are + * already known to be in the correct order. Loading unsorted keys + * with this flag will cause a #MDB_KEYEXIST error. + * <li>#MDB_APPENDDUP - as above, but for sorted dup data. + * </ul> + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>#MDB_MAP_FULL - the database is full, see #mdb_env_set_mapsize(). + * <li>#MDB_TXN_FULL - the transaction has too many dirty pages. + * <li>EACCES - an attempt was made to write in a read-only transaction. + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_put(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data, + unsigned int flags); + + /** @brief Delete items from a database. + * + * This function removes key/data pairs from the database. + * If the database does not support sorted duplicate data items + * (#MDB_DUPSORT) the data parameter is ignored. + * If the database supports sorted duplicates and the data parameter + * is NULL, all of the duplicate data items for the key will be + * deleted. Otherwise, if the data parameter is non-NULL + * only the matching data item will be deleted. + * This function will return #MDB_NOTFOUND if the specified key/data + * pair is not in the database. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] key The key to delete from the database + * @param[in] data The data to delete + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EACCES - an attempt was made to write in a read-only transaction. + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_del(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data); + + /** @brief Create a cursor handle. + * + * A cursor is associated with a specific transaction and database. + * A cursor cannot be used when its database handle is closed. Nor + * when its transaction has ended, except with #mdb_cursor_renew(). + * It can be discarded with #mdb_cursor_close(). + * A cursor in a write-transaction can be closed before its transaction + * ends, and will otherwise be closed when its transaction ends. + * A cursor in a read-only transaction must be closed explicitly, before + * or after its transaction ends. It can be reused with + * #mdb_cursor_renew() before finally closing it. + * @note Earlier documentation said that cursors in every transaction + * were closed when the transaction committed or aborted. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[out] cursor Address where the new #MDB_cursor handle will be stored + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **cursor); + + /** @brief Close a cursor handle. + * + * The cursor handle will be freed and must not be used again after this call. + * Its transaction must still be live if it is a write-transaction. + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + */ +void mdb_cursor_close(MDB_cursor *cursor); + + /** @brief Renew a cursor handle. + * + * A cursor is associated with a specific transaction and database. + * Cursors that are only used in read-only + * transactions may be re-used, to avoid unnecessary malloc/free overhead. + * The cursor may be associated with a new read-only transaction, and + * referencing the same database handle as it was created with. + * This may be done whether the previous transaction is live or dead. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_cursor_renew(MDB_txn *txn, MDB_cursor *cursor); + + /** @brief Return the cursor's transaction handle. + * + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + */ +MDB_txn *mdb_cursor_txn(MDB_cursor *cursor); + + /** @brief Return the cursor's database handle. + * + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + */ +MDB_dbi mdb_cursor_dbi(MDB_cursor *cursor); + + /** @brief Retrieve by cursor. + * + * This function retrieves key/data pairs from the database. The address and length + * of the key are returned in the object to which \b key refers (except for the + * case of the #MDB_SET option, in which the \b key object is unchanged), and + * the address and length of the data are returned in the object to which \b data + * refers. + * See #mdb_get() for restrictions on using the output values. + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + * @param[in,out] key The key for a retrieved item + * @param[in,out] data The data of a retrieved item + * @param[in] op A cursor operation #MDB_cursor_op + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>#MDB_NOTFOUND - no matching key found. + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_cursor_get(MDB_cursor *cursor, MDB_val *key, MDB_val *data, + MDB_cursor_op op); + + /** @brief Store by cursor. + * + * This function stores key/data pairs into the database. + * The cursor is positioned at the new item, or on failure usually near it. + * @note Earlier documentation incorrectly said errors would leave the + * state of the cursor unchanged. + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + * @param[in] key The key operated on. + * @param[in] data The data operated on. + * @param[in] flags Options for this operation. This parameter + * must be set to 0 or one of the values described here. + * <ul> + * <li>#MDB_CURRENT - replace the item at the current cursor position. + * The \b key parameter must still be provided, and must match it. + * If using sorted duplicates (#MDB_DUPSORT) the data item must still + * sort into the same place. This is intended to be used when the + * new data is the same size as the old. Otherwise it will simply + * perform a delete of the old record followed by an insert. + * <li>#MDB_NODUPDATA - enter the new key/data pair only if it does not + * already appear in the database. This flag may only be specified + * if the database was opened with #MDB_DUPSORT. The function will + * return #MDB_KEYEXIST if the key/data pair already appears in the + * database. + * <li>#MDB_NOOVERWRITE - enter the new key/data pair only if the key + * does not already appear in the database. The function will return + * #MDB_KEYEXIST if the key already appears in the database, even if + * the database supports duplicates (#MDB_DUPSORT). + * <li>#MDB_RESERVE - reserve space for data of the given size, but + * don't copy the given data. Instead, return a pointer to the + * reserved space, which the caller can fill in later. This saves + * an extra memcpy if the data is being generated later. + * <li>#MDB_APPEND - append the given key/data pair to the end of the + * database. No key comparisons are performed. This option allows + * fast bulk loading when keys are already known to be in the + * correct order. Loading unsorted keys with this flag will cause + * data corruption. + * <li>#MDB_APPENDDUP - as above, but for sorted dup data. + * <li>#MDB_MULTIPLE - store multiple contiguous data elements in a + * single request. This flag may only be specified if the database + * was opened with #MDB_DUPFIXED. The \b data argument must be an + * array of two MDB_vals. The mv_size of the first MDB_val must be + * the size of a single data element. The mv_data of the first MDB_val + * must point to the beginning of the array of contiguous data elements. + * The mv_size of the second MDB_val must be the count of the number + * of data elements to store. On return this field will be set to + * the count of the number of elements actually written. The mv_data + * of the second MDB_val is unused. + * </ul> + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>#MDB_MAP_FULL - the database is full, see #mdb_env_set_mapsize(). + * <li>#MDB_TXN_FULL - the transaction has too many dirty pages. + * <li>EACCES - an attempt was made to modify a read-only database. + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_cursor_put(MDB_cursor *cursor, MDB_val *key, MDB_val *data, + unsigned int flags); + + /** @brief Delete current key/data pair + * + * This function deletes the key/data pair to which the cursor refers. + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + * @param[in] flags Options for this operation. This parameter + * must be set to 0 or one of the values described here. + * <ul> + * <li>#MDB_NODUPDATA - delete all of the data items for the current key. + * This flag may only be specified if the database was opened with #MDB_DUPSORT. + * </ul> + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EACCES - an attempt was made to modify a read-only database. + * <li>EINVAL - an invalid parameter was specified. + * </ul> + */ +int mdb_cursor_del(MDB_cursor *cursor, unsigned int flags); + + /** @brief Return count of duplicates for current key. + * + * This call is only valid on databases that support sorted duplicate + * data items #MDB_DUPSORT. + * @param[in] cursor A cursor handle returned by #mdb_cursor_open() + * @param[out] countp Address where the count will be stored + * @return A non-zero error value on failure and 0 on success. Some possible + * errors are: + * <ul> + * <li>EINVAL - cursor is not initialized, or an invalid parameter was specified. + * </ul> + */ +int mdb_cursor_count(MDB_cursor *cursor, size_t *countp); + + /** @brief Compare two data items according to a particular database. + * + * This returns a comparison as if the two data items were keys in the + * specified database. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] a The first item to compare + * @param[in] b The second item to compare + * @return < 0 if a < b, 0 if a == b, > 0 if a > b + */ +int mdb_cmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b); + + /** @brief Compare two data items according to a particular database. + * + * This returns a comparison as if the two items were data items of + * the specified database. The database must have the #MDB_DUPSORT flag. + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + * @param[in] a The first item to compare + * @param[in] b The second item to compare + * @return < 0 if a < b, 0 if a == b, > 0 if a > b + */ +int mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b); + + /** @brief A callback function used to print a message from the library. + * + * @param[in] msg The string to be printed. + * @param[in] ctx An arbitrary context pointer for the callback. + * @return < 0 on failure, >= 0 on success. + */ +typedef int (MDB_msg_func)(const char *msg, void *ctx); + + /** @brief Dump the entries in the reader lock table. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[in] func A #MDB_msg_func function + * @param[in] ctx Anything the message function needs + * @return < 0 on failure, >= 0 on success. + */ +int mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx); + + /** @brief Check for stale entries in the reader lock table. + * + * @param[in] env An environment handle returned by #mdb_env_create() + * @param[out] dead Number of stale slots that were cleared + * @return 0 on success, non-zero on failure. + */ +int mdb_reader_check(MDB_env *env, int *dead); +/** @} */ + +#ifdef __cplusplus +} +#endif +/** @page tools LMDB Command Line Tools + The following describes the command line tools that are available for LMDB. + \li \ref mdb_copy_1 + \li \ref mdb_dump_1 + \li \ref mdb_load_1 + \li \ref mdb_stat_1 +*/ + +#endif /* _LMDB_H_ */ diff --git a/external/db_drivers/liblmdb64/mdb.c b/external/db_drivers/liblmdb64/mdb.c new file mode 100644 index 000000000..c788b8534 --- /dev/null +++ b/external/db_drivers/liblmdb64/mdb.c @@ -0,0 +1,9503 @@ +/** @file mdb.c + * @brief Lightning memory-mapped database library + * + * A Btree-based database management library modeled loosely on the + * BerkeleyDB API, but much simplified. + */ +/* + * Copyright 2011-2015 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + * + * This code is derived from btree.c written by Martin Hedenfalk. + * + * Copyright (c) 2009, 2010 Martin Hedenfalk <martin@bzero.se> + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE 1 +#endif +#ifdef _WIN32 +#include <malloc.h> +#include <windows.h> +/** getpid() returns int; MinGW defines pid_t but MinGW64 typedefs it + * as int64 which is wrong. MSVC doesn't define it at all, so just + * don't use it. + */ +#define MDB_PID_T int +#define MDB_THR_T DWORD +#include <sys/types.h> +#include <sys/stat.h> +#ifdef __GNUC__ +# include <sys/param.h> +#else +# define LITTLE_ENDIAN 1234 +# define BIG_ENDIAN 4321 +# define BYTE_ORDER LITTLE_ENDIAN +# ifndef SSIZE_MAX +# define SSIZE_MAX INT_MAX +# endif +#endif +#else +#include <sys/types.h> +#include <sys/stat.h> +#define MDB_PID_T pid_t +#define MDB_THR_T pthread_t +#include <sys/param.h> +#include <sys/uio.h> +#include <sys/mman.h> +#ifdef HAVE_SYS_FILE_H +#include <sys/file.h> +#endif +#include <fcntl.h> +#endif + +#if defined(__mips) && defined(__linux) +/* MIPS has cache coherency issues, requires explicit cache control */ +#include <asm/cachectl.h> +extern int cacheflush(char *addr, int nbytes, int cache); +#define CACHEFLUSH(addr, bytes, cache) cacheflush(addr, bytes, cache) +#else +#define CACHEFLUSH(addr, bytes, cache) +#endif + +#if defined(__linux) && !defined(MDB_FDATASYNC_WORKS) +/** fdatasync is broken on ext3/ext4fs on older kernels, see + * description in #mdb_env_open2 comments. You can safely + * define MDB_FDATASYNC_WORKS if this code will only be run + * on kernels 3.6 and newer. + */ +#define BROKEN_FDATASYNC +#endif + +#include <errno.h> +#include <limits.h> +#include <stddef.h> +#include <inttypes.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <unistd.h> + +#if defined(__sun) || defined(ANDROID) +/* Most platforms have posix_memalign, older may only have memalign */ +#define HAVE_MEMALIGN 1 +#include <malloc.h> +#endif + +#if !(defined(BYTE_ORDER) || defined(__BYTE_ORDER)) +#include <netinet/in.h> +#include <resolv.h> /* defines BYTE_ORDER on HPUX and Solaris */ +#endif + +#if defined(__APPLE__) || defined (BSD) +# define MDB_USE_POSIX_SEM 1 +# define MDB_FDATASYNC fsync +#elif defined(ANDROID) +# define MDB_FDATASYNC fsync +#endif + +#ifndef _WIN32 +#include <pthread.h> +#ifdef MDB_USE_POSIX_SEM +# define MDB_USE_HASH 1 +#include <semaphore.h> +#endif +#endif + +#ifdef USE_VALGRIND +#include <valgrind/memcheck.h> +#define VGMEMP_CREATE(h,r,z) VALGRIND_CREATE_MEMPOOL(h,r,z) +#define VGMEMP_ALLOC(h,a,s) VALGRIND_MEMPOOL_ALLOC(h,a,s) +#define VGMEMP_FREE(h,a) VALGRIND_MEMPOOL_FREE(h,a) +#define VGMEMP_DESTROY(h) VALGRIND_DESTROY_MEMPOOL(h) +#define VGMEMP_DEFINED(a,s) VALGRIND_MAKE_MEM_DEFINED(a,s) +#else +#define VGMEMP_CREATE(h,r,z) +#define VGMEMP_ALLOC(h,a,s) +#define VGMEMP_FREE(h,a) +#define VGMEMP_DESTROY(h) +#define VGMEMP_DEFINED(a,s) +#endif + +#ifndef BYTE_ORDER +# if (defined(_LITTLE_ENDIAN) || defined(_BIG_ENDIAN)) && !(defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN)) +/* Solaris just defines one or the other */ +# define LITTLE_ENDIAN 1234 +# define BIG_ENDIAN 4321 +# ifdef _LITTLE_ENDIAN +# define BYTE_ORDER LITTLE_ENDIAN +# else +# define BYTE_ORDER BIG_ENDIAN +# endif +# else +# define BYTE_ORDER __BYTE_ORDER +# endif +#endif + +#ifndef LITTLE_ENDIAN +#define LITTLE_ENDIAN __LITTLE_ENDIAN +#endif +#ifndef BIG_ENDIAN +#define BIG_ENDIAN __BIG_ENDIAN +#endif + +#if defined(__i386) || defined(__x86_64) || defined(_M_IX86) +#define MISALIGNED_OK 1 +#endif + +#include "lmdb.h" +#include "midl.h" + +#if (BYTE_ORDER == LITTLE_ENDIAN) == (BYTE_ORDER == BIG_ENDIAN) +# error "Unknown or unsupported endianness (BYTE_ORDER)" +#elif (-6 & 5) || CHAR_BIT != 8 || UINT_MAX < 0xffffffff || ULONG_MAX % 0xFFFF +# error "Two's complement, reasonably sized integer types, please" +#endif + +#ifdef __GNUC__ +/** Put infrequently used env functions in separate section */ +# ifdef __APPLE__ +# define ESECT __attribute__ ((section("__TEXT,text_env"))) +# else +# define ESECT __attribute__ ((section("text_env"))) +# endif +#else +#define ESECT +#endif + +/** @defgroup internal LMDB Internals + * @{ + */ +/** @defgroup compat Compatibility Macros + * A bunch of macros to minimize the amount of platform-specific ifdefs + * needed throughout the rest of the code. When the features this library + * needs are similar enough to POSIX to be hidden in a one-or-two line + * replacement, this macro approach is used. + * @{ + */ + + /** Features under development */ +#ifndef MDB_DEVEL +#define MDB_DEVEL 0 +#endif + + /** Wrapper around __func__, which is a C99 feature */ +#if __STDC_VERSION__ >= 199901L +# define mdb_func_ __func__ +#elif __GNUC__ >= 2 || _MSC_VER >= 1300 +# define mdb_func_ __FUNCTION__ +#else +/* If a debug message says <mdb_unknown>(), update the #if statements above */ +# define mdb_func_ "<mdb_unknown>" +#endif + +#ifdef _WIN32 +#define MDB_USE_HASH 1 +#define MDB_PIDLOCK 0 +#define THREAD_RET DWORD +#define pthread_t HANDLE +#define pthread_mutex_t HANDLE +#define pthread_cond_t HANDLE +#define pthread_key_t DWORD +#define pthread_self() GetCurrentThreadId() +#define pthread_key_create(x,y) \ + ((*(x) = TlsAlloc()) == TLS_OUT_OF_INDEXES ? ErrCode() : 0) +#define pthread_key_delete(x) TlsFree(x) +#define pthread_getspecific(x) TlsGetValue(x) +#define pthread_setspecific(x,y) (TlsSetValue(x,y) ? 0 : ErrCode()) +#define pthread_mutex_unlock(x) ReleaseMutex(*x) +#define pthread_mutex_lock(x) WaitForSingleObject(*x, INFINITE) +#define pthread_cond_signal(x) SetEvent(*x) +#define pthread_cond_wait(cond,mutex) do{SignalObjectAndWait(*mutex, *cond, INFINITE, FALSE); WaitForSingleObject(*mutex, INFINITE);}while(0) +#define THREAD_CREATE(thr,start,arg) thr=CreateThread(NULL,0,start,arg,0,NULL) +#define THREAD_FINISH(thr) WaitForSingleObject(thr, INFINITE) +#define LOCK_MUTEX_R(env) pthread_mutex_lock(&(env)->me_rmutex) +#define UNLOCK_MUTEX_R(env) pthread_mutex_unlock(&(env)->me_rmutex) +#define LOCK_MUTEX_W(env) pthread_mutex_lock(&(env)->me_wmutex) +#define UNLOCK_MUTEX_W(env) pthread_mutex_unlock(&(env)->me_wmutex) +#define getpid() GetCurrentProcessId() +#define MDB_FDATASYNC(fd) (!FlushFileBuffers(fd)) +#define MDB_MSYNC(addr,len,flags) (!FlushViewOfFile(addr,len)) +#define ErrCode() GetLastError() +#define GET_PAGESIZE(x) {SYSTEM_INFO si; GetSystemInfo(&si); (x) = si.dwPageSize;} +#define close(fd) (CloseHandle(fd) ? 0 : -1) +#define munmap(ptr,len) UnmapViewOfFile(ptr) +#ifdef PROCESS_QUERY_LIMITED_INFORMATION +#define MDB_PROCESS_QUERY_LIMITED_INFORMATION PROCESS_QUERY_LIMITED_INFORMATION +#else +#define MDB_PROCESS_QUERY_LIMITED_INFORMATION 0x1000 +#endif +#define Z "I" +#else +#define THREAD_RET void * +#define THREAD_CREATE(thr,start,arg) pthread_create(&thr,NULL,start,arg) +#define THREAD_FINISH(thr) pthread_join(thr,NULL) +#define Z "z" /**< printf format modifier for size_t */ + + /** For MDB_LOCK_FORMAT: True if readers take a pid lock in the lockfile */ +#define MDB_PIDLOCK 1 + +#ifdef MDB_USE_POSIX_SEM + +#define LOCK_MUTEX_R(env) mdb_sem_wait((env)->me_rmutex) +#define UNLOCK_MUTEX_R(env) sem_post((env)->me_rmutex) +#define LOCK_MUTEX_W(env) mdb_sem_wait((env)->me_wmutex) +#define UNLOCK_MUTEX_W(env) sem_post((env)->me_wmutex) + +static int +mdb_sem_wait(sem_t *sem) +{ + int rc; + while ((rc = sem_wait(sem)) && (rc = errno) == EINTR) ; + return rc; +} + +#else + /** Lock the reader mutex. + */ +#define LOCK_MUTEX_R(env) pthread_mutex_lock(&(env)->me_txns->mti_mutex) + /** Unlock the reader mutex. + */ +#define UNLOCK_MUTEX_R(env) pthread_mutex_unlock(&(env)->me_txns->mti_mutex) + + /** Lock the writer mutex. + * Only a single write transaction is allowed at a time. Other writers + * will block waiting for this mutex. + */ +#define LOCK_MUTEX_W(env) pthread_mutex_lock(&(env)->me_txns->mti_wmutex) + /** Unlock the writer mutex. + */ +#define UNLOCK_MUTEX_W(env) pthread_mutex_unlock(&(env)->me_txns->mti_wmutex) +#endif /* MDB_USE_POSIX_SEM */ + + /** Get the error code for the last failed system function. + */ +#define ErrCode() errno + + /** An abstraction for a file handle. + * On POSIX systems file handles are small integers. On Windows + * they're opaque pointers. + */ +#define HANDLE int + + /** A value for an invalid file handle. + * Mainly used to initialize file variables and signify that they are + * unused. + */ +#define INVALID_HANDLE_VALUE (-1) + + /** Get the size of a memory page for the system. + * This is the basic size that the platform's memory manager uses, and is + * fundamental to the use of memory-mapped files. + */ +#define GET_PAGESIZE(x) ((x) = sysconf(_SC_PAGE_SIZE)) +#endif + +#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) +#define MNAME_LEN 32 +#else +#define MNAME_LEN (sizeof(pthread_mutex_t)) +#endif + +/** @} */ + +#ifndef _WIN32 +/** A flag for opening a file and requesting synchronous data writes. + * This is only used when writing a meta page. It's not strictly needed; + * we could just do a normal write and then immediately perform a flush. + * But if this flag is available it saves us an extra system call. + * + * @note If O_DSYNC is undefined but exists in /usr/include, + * preferably set some compiler flag to get the definition. + * Otherwise compile with the less efficient -DMDB_DSYNC=O_SYNC. + */ +#ifndef MDB_DSYNC +# define MDB_DSYNC O_DSYNC +#endif +#endif + +/** Function for flushing the data of a file. Define this to fsync + * if fdatasync() is not supported. + */ +#ifndef MDB_FDATASYNC +# define MDB_FDATASYNC fdatasync +#endif + +#ifndef MDB_MSYNC +# define MDB_MSYNC(addr,len,flags) msync(addr,len,flags) +#endif + +#ifndef MS_SYNC +#define MS_SYNC 1 +#endif + +#ifndef MS_ASYNC +#define MS_ASYNC 0 +#endif + + /** A page number in the database. + * Note that 64 bit page numbers are overkill, since pages themselves + * already represent 12-13 bits of addressable memory, and the OS will + * always limit applications to a maximum of 63 bits of address space. + * + * @note In the #MDB_node structure, we only store 48 bits of this value, + * which thus limits us to only 60 bits of addressable data. + */ +typedef MDB_ID pgno_t; + + /** A transaction ID. + * See struct MDB_txn.mt_txnid for details. + */ +typedef MDB_ID txnid_t; + +/** @defgroup debug Debug Macros + * @{ + */ +#ifndef MDB_DEBUG + /** Enable debug output. Needs variable argument macros (a C99 feature). + * Set this to 1 for copious tracing. Set to 2 to add dumps of all IDLs + * read from and written to the database (used for free space management). + */ +#define MDB_DEBUG 0 +#endif + +#if MDB_DEBUG +static int mdb_debug; +static txnid_t mdb_debug_start; + + /** Print a debug message with printf formatting. + * Requires double parenthesis around 2 or more args. + */ +# define DPRINTF(args) ((void) ((mdb_debug) && DPRINTF0 args)) +# define DPRINTF0(fmt, ...) \ + fprintf(stderr, "%s:%d " fmt "\n", mdb_func_, __LINE__, __VA_ARGS__) +#else +# define DPRINTF(args) ((void) 0) +#endif + /** Print a debug string. + * The string is printed literally, with no format processing. + */ +#define DPUTS(arg) DPRINTF(("%s", arg)) + /** Debuging output value of a cursor DBI: Negative in a sub-cursor. */ +#define DDBI(mc) \ + (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi) +/** @} */ + + /** @brief The maximum size of a database page. + * + * It is 32k or 64k, since value-PAGEBASE must fit in + * #MDB_page.%mp_upper. + * + * LMDB will use database pages < OS pages if needed. + * That causes more I/O in write transactions: The OS must + * know (read) the whole page before writing a partial page. + * + * Note that we don't currently support Huge pages. On Linux, + * regular data files cannot use Huge pages, and in general + * Huge pages aren't actually pageable. We rely on the OS + * demand-pager to read our data and page it out when memory + * pressure from other processes is high. So until OSs have + * actual paging support for Huge pages, they're not viable. + */ +#define MAX_PAGESIZE (PAGEBASE ? 0x10000 : 0x8000) + + /** The minimum number of keys required in a database page. + * Setting this to a larger value will place a smaller bound on the + * maximum size of a data item. Data items larger than this size will + * be pushed into overflow pages instead of being stored directly in + * the B-tree node. This value used to default to 4. With a page size + * of 4096 bytes that meant that any item larger than 1024 bytes would + * go into an overflow page. That also meant that on average 2-3KB of + * each overflow page was wasted space. The value cannot be lower than + * 2 because then there would no longer be a tree structure. With this + * value, items larger than 2KB will go into overflow pages, and on + * average only 1KB will be wasted. + */ +#define MDB_MINKEYS 2 + + /** A stamp that identifies a file as an LMDB file. + * There's nothing special about this value other than that it is easily + * recognizable, and it will reflect any byte order mismatches. + */ +#define MDB_MAGIC 0xBEEFC0DE + + /** The version number for a database's datafile format. */ +#define MDB_DATA_VERSION ((MDB_DEVEL) ? 999 : 1) + /** The version number for a database's lockfile format. */ +#define MDB_LOCK_VERSION 1 + + /** @brief The max size of a key we can write, or 0 for dynamic max. + * + * Define this as 0 to compute the max from the page size. 511 + * is default for backwards compat: liblmdb <= 0.9.10 can break + * when modifying a DB with keys/dupsort data bigger than its max. + * #MDB_DEVEL sets the default to 0. + * + * Data items in an #MDB_DUPSORT database are also limited to + * this size, since they're actually keys of a sub-DB. Keys and + * #MDB_DUPSORT data items must fit on a node in a regular page. + */ +#ifndef MDB_MAXKEYSIZE +#define MDB_MAXKEYSIZE ((MDB_DEVEL) ? 0 : 511) +#endif + + /** The maximum size of a key we can write to the environment. */ +#if MDB_MAXKEYSIZE +#define ENV_MAXKEY(env) (MDB_MAXKEYSIZE) +#else +#define ENV_MAXKEY(env) ((env)->me_maxkey) +#endif + + /** @brief The maximum size of a data item. + * + * We only store a 32 bit value for node sizes. + */ +#define MAXDATASIZE 0xffffffffUL + +#if MDB_DEBUG + /** Key size which fits in a #DKBUF. + * @ingroup debug + */ +#define DKBUF_MAXKEYSIZE ((MDB_MAXKEYSIZE) > 0 ? (MDB_MAXKEYSIZE) : 511) + /** A key buffer. + * @ingroup debug + * This is used for printing a hex dump of a key's contents. + */ +#define DKBUF char kbuf[DKBUF_MAXKEYSIZE*2+1] + /** Display a key in hex. + * @ingroup debug + * Invoke a function to display a key in hex. + */ +#define DKEY(x) mdb_dkey(x, kbuf) +#else +#define DKBUF +#define DKEY(x) 0 +#endif + + /** An invalid page number. + * Mainly used to denote an empty tree. + */ +#define P_INVALID (~(pgno_t)0) + + /** Test if the flags \b f are set in a flag word \b w. */ +#define F_ISSET(w, f) (((w) & (f)) == (f)) + + /** Round \b n up to an even number. */ +#define EVEN(n) (((n) + 1U) & -2) /* sign-extending -2 to match n+1U */ + + /** Used for offsets within a single page. + * Since memory pages are typically 4 or 8KB in size, 12-13 bits, + * this is plenty. + */ +typedef uint16_t indx_t; + + /** Default size of memory map. + * This is certainly too small for any actual applications. Apps should always set + * the size explicitly using #mdb_env_set_mapsize(). + */ +#define DEFAULT_MAPSIZE 1048576 + +/** @defgroup readers Reader Lock Table + * Readers don't acquire any locks for their data access. Instead, they + * simply record their transaction ID in the reader table. The reader + * mutex is needed just to find an empty slot in the reader table. The + * slot's address is saved in thread-specific data so that subsequent read + * transactions started by the same thread need no further locking to proceed. + * + * If #MDB_NOTLS is set, the slot address is not saved in thread-specific data. + * + * No reader table is used if the database is on a read-only filesystem, or + * if #MDB_NOLOCK is set. + * + * Since the database uses multi-version concurrency control, readers don't + * actually need any locking. This table is used to keep track of which + * readers are using data from which old transactions, so that we'll know + * when a particular old transaction is no longer in use. Old transactions + * that have discarded any data pages can then have those pages reclaimed + * for use by a later write transaction. + * + * The lock table is constructed such that reader slots are aligned with the + * processor's cache line size. Any slot is only ever used by one thread. + * This alignment guarantees that there will be no contention or cache + * thrashing as threads update their own slot info, and also eliminates + * any need for locking when accessing a slot. + * + * A writer thread will scan every slot in the table to determine the oldest + * outstanding reader transaction. Any freed pages older than this will be + * reclaimed by the writer. The writer doesn't use any locks when scanning + * this table. This means that there's no guarantee that the writer will + * see the most up-to-date reader info, but that's not required for correct + * operation - all we need is to know the upper bound on the oldest reader, + * we don't care at all about the newest reader. So the only consequence of + * reading stale information here is that old pages might hang around a + * while longer before being reclaimed. That's actually good anyway, because + * the longer we delay reclaiming old pages, the more likely it is that a + * string of contiguous pages can be found after coalescing old pages from + * many old transactions together. + * @{ + */ + /** Number of slots in the reader table. + * This value was chosen somewhat arbitrarily. 126 readers plus a + * couple mutexes fit exactly into 8KB on my development machine. + * Applications should set the table size using #mdb_env_set_maxreaders(). + */ +#define DEFAULT_READERS 126 + + /** The size of a CPU cache line in bytes. We want our lock structures + * aligned to this size to avoid false cache line sharing in the + * lock table. + * This value works for most CPUs. For Itanium this should be 128. + */ +#ifndef CACHELINE +#define CACHELINE 64 +#endif + + /** The information we store in a single slot of the reader table. + * In addition to a transaction ID, we also record the process and + * thread ID that owns a slot, so that we can detect stale information, + * e.g. threads or processes that went away without cleaning up. + * @note We currently don't check for stale records. We simply re-init + * the table when we know that we're the only process opening the + * lock file. + */ +typedef struct MDB_rxbody { + /** Current Transaction ID when this transaction began, or (txnid_t)-1. + * Multiple readers that start at the same time will probably have the + * same ID here. Again, it's not important to exclude them from + * anything; all we need to know is which version of the DB they + * started from so we can avoid overwriting any data used in that + * particular version. + */ + volatile txnid_t mrb_txnid; + /** The process ID of the process owning this reader txn. */ + volatile MDB_PID_T mrb_pid; + /** The thread ID of the thread owning this txn. */ + volatile MDB_THR_T mrb_tid; +} MDB_rxbody; + + /** The actual reader record, with cacheline padding. */ +typedef struct MDB_reader { + union { + MDB_rxbody mrx; + /** shorthand for mrb_txnid */ +#define mr_txnid mru.mrx.mrb_txnid +#define mr_pid mru.mrx.mrb_pid +#define mr_tid mru.mrx.mrb_tid + /** cache line alignment */ + char pad[(sizeof(MDB_rxbody)+CACHELINE-1) & ~(CACHELINE-1)]; + } mru; +} MDB_reader; + + /** The header for the reader table. + * The table resides in a memory-mapped file. (This is a different file + * than is used for the main database.) + * + * For POSIX the actual mutexes reside in the shared memory of this + * mapped file. On Windows, mutexes are named objects allocated by the + * kernel; we store the mutex names in this mapped file so that other + * processes can grab them. This same approach is also used on + * MacOSX/Darwin (using named semaphores) since MacOSX doesn't support + * process-shared POSIX mutexes. For these cases where a named object + * is used, the object name is derived from a 64 bit FNV hash of the + * environment pathname. As such, naming collisions are extremely + * unlikely. If a collision occurs, the results are unpredictable. + */ +typedef struct MDB_txbody { + /** Stamp identifying this as an LMDB file. It must be set + * to #MDB_MAGIC. */ + uint32_t mtb_magic; + /** Format of this lock file. Must be set to #MDB_LOCK_FORMAT. */ + uint32_t mtb_format; +#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) + char mtb_rmname[MNAME_LEN]; +#else + /** Mutex protecting access to this table. + * This is the reader lock that #LOCK_MUTEX_R acquires. + */ + pthread_mutex_t mtb_mutex; +#endif + /** The ID of the last transaction committed to the database. + * This is recorded here only for convenience; the value can always + * be determined by reading the main database meta pages. + */ + volatile txnid_t mtb_txnid; + /** The number of slots that have been used in the reader table. + * This always records the maximum count, it is not decremented + * when readers release their slots. + */ + volatile unsigned mtb_numreaders; +} MDB_txbody; + + /** The actual reader table definition. */ +typedef struct MDB_txninfo { + union { + MDB_txbody mtb; +#define mti_magic mt1.mtb.mtb_magic +#define mti_format mt1.mtb.mtb_format +#define mti_mutex mt1.mtb.mtb_mutex +#define mti_rmname mt1.mtb.mtb_rmname +#define mti_txnid mt1.mtb.mtb_txnid +#define mti_numreaders mt1.mtb.mtb_numreaders + char pad[(sizeof(MDB_txbody)+CACHELINE-1) & ~(CACHELINE-1)]; + } mt1; + union { +#if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) + char mt2_wmname[MNAME_LEN]; +#define mti_wmname mt2.mt2_wmname +#else + pthread_mutex_t mt2_wmutex; +#define mti_wmutex mt2.mt2_wmutex +#endif + char pad[(MNAME_LEN+CACHELINE-1) & ~(CACHELINE-1)]; + } mt2; + MDB_reader mti_readers[1]; +} MDB_txninfo; + + /** Lockfile format signature: version, features and field layout */ +#define MDB_LOCK_FORMAT \ + ((uint32_t) \ + ((MDB_LOCK_VERSION) \ + /* Flags which describe functionality */ \ + + (((MDB_PIDLOCK) != 0) << 16))) +/** @} */ + +/** Common header for all page types. + * Overflow records occupy a number of contiguous pages with no + * headers on any page after the first. + */ +typedef struct MDB_page { +#define mp_pgno mp_p.p_pgno +#define mp_next mp_p.p_next + union { + pgno_t p_pgno; /**< page number */ + struct MDB_page *p_next; /**< for in-memory list of freed pages */ + } mp_p; + uint16_t mp_pad; +/** @defgroup mdb_page Page Flags + * @ingroup internal + * Flags for the page headers. + * @{ + */ +#define P_BRANCH 0x01 /**< branch page */ +#define P_LEAF 0x02 /**< leaf page */ +#define P_OVERFLOW 0x04 /**< overflow page */ +#define P_META 0x08 /**< meta page */ +#define P_DIRTY 0x10 /**< dirty page, also set for #P_SUBP pages */ +#define P_LEAF2 0x20 /**< for #MDB_DUPFIXED records */ +#define P_SUBP 0x40 /**< for #MDB_DUPSORT sub-pages */ +#define P_LOOSE 0x4000 /**< page was dirtied then freed, can be reused */ +#define P_KEEP 0x8000 /**< leave this page alone during spill */ +/** @} */ + uint16_t mp_flags; /**< @ref mdb_page */ +#define mp_lower mp_pb.pb.pb_lower +#define mp_upper mp_pb.pb.pb_upper +#define mp_pages mp_pb.pb_pages + union { + struct { + indx_t pb_lower; /**< lower bound of free space */ + indx_t pb_upper; /**< upper bound of free space */ + } pb; + uint32_t pb_pages; /**< number of overflow pages */ + } mp_pb; + indx_t mp_ptrs[1]; /**< dynamic size */ +} MDB_page; + + /** Size of the page header, excluding dynamic data at the end */ +#define PAGEHDRSZ ((unsigned) offsetof(MDB_page, mp_ptrs)) + + /** Address of first usable data byte in a page, after the header */ +#define METADATA(p) ((void *)((char *)(p) + PAGEHDRSZ)) + + /** ITS#7713, change PAGEBASE to handle 65536 byte pages */ +#define PAGEBASE ((MDB_DEVEL) ? PAGEHDRSZ : 0) + + /** Number of nodes on a page */ +#define NUMKEYS(p) (((p)->mp_lower - (PAGEHDRSZ-PAGEBASE)) >> 1) + + /** The amount of space remaining in the page */ +#define SIZELEFT(p) (indx_t)((p)->mp_upper - (p)->mp_lower) + + /** The percentage of space used in the page, in tenths of a percent. */ +#define PAGEFILL(env, p) (1000L * ((env)->me_psize - PAGEHDRSZ - SIZELEFT(p)) / \ + ((env)->me_psize - PAGEHDRSZ)) + /** The minimum page fill factor, in tenths of a percent. + * Pages emptier than this are candidates for merging. + */ +#define FILL_THRESHOLD 250 + + /** Test if a page is a leaf page */ +#define IS_LEAF(p) F_ISSET((p)->mp_flags, P_LEAF) + /** Test if a page is a LEAF2 page */ +#define IS_LEAF2(p) F_ISSET((p)->mp_flags, P_LEAF2) + /** Test if a page is a branch page */ +#define IS_BRANCH(p) F_ISSET((p)->mp_flags, P_BRANCH) + /** Test if a page is an overflow page */ +#define IS_OVERFLOW(p) F_ISSET((p)->mp_flags, P_OVERFLOW) + /** Test if a page is a sub page */ +#define IS_SUBP(p) F_ISSET((p)->mp_flags, P_SUBP) + + /** The number of overflow pages needed to store the given size. */ +#define OVPAGES(size, psize) ((PAGEHDRSZ-1 + (size)) / (psize) + 1) + + /** Link in #MDB_txn.%mt_loose_pgs list */ +#define NEXT_LOOSE_PAGE(p) (*(MDB_page **)((p) + 2)) + + /** Header for a single key/data pair within a page. + * Used in pages of type #P_BRANCH and #P_LEAF without #P_LEAF2. + * We guarantee 2-byte alignment for 'MDB_node's. + */ +typedef struct MDB_node { + /** lo and hi are used for data size on leaf nodes and for + * child pgno on branch nodes. On 64 bit platforms, flags + * is also used for pgno. (Branch nodes have no flags). + * They are in host byte order in case that lets some + * accesses be optimized into a 32-bit word access. + */ +#if BYTE_ORDER == LITTLE_ENDIAN + unsigned short mn_lo, mn_hi; /**< part of data size or pgno */ +#else + unsigned short mn_hi, mn_lo; +#endif +/** @defgroup mdb_node Node Flags + * @ingroup internal + * Flags for node headers. + * @{ + */ +#define F_BIGDATA 0x01 /**< data put on overflow page */ +#define F_SUBDATA 0x02 /**< data is a sub-database */ +#define F_DUPDATA 0x04 /**< data has duplicates */ + +/** valid flags for #mdb_node_add() */ +#define NODE_ADD_FLAGS (F_DUPDATA|F_SUBDATA|MDB_RESERVE|MDB_APPEND) + +/** @} */ + unsigned short mn_flags; /**< @ref mdb_node */ + unsigned short mn_ksize; /**< key size */ + char mn_data[1]; /**< key and data are appended here */ +} MDB_node; + + /** Size of the node header, excluding dynamic data at the end */ +#define NODESIZE offsetof(MDB_node, mn_data) + + /** Bit position of top word in page number, for shifting mn_flags */ +#define PGNO_TOPWORD ((pgno_t)-1 > 0xffffffffu ? 32 : 0) + + /** Size of a node in a branch page with a given key. + * This is just the node header plus the key, there is no data. + */ +#define INDXSIZE(k) (NODESIZE + ((k) == NULL ? 0 : (k)->mv_size)) + + /** Size of a node in a leaf page with a given key and data. + * This is node header plus key plus data size. + */ +#define LEAFSIZE(k, d) (NODESIZE + (k)->mv_size + (d)->mv_size) + + /** Address of node \b i in page \b p */ +#define NODEPTR(p, i) ((MDB_node *)((char *)(p) + (p)->mp_ptrs[i] + PAGEBASE)) + + /** Address of the key for the node */ +#define NODEKEY(node) (void *)((node)->mn_data) + + /** Address of the data for a node */ +#define NODEDATA(node) (void *)((char *)(node)->mn_data + (node)->mn_ksize) + + /** Get the page number pointed to by a branch node */ +#define NODEPGNO(node) \ + ((node)->mn_lo | ((pgno_t) (node)->mn_hi << 16) | \ + (PGNO_TOPWORD ? ((pgno_t) (node)->mn_flags << PGNO_TOPWORD) : 0)) + /** Set the page number in a branch node */ +#define SETPGNO(node,pgno) do { \ + (node)->mn_lo = (pgno) & 0xffff; (node)->mn_hi = (pgno) >> 16; \ + if (PGNO_TOPWORD) (node)->mn_flags = (pgno) >> PGNO_TOPWORD; } while(0) + + /** Get the size of the data in a leaf node */ +#define NODEDSZ(node) ((node)->mn_lo | ((unsigned)(node)->mn_hi << 16)) + /** Set the size of the data for a leaf node */ +#define SETDSZ(node,size) do { \ + (node)->mn_lo = (size) & 0xffff; (node)->mn_hi = (size) >> 16;} while(0) + /** The size of a key in a node */ +#define NODEKSZ(node) ((node)->mn_ksize) + + /** Copy a page number from src to dst */ +#ifdef MISALIGNED_OK +#define COPY_PGNO(dst,src) dst = src +#else +#if SIZE_MAX > 4294967295UL +#define COPY_PGNO(dst,src) do { \ + unsigned short *s, *d; \ + s = (unsigned short *)&(src); \ + d = (unsigned short *)&(dst); \ + *d++ = *s++; \ + *d++ = *s++; \ + *d++ = *s++; \ + *d = *s; \ +} while (0) +#else +#define COPY_PGNO(dst,src) do { \ + unsigned short *s, *d; \ + s = (unsigned short *)&(src); \ + d = (unsigned short *)&(dst); \ + *d++ = *s++; \ + *d = *s; \ +} while (0) +#endif +#endif + /** The address of a key in a LEAF2 page. + * LEAF2 pages are used for #MDB_DUPFIXED sorted-duplicate sub-DBs. + * There are no node headers, keys are stored contiguously. + */ +#define LEAF2KEY(p, i, ks) ((char *)(p) + PAGEHDRSZ + ((i)*(ks))) + + /** Set the \b node's key into \b keyptr, if requested. */ +#define MDB_GET_KEY(node, keyptr) { if ((keyptr) != NULL) { \ + (keyptr)->mv_size = NODEKSZ(node); (keyptr)->mv_data = NODEKEY(node); } } + + /** Set the \b node's key into \b key. */ +#define MDB_GET_KEY2(node, key) { key.mv_size = NODEKSZ(node); key.mv_data = NODEKEY(node); } + + /** Information about a single database in the environment. */ +typedef struct MDB_db { + uint32_t md_pad; /**< also ksize for LEAF2 pages */ + uint16_t md_flags; /**< @ref mdb_dbi_open */ + uint16_t md_depth; /**< depth of this tree */ + pgno_t md_branch_pages; /**< number of internal pages */ + pgno_t md_leaf_pages; /**< number of leaf pages */ + pgno_t md_overflow_pages; /**< number of overflow pages */ + size_t md_entries; /**< number of data items */ + pgno_t md_root; /**< the root page of this tree */ +} MDB_db; + + /** mdb_dbi_open flags */ +#define MDB_VALID 0x8000 /**< DB handle is valid, for me_dbflags */ +#define PERSISTENT_FLAGS (0xffff & ~(MDB_VALID)) +#define VALID_FLAGS (MDB_REVERSEKEY|MDB_DUPSORT|MDB_INTEGERKEY|MDB_DUPFIXED|\ + MDB_INTEGERDUP|MDB_REVERSEDUP|MDB_CREATE) + + /** Handle for the DB used to track free pages. */ +#define FREE_DBI 0 + /** Handle for the default DB. */ +#define MAIN_DBI 1 + + /** Meta page content. + * A meta page is the start point for accessing a database snapshot. + * Pages 0-1 are meta pages. Transaction N writes meta page #(N % 2). + */ +typedef struct MDB_meta { + /** Stamp identifying this as an LMDB file. It must be set + * to #MDB_MAGIC. */ + uint32_t mm_magic; + /** Version number of this file. Must be set to #MDB_DATA_VERSION. */ + uint32_t mm_version; + void *mm_address; /**< address for fixed mapping */ + size_t mm_mapsize; /**< size of mmap region */ + MDB_db mm_dbs[2]; /**< first is free space, 2nd is main db */ + /** The size of pages used in this DB */ +#define mm_psize mm_dbs[0].md_pad + /** Any persistent environment flags. @ref mdb_env */ +#define mm_flags mm_dbs[0].md_flags + pgno_t mm_last_pg; /**< last used page in file */ + volatile txnid_t mm_txnid; /**< txnid that committed this page */ +} MDB_meta; + + /** Buffer for a stack-allocated meta page. + * The members define size and alignment, and silence type + * aliasing warnings. They are not used directly; that could + * mean incorrectly using several union members in parallel. + */ +typedef union MDB_metabuf { + MDB_page mb_page; + struct { + char mm_pad[PAGEHDRSZ]; + MDB_meta mm_meta; + } mb_metabuf; +} MDB_metabuf; + + /** Auxiliary DB info. + * The information here is mostly static/read-only. There is + * only a single copy of this record in the environment. + */ +typedef struct MDB_dbx { + MDB_val md_name; /**< name of the database */ + MDB_cmp_func *md_cmp; /**< function for comparing keys */ + MDB_cmp_func *md_dcmp; /**< function for comparing data items */ + MDB_rel_func *md_rel; /**< user relocate function */ + void *md_relctx; /**< user-provided context for md_rel */ +} MDB_dbx; + + /** A database transaction. + * Every operation requires a transaction handle. + */ +struct MDB_txn { + MDB_txn *mt_parent; /**< parent of a nested txn */ + MDB_txn *mt_child; /**< nested txn under this txn */ + pgno_t mt_next_pgno; /**< next unallocated page */ + /** The ID of this transaction. IDs are integers incrementing from 1. + * Only committed write transactions increment the ID. If a transaction + * aborts, the ID may be re-used by the next writer. + */ + txnid_t mt_txnid; + MDB_env *mt_env; /**< the DB environment */ + /** The list of pages that became unused during this transaction. + */ + MDB_IDL mt_free_pgs; + /** The list of loose pages that became unused and may be reused + * in this transaction, linked through #NEXT_LOOSE_PAGE(page). + */ + MDB_page *mt_loose_pgs; + /* #Number of loose pages (#mt_loose_pgs) */ + int mt_loose_count; + /** The sorted list of dirty pages we temporarily wrote to disk + * because the dirty list was full. page numbers in here are + * shifted left by 1, deleted slots have the LSB set. + */ + MDB_IDL mt_spill_pgs; + union { + /** For write txns: Modified pages. Sorted when not MDB_WRITEMAP. */ + MDB_ID2L dirty_list; + /** For read txns: This thread/txn's reader table slot, or NULL. */ + MDB_reader *reader; + } mt_u; + /** Array of records for each DB known in the environment. */ + MDB_dbx *mt_dbxs; + /** Array of MDB_db records for each known DB */ + MDB_db *mt_dbs; + /** Array of sequence numbers for each DB handle */ + unsigned int *mt_dbiseqs; +/** @defgroup mt_dbflag Transaction DB Flags + * @ingroup internal + * @{ + */ +#define DB_DIRTY 0x01 /**< DB was modified or is DUPSORT data */ +#define DB_STALE 0x02 /**< Named-DB record is older than txnID */ +#define DB_NEW 0x04 /**< Named-DB handle opened in this txn */ +#define DB_VALID 0x08 /**< DB handle is valid, see also #MDB_VALID */ +/** @} */ + /** In write txns, array of cursors for each DB */ + MDB_cursor **mt_cursors; + /** Array of flags for each DB */ + unsigned char *mt_dbflags; + /** Number of DB records in use. This number only ever increments; + * we don't decrement it when individual DB handles are closed. + */ + MDB_dbi mt_numdbs; + +/** @defgroup mdb_txn Transaction Flags + * @ingroup internal + * @{ + */ +#define MDB_TXN_RDONLY 0x01 /**< read-only transaction */ +#define MDB_TXN_ERROR 0x02 /**< txn is unusable after an error */ +#define MDB_TXN_DIRTY 0x04 /**< must write, even if dirty list is empty */ +#define MDB_TXN_SPILLS 0x08 /**< txn or a parent has spilled pages */ +/** @} */ + unsigned int mt_flags; /**< @ref mdb_txn */ + /** #dirty_list room: Array size - \#dirty pages visible to this txn. + * Includes ancestor txns' dirty pages not hidden by other txns' + * dirty/spilled pages. Thus commit(nested txn) has room to merge + * dirty_list into mt_parent after freeing hidden mt_parent pages. + */ + unsigned int mt_dirty_room; +}; + +/** Enough space for 2^32 nodes with minimum of 2 keys per node. I.e., plenty. + * At 4 keys per node, enough for 2^64 nodes, so there's probably no need to + * raise this on a 64 bit machine. + */ +#define CURSOR_STACK 32 + +struct MDB_xcursor; + + /** Cursors are used for all DB operations. + * A cursor holds a path of (page pointer, key index) from the DB + * root to a position in the DB, plus other state. #MDB_DUPSORT + * cursors include an xcursor to the current data item. Write txns + * track their cursors and keep them up to date when data moves. + * Exception: An xcursor's pointer to a #P_SUBP page can be stale. + * (A node with #F_DUPDATA but no #F_SUBDATA contains a subpage). + */ +struct MDB_cursor { + /** Next cursor on this DB in this txn */ + MDB_cursor *mc_next; + /** Backup of the original cursor if this cursor is a shadow */ + MDB_cursor *mc_backup; + /** Context used for databases with #MDB_DUPSORT, otherwise NULL */ + struct MDB_xcursor *mc_xcursor; + /** The transaction that owns this cursor */ + MDB_txn *mc_txn; + /** The database handle this cursor operates on */ + MDB_dbi mc_dbi; + /** The database record for this cursor */ + MDB_db *mc_db; + /** The database auxiliary record for this cursor */ + MDB_dbx *mc_dbx; + /** The @ref mt_dbflag for this database */ + unsigned char *mc_dbflag; + unsigned short mc_snum; /**< number of pushed pages */ + unsigned short mc_top; /**< index of top page, normally mc_snum-1 */ +/** @defgroup mdb_cursor Cursor Flags + * @ingroup internal + * Cursor state flags. + * @{ + */ +#define C_INITIALIZED 0x01 /**< cursor has been initialized and is valid */ +#define C_EOF 0x02 /**< No more data */ +#define C_SUB 0x04 /**< Cursor is a sub-cursor */ +#define C_DEL 0x08 /**< last op was a cursor_del */ +#define C_SPLITTING 0x20 /**< Cursor is in page_split */ +#define C_UNTRACK 0x40 /**< Un-track cursor when closing */ +/** @} */ + unsigned int mc_flags; /**< @ref mdb_cursor */ + MDB_page *mc_pg[CURSOR_STACK]; /**< stack of pushed pages */ + indx_t mc_ki[CURSOR_STACK]; /**< stack of page indices */ +}; + + /** Context for sorted-dup records. + * We could have gone to a fully recursive design, with arbitrarily + * deep nesting of sub-databases. But for now we only handle these + * levels - main DB, optional sub-DB, sorted-duplicate DB. + */ +typedef struct MDB_xcursor { + /** A sub-cursor for traversing the Dup DB */ + MDB_cursor mx_cursor; + /** The database record for this Dup DB */ + MDB_db mx_db; + /** The auxiliary DB record for this Dup DB */ + MDB_dbx mx_dbx; + /** The @ref mt_dbflag for this Dup DB */ + unsigned char mx_dbflag; +} MDB_xcursor; + + /** State of FreeDB old pages, stored in the MDB_env */ +typedef struct MDB_pgstate { + pgno_t *mf_pghead; /**< Reclaimed freeDB pages, or NULL before use */ + txnid_t mf_pglast; /**< ID of last used record, or 0 if !mf_pghead */ +} MDB_pgstate; + + /** The database environment. */ +struct MDB_env { + HANDLE me_fd; /**< The main data file */ + HANDLE me_lfd; /**< The lock file */ + HANDLE me_mfd; /**< just for writing the meta pages */ + /** Failed to update the meta page. Probably an I/O error. */ +#define MDB_FATAL_ERROR 0x80000000U + /** Some fields are initialized. */ +#define MDB_ENV_ACTIVE 0x20000000U + /** me_txkey is set */ +#define MDB_ENV_TXKEY 0x10000000U + /** fdatasync is unreliable */ +#define MDB_FSYNCONLY 0x08000000U + uint32_t me_flags; /**< @ref mdb_env */ + unsigned int me_psize; /**< DB page size, inited from me_os_psize */ + unsigned int me_os_psize; /**< OS page size, from #GET_PAGESIZE */ + unsigned int me_maxreaders; /**< size of the reader table */ + unsigned int me_numreaders; /**< max numreaders set by this env */ + MDB_dbi me_numdbs; /**< number of DBs opened */ + MDB_dbi me_maxdbs; /**< size of the DB table */ + MDB_PID_T me_pid; /**< process ID of this env */ + char *me_path; /**< path to the DB files */ + char *me_map; /**< the memory map of the data file */ + MDB_txninfo *me_txns; /**< the memory map of the lock file or NULL */ + MDB_meta *me_metas[2]; /**< pointers to the two meta pages */ + void *me_pbuf; /**< scratch area for DUPSORT put() */ + MDB_txn *me_txn; /**< current write transaction */ + MDB_txn *me_txn0; /**< prealloc'd write transaction */ + size_t me_mapsize; /**< size of the data memory map */ + off_t me_size; /**< current file size */ + pgno_t me_maxpg; /**< me_mapsize / me_psize */ + MDB_dbx *me_dbxs; /**< array of static DB info */ + uint16_t *me_dbflags; /**< array of flags from MDB_db.md_flags */ + unsigned int *me_dbiseqs; /**< array of dbi sequence numbers */ + pthread_key_t me_txkey; /**< thread-key for readers */ + txnid_t me_pgoldest; /**< ID of oldest reader last time we looked */ + MDB_pgstate me_pgstate; /**< state of old pages from freeDB */ +# define me_pglast me_pgstate.mf_pglast +# define me_pghead me_pgstate.mf_pghead + MDB_page *me_dpages; /**< list of malloc'd blocks for re-use */ + /** IDL of pages that became unused in a write txn */ + MDB_IDL me_free_pgs; + /** ID2L of pages written during a write txn. Length MDB_IDL_UM_SIZE. */ + MDB_ID2L me_dirty_list; + /** Max number of freelist items that can fit in a single overflow page */ + int me_maxfree_1pg; + /** Max size of a node on a page */ + unsigned int me_nodemax; +#if !(MDB_MAXKEYSIZE) + unsigned int me_maxkey; /**< max size of a key */ +#endif + int me_live_reader; /**< have liveness lock in reader table */ +#ifdef _WIN32 + int me_pidquery; /**< Used in OpenProcess */ + HANDLE me_rmutex; /* Windows mutexes don't reside in shared mem */ + HANDLE me_wmutex; +#elif defined(MDB_USE_POSIX_SEM) + sem_t *me_rmutex; /* Shared mutexes are not supported */ + sem_t *me_wmutex; +#endif + void *me_userctx; /**< User-settable context */ + MDB_assert_func *me_assert_func; /**< Callback for assertion failures */ +}; + + /** Nested transaction */ +typedef struct MDB_ntxn { + MDB_txn mnt_txn; /**< the transaction */ + MDB_pgstate mnt_pgstate; /**< parent transaction's saved freestate */ +} MDB_ntxn; + + /** max number of pages to commit in one writev() call */ +#define MDB_COMMIT_PAGES 64 +#if defined(IOV_MAX) && IOV_MAX < MDB_COMMIT_PAGES +#undef MDB_COMMIT_PAGES +#define MDB_COMMIT_PAGES IOV_MAX +#endif + + /** max bytes to write in one call */ +#define MAX_WRITE (0x80000000U >> (sizeof(ssize_t) == 4)) + + /** Check \b txn and \b dbi arguments to a function */ +#define TXN_DBI_EXIST(txn, dbi) \ + ((txn) && (dbi) < (txn)->mt_numdbs && ((txn)->mt_dbflags[dbi] & DB_VALID)) + + /** Check for misused \b dbi handles */ +#define TXN_DBI_CHANGED(txn, dbi) \ + ((txn)->mt_dbiseqs[dbi] != (txn)->mt_env->me_dbiseqs[dbi]) + +static int mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp); +static int mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp); +static int mdb_page_touch(MDB_cursor *mc); + +static int mdb_page_get(MDB_txn *txn, pgno_t pgno, MDB_page **mp, int *lvl); +static int mdb_page_search_root(MDB_cursor *mc, + MDB_val *key, int modify); +#define MDB_PS_MODIFY 1 +#define MDB_PS_ROOTONLY 2 +#define MDB_PS_FIRST 4 +#define MDB_PS_LAST 8 +static int mdb_page_search(MDB_cursor *mc, + MDB_val *key, int flags); +static int mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst); + +#define MDB_SPLIT_REPLACE MDB_APPENDDUP /**< newkey is not new */ +static int mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, + pgno_t newpgno, unsigned int nflags); + +static int mdb_env_read_header(MDB_env *env, MDB_meta *meta); +static int mdb_env_pick_meta(const MDB_env *env); +static int mdb_env_write_meta(MDB_txn *txn); +#if !(defined(_WIN32) || defined(MDB_USE_POSIX_SEM)) /* Drop unused excl arg */ +# define mdb_env_close0(env, excl) mdb_env_close1(env) +#endif +static void mdb_env_close0(MDB_env *env, int excl); + +static MDB_node *mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp); +static int mdb_node_add(MDB_cursor *mc, indx_t indx, + MDB_val *key, MDB_val *data, pgno_t pgno, unsigned int flags); +static void mdb_node_del(MDB_cursor *mc, int ksize); +static void mdb_node_shrink(MDB_page *mp, indx_t indx); +static int mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst); +static int mdb_node_read(MDB_txn *txn, MDB_node *leaf, MDB_val *data); +static size_t mdb_leaf_size(MDB_env *env, MDB_val *key, MDB_val *data); +static size_t mdb_branch_size(MDB_env *env, MDB_val *key); + +static int mdb_rebalance(MDB_cursor *mc); +static int mdb_update_key(MDB_cursor *mc, MDB_val *key); + +static void mdb_cursor_pop(MDB_cursor *mc); +static int mdb_cursor_push(MDB_cursor *mc, MDB_page *mp); + +static int mdb_cursor_del0(MDB_cursor *mc); +static int mdb_del0(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data, unsigned flags); +static int mdb_cursor_sibling(MDB_cursor *mc, int move_right); +static int mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op); +static int mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op); +static int mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op, + int *exactp); +static int mdb_cursor_first(MDB_cursor *mc, MDB_val *key, MDB_val *data); +static int mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data); + +static void mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx); +static void mdb_xcursor_init0(MDB_cursor *mc); +static void mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node); + +static int mdb_drop0(MDB_cursor *mc, int subs); +static void mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi); + +/** @cond */ +static MDB_cmp_func mdb_cmp_memn, mdb_cmp_memnr, mdb_cmp_int, mdb_cmp_cint, mdb_cmp_long; +/** @endcond */ + +#ifdef _WIN32 +static SECURITY_DESCRIPTOR mdb_null_sd; +static SECURITY_ATTRIBUTES mdb_all_sa; +static int mdb_sec_inited; +#endif + +/** Return the library version info. */ +char * +mdb_version(int *major, int *minor, int *patch) +{ + if (major) *major = MDB_VERSION_MAJOR; + if (minor) *minor = MDB_VERSION_MINOR; + if (patch) *patch = MDB_VERSION_PATCH; + return MDB_VERSION_STRING; +} + +/** Table of descriptions for LMDB @ref errors */ +static char *const mdb_errstr[] = { + "MDB_KEYEXIST: Key/data pair already exists", + "MDB_NOTFOUND: No matching key/data pair found", + "MDB_PAGE_NOTFOUND: Requested page not found", + "MDB_CORRUPTED: Located page was wrong type", + "MDB_PANIC: Update of meta page failed", + "MDB_VERSION_MISMATCH: Database environment version mismatch", + "MDB_INVALID: File is not an LMDB file", + "MDB_MAP_FULL: Environment mapsize limit reached", + "MDB_DBS_FULL: Environment maxdbs limit reached", + "MDB_READERS_FULL: Environment maxreaders limit reached", + "MDB_TLS_FULL: Thread-local storage keys full - too many environments open", + "MDB_TXN_FULL: Transaction has too many dirty pages - transaction too big", + "MDB_CURSOR_FULL: Internal error - cursor stack limit reached", + "MDB_PAGE_FULL: Internal error - page has no more space", + "MDB_MAP_RESIZED: Database contents grew beyond environment mapsize", + "MDB_INCOMPATIBLE: Operation and DB incompatible, or DB flags changed", + "MDB_BAD_RSLOT: Invalid reuse of reader locktable slot", + "MDB_BAD_TXN: Transaction cannot recover - it must be aborted", + "MDB_BAD_VALSIZE: Unsupported size of key/DB name/data, or wrong DUPFIXED size", + "MDB_BAD_DBI: The specified DBI handle was closed/changed unexpectedly", +}; + +char * +mdb_strerror(int err) +{ +#ifdef _WIN32 + /** HACK: pad 4KB on stack over the buf. Return system msgs in buf. + * This works as long as no function between the call to mdb_strerror + * and the actual use of the message uses more than 4K of stack. + */ + char pad[4096]; + char buf[1024], *ptr = buf; +#endif + int i; + if (!err) + return ("Successful return: 0"); + + if (err >= MDB_KEYEXIST && err <= MDB_LAST_ERRCODE) { + i = err - MDB_KEYEXIST; + return mdb_errstr[i]; + } + +#ifdef _WIN32 + /* These are the C-runtime error codes we use. The comment indicates + * their numeric value, and the Win32 error they would correspond to + * if the error actually came from a Win32 API. A major mess, we should + * have used LMDB-specific error codes for everything. + */ + switch(err) { + case ENOENT: /* 2, FILE_NOT_FOUND */ + case EIO: /* 5, ACCESS_DENIED */ + case ENOMEM: /* 12, INVALID_ACCESS */ + case EACCES: /* 13, INVALID_DATA */ + case EBUSY: /* 16, CURRENT_DIRECTORY */ + case EINVAL: /* 22, BAD_COMMAND */ + case ENOSPC: /* 28, OUT_OF_PAPER */ + return strerror(err); + default: + ; + } + buf[0] = 0; + FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, err, 0, ptr, sizeof(buf), (va_list *)pad); + return ptr; +#else + return strerror(err); +#endif +} + +/** assert(3) variant in cursor context */ +#define mdb_cassert(mc, expr) mdb_assert0((mc)->mc_txn->mt_env, expr, #expr) +/** assert(3) variant in transaction context */ +#define mdb_tassert(mc, expr) mdb_assert0((txn)->mt_env, expr, #expr) +/** assert(3) variant in environment context */ +#define mdb_eassert(env, expr) mdb_assert0(env, expr, #expr) + +#ifndef NDEBUG +# define mdb_assert0(env, expr, expr_txt) ((expr) ? (void)0 : \ + mdb_assert_fail(env, expr_txt, mdb_func_, __FILE__, __LINE__)) + +static void +mdb_assert_fail(MDB_env *env, const char *expr_txt, + const char *func, const char *file, int line) +{ + char buf[400]; + sprintf(buf, "%.100s:%d: Assertion '%.200s' failed in %.40s()", + file, line, expr_txt, func); + if (env->me_assert_func) + env->me_assert_func(env, buf); + fprintf(stderr, "%s\n", buf); + abort(); +} +#else +# define mdb_assert0(env, expr, expr_txt) ((void) 0) +#endif /* NDEBUG */ + +#if MDB_DEBUG +/** Return the page number of \b mp which may be sub-page, for debug output */ +static pgno_t +mdb_dbg_pgno(MDB_page *mp) +{ + pgno_t ret; + COPY_PGNO(ret, mp->mp_pgno); + return ret; +} + +/** Display a key in hexadecimal and return the address of the result. + * @param[in] key the key to display + * @param[in] buf the buffer to write into. Should always be #DKBUF. + * @return The key in hexadecimal form. + */ +char * +mdb_dkey(MDB_val *key, char *buf) +{ + char *ptr = buf; + unsigned char *c = key->mv_data; + unsigned int i; + + if (!key) + return ""; + + if (key->mv_size > DKBUF_MAXKEYSIZE) + return "MDB_MAXKEYSIZE"; + /* may want to make this a dynamic check: if the key is mostly + * printable characters, print it as-is instead of converting to hex. + */ +#if 1 + buf[0] = '\0'; + for (i=0; i<key->mv_size; i++) + ptr += sprintf(ptr, "%02x", *c++); +#else + sprintf(buf, "%.*s", key->mv_size, key->mv_data); +#endif + return buf; +} + +static const char * +mdb_leafnode_type(MDB_node *n) +{ + static char *const tp[2][2] = {{"", ": DB"}, {": sub-page", ": sub-DB"}}; + return F_ISSET(n->mn_flags, F_BIGDATA) ? ": overflow page" : + tp[F_ISSET(n->mn_flags, F_DUPDATA)][F_ISSET(n->mn_flags, F_SUBDATA)]; +} + +/** Display all the keys in the page. */ +void +mdb_page_list(MDB_page *mp) +{ + pgno_t pgno = mdb_dbg_pgno(mp); + const char *type, *state = (mp->mp_flags & P_DIRTY) ? ", dirty" : ""; + MDB_node *node; + unsigned int i, nkeys, nsize, total = 0; + MDB_val key; + DKBUF; + + switch (mp->mp_flags & (P_BRANCH|P_LEAF|P_LEAF2|P_META|P_OVERFLOW|P_SUBP)) { + case P_BRANCH: type = "Branch page"; break; + case P_LEAF: type = "Leaf page"; break; + case P_LEAF|P_SUBP: type = "Sub-page"; break; + case P_LEAF|P_LEAF2: type = "LEAF2 page"; break; + case P_LEAF|P_LEAF2|P_SUBP: type = "LEAF2 sub-page"; break; + case P_OVERFLOW: + fprintf(stderr, "Overflow page %"Z"u pages %u%s\n", + pgno, mp->mp_pages, state); + return; + case P_META: + fprintf(stderr, "Meta-page %"Z"u txnid %"Z"u\n", + pgno, ((MDB_meta *)METADATA(mp))->mm_txnid); + return; + default: + fprintf(stderr, "Bad page %"Z"u flags 0x%u\n", pgno, mp->mp_flags); + return; + } + + nkeys = NUMKEYS(mp); + fprintf(stderr, "%s %"Z"u numkeys %d%s\n", type, pgno, nkeys, state); + + for (i=0; i<nkeys; i++) { + if (IS_LEAF2(mp)) { /* LEAF2 pages have no mp_ptrs[] or node headers */ + key.mv_size = nsize = mp->mp_pad; + key.mv_data = LEAF2KEY(mp, i, nsize); + total += nsize; + fprintf(stderr, "key %d: nsize %d, %s\n", i, nsize, DKEY(&key)); + continue; + } + node = NODEPTR(mp, i); + key.mv_size = node->mn_ksize; + key.mv_data = node->mn_data; + nsize = NODESIZE + key.mv_size; + if (IS_BRANCH(mp)) { + fprintf(stderr, "key %d: page %"Z"u, %s\n", i, NODEPGNO(node), + DKEY(&key)); + total += nsize; + } else { + if (F_ISSET(node->mn_flags, F_BIGDATA)) + nsize += sizeof(pgno_t); + else + nsize += NODEDSZ(node); + total += nsize; + nsize += sizeof(indx_t); + fprintf(stderr, "key %d: nsize %d, %s%s\n", + i, nsize, DKEY(&key), mdb_leafnode_type(node)); + } + total = EVEN(total); + } + fprintf(stderr, "Total: header %d + contents %d + unused %d\n", + IS_LEAF2(mp) ? PAGEHDRSZ : PAGEBASE + mp->mp_lower, total, SIZELEFT(mp)); +} + +void +mdb_cursor_chk(MDB_cursor *mc) +{ + unsigned int i; + MDB_node *node; + MDB_page *mp; + + if (!mc->mc_snum && !(mc->mc_flags & C_INITIALIZED)) return; + for (i=0; i<mc->mc_top; i++) { + mp = mc->mc_pg[i]; + node = NODEPTR(mp, mc->mc_ki[i]); + if (NODEPGNO(node) != mc->mc_pg[i+1]->mp_pgno) + printf("oops!\n"); + } + if (mc->mc_ki[i] >= NUMKEYS(mc->mc_pg[i])) + printf("ack!\n"); +} +#endif + +#if (MDB_DEBUG) > 2 +/** Count all the pages in each DB and in the freelist + * and make sure it matches the actual number of pages + * being used. + * All named DBs must be open for a correct count. + */ +static void mdb_audit(MDB_txn *txn) +{ + MDB_cursor mc; + MDB_val key, data; + MDB_ID freecount, count; + MDB_dbi i; + int rc; + + freecount = 0; + mdb_cursor_init(&mc, txn, FREE_DBI, NULL); + while ((rc = mdb_cursor_get(&mc, &key, &data, MDB_NEXT)) == 0) + freecount += *(MDB_ID *)data.mv_data; + mdb_tassert(txn, rc == MDB_NOTFOUND); + + count = 0; + for (i = 0; i<txn->mt_numdbs; i++) { + MDB_xcursor mx; + if (!(txn->mt_dbflags[i] & DB_VALID)) + continue; + mdb_cursor_init(&mc, txn, i, &mx); + if (txn->mt_dbs[i].md_root == P_INVALID) + continue; + count += txn->mt_dbs[i].md_branch_pages + + txn->mt_dbs[i].md_leaf_pages + + txn->mt_dbs[i].md_overflow_pages; + if (txn->mt_dbs[i].md_flags & MDB_DUPSORT) { + rc = mdb_page_search(&mc, NULL, MDB_PS_FIRST); + for (; rc == MDB_SUCCESS; rc = mdb_cursor_sibling(&mc, 1)) { + unsigned j; + MDB_page *mp; + mp = mc.mc_pg[mc.mc_top]; + for (j=0; j<NUMKEYS(mp); j++) { + MDB_node *leaf = NODEPTR(mp, j); + if (leaf->mn_flags & F_SUBDATA) { + MDB_db db; + memcpy(&db, NODEDATA(leaf), sizeof(db)); + count += db.md_branch_pages + db.md_leaf_pages + + db.md_overflow_pages; + } + } + } + mdb_tassert(txn, rc == MDB_NOTFOUND); + } + } + if (freecount + count + 2 /* metapages */ != txn->mt_next_pgno) { + fprintf(stderr, "audit: %lu freecount: %lu count: %lu total: %lu next_pgno: %lu\n", + txn->mt_txnid, freecount, count+2, freecount+count+2, txn->mt_next_pgno); + } +} +#endif + +int +mdb_cmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b) +{ + return txn->mt_dbxs[dbi].md_cmp(a, b); +} + +int +mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b) +{ + return txn->mt_dbxs[dbi].md_dcmp(a, b); +} + +/** Allocate memory for a page. + * Re-use old malloc'd pages first for singletons, otherwise just malloc. + */ +static MDB_page * +mdb_page_malloc(MDB_txn *txn, unsigned num) +{ + MDB_env *env = txn->mt_env; + MDB_page *ret = env->me_dpages; + size_t psize = env->me_psize, sz = psize, off; + /* For ! #MDB_NOMEMINIT, psize counts how much to init. + * For a single page alloc, we init everything after the page header. + * For multi-page, we init the final page; if the caller needed that + * many pages they will be filling in at least up to the last page. + */ + if (num == 1) { + if (ret) { + VGMEMP_ALLOC(env, ret, sz); + VGMEMP_DEFINED(ret, sizeof(ret->mp_next)); + env->me_dpages = ret->mp_next; + return ret; + } + psize -= off = PAGEHDRSZ; + } else { + sz *= num; + off = sz - psize; + } + if ((ret = malloc(sz)) != NULL) { + VGMEMP_ALLOC(env, ret, sz); + if (!(env->me_flags & MDB_NOMEMINIT)) { + memset((char *)ret + off, 0, psize); + ret->mp_pad = 0; + } + } else { + txn->mt_flags |= MDB_TXN_ERROR; + } + return ret; +} +/** Free a single page. + * Saves single pages to a list, for future reuse. + * (This is not used for multi-page overflow pages.) + */ +static void +mdb_page_free(MDB_env *env, MDB_page *mp) +{ + mp->mp_next = env->me_dpages; + VGMEMP_FREE(env, mp); + env->me_dpages = mp; +} + +/** Free a dirty page */ +static void +mdb_dpage_free(MDB_env *env, MDB_page *dp) +{ + if (!IS_OVERFLOW(dp) || dp->mp_pages == 1) { + mdb_page_free(env, dp); + } else { + /* large pages just get freed directly */ + VGMEMP_FREE(env, dp); + free(dp); + } +} + +/** Return all dirty pages to dpage list */ +static void +mdb_dlist_free(MDB_txn *txn) +{ + MDB_env *env = txn->mt_env; + MDB_ID2L dl = txn->mt_u.dirty_list; + unsigned i, n = dl[0].mid; + + for (i = 1; i <= n; i++) { + mdb_dpage_free(env, dl[i].mptr); + } + dl[0].mid = 0; +} + +/** Loosen or free a single page. + * Saves single pages to a list for future reuse + * in this same txn. It has been pulled from the freeDB + * and already resides on the dirty list, but has been + * deleted. Use these pages first before pulling again + * from the freeDB. + * + * If the page wasn't dirtied in this txn, just add it + * to this txn's free list. + */ +static int +mdb_page_loose(MDB_cursor *mc, MDB_page *mp) +{ + int loose = 0; + pgno_t pgno = mp->mp_pgno; + MDB_txn *txn = mc->mc_txn; + + if ((mp->mp_flags & P_DIRTY) && mc->mc_dbi != FREE_DBI) { + if (txn->mt_parent) { + MDB_ID2 *dl = txn->mt_u.dirty_list; + /* If txn has a parent, make sure the page is in our + * dirty list. + */ + if (dl[0].mid) { + unsigned x = mdb_mid2l_search(dl, pgno); + if (x <= dl[0].mid && dl[x].mid == pgno) { + if (mp != dl[x].mptr) { /* bad cursor? */ + mc->mc_flags &= ~(C_INITIALIZED|C_EOF); + txn->mt_flags |= MDB_TXN_ERROR; + return MDB_CORRUPTED; + } + /* ok, it's ours */ + loose = 1; + } + } + } else { + /* no parent txn, so it's just ours */ + loose = 1; + } + } + if (loose) { + DPRINTF(("loosen db %d page %"Z"u", DDBI(mc), + mp->mp_pgno)); + NEXT_LOOSE_PAGE(mp) = txn->mt_loose_pgs; + txn->mt_loose_pgs = mp; + txn->mt_loose_count++; + mp->mp_flags |= P_LOOSE; + } else { + int rc = mdb_midl_append(&txn->mt_free_pgs, pgno); + if (rc) + return rc; + } + + return MDB_SUCCESS; +} + +/** Set or clear P_KEEP in dirty, non-overflow, non-sub pages watched by txn. + * @param[in] mc A cursor handle for the current operation. + * @param[in] pflags Flags of the pages to update: + * P_DIRTY to set P_KEEP, P_DIRTY|P_KEEP to clear it. + * @param[in] all No shortcuts. Needed except after a full #mdb_page_flush(). + * @return 0 on success, non-zero on failure. + */ +static int +mdb_pages_xkeep(MDB_cursor *mc, unsigned pflags, int all) +{ + enum { Mask = P_SUBP|P_DIRTY|P_LOOSE|P_KEEP }; + MDB_txn *txn = mc->mc_txn; + MDB_cursor *m3; + MDB_xcursor *mx; + MDB_page *dp, *mp; + MDB_node *leaf; + unsigned i, j; + int rc = MDB_SUCCESS, level; + + /* Mark pages seen by cursors */ + if (mc->mc_flags & C_UNTRACK) + mc = NULL; /* will find mc in mt_cursors */ + for (i = txn->mt_numdbs;; mc = txn->mt_cursors[--i]) { + for (; mc; mc=mc->mc_next) { + if (!(mc->mc_flags & C_INITIALIZED)) + continue; + for (m3 = mc;; m3 = &mx->mx_cursor) { + mp = NULL; + for (j=0; j<m3->mc_snum; j++) { + mp = m3->mc_pg[j]; + if ((mp->mp_flags & Mask) == pflags) + mp->mp_flags ^= P_KEEP; + } + mx = m3->mc_xcursor; + /* Proceed to mx if it is at a sub-database */ + if (! (mx && (mx->mx_cursor.mc_flags & C_INITIALIZED))) + break; + if (! (mp && (mp->mp_flags & P_LEAF))) + break; + leaf = NODEPTR(mp, m3->mc_ki[j-1]); + if (!(leaf->mn_flags & F_SUBDATA)) + break; + } + } + if (i == 0) + break; + } + + if (all) { + /* Mark dirty root pages */ + for (i=0; i<txn->mt_numdbs; i++) { + if (txn->mt_dbflags[i] & DB_DIRTY) { + pgno_t pgno = txn->mt_dbs[i].md_root; + if (pgno == P_INVALID) + continue; + if ((rc = mdb_page_get(txn, pgno, &dp, &level)) != MDB_SUCCESS) + break; + if ((dp->mp_flags & Mask) == pflags && level <= 1) + dp->mp_flags ^= P_KEEP; + } + } + } + + return rc; +} + +static int mdb_page_flush(MDB_txn *txn, int keep); + +/** Spill pages from the dirty list back to disk. + * This is intended to prevent running into #MDB_TXN_FULL situations, + * but note that they may still occur in a few cases: + * 1) our estimate of the txn size could be too small. Currently this + * seems unlikely, except with a large number of #MDB_MULTIPLE items. + * 2) child txns may run out of space if their parents dirtied a + * lot of pages and never spilled them. TODO: we probably should do + * a preemptive spill during #mdb_txn_begin() of a child txn, if + * the parent's dirty_room is below a given threshold. + * + * Otherwise, if not using nested txns, it is expected that apps will + * not run into #MDB_TXN_FULL any more. The pages are flushed to disk + * the same way as for a txn commit, e.g. their P_DIRTY flag is cleared. + * If the txn never references them again, they can be left alone. + * If the txn only reads them, they can be used without any fuss. + * If the txn writes them again, they can be dirtied immediately without + * going thru all of the work of #mdb_page_touch(). Such references are + * handled by #mdb_page_unspill(). + * + * Also note, we never spill DB root pages, nor pages of active cursors, + * because we'll need these back again soon anyway. And in nested txns, + * we can't spill a page in a child txn if it was already spilled in a + * parent txn. That would alter the parent txns' data even though + * the child hasn't committed yet, and we'd have no way to undo it if + * the child aborted. + * + * @param[in] m0 cursor A cursor handle identifying the transaction and + * database for which we are checking space. + * @param[in] key For a put operation, the key being stored. + * @param[in] data For a put operation, the data being stored. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_spill(MDB_cursor *m0, MDB_val *key, MDB_val *data) +{ + MDB_txn *txn = m0->mc_txn; + MDB_page *dp; + MDB_ID2L dl = txn->mt_u.dirty_list; + unsigned int i, j, need; + int rc; + + if (m0->mc_flags & C_SUB) + return MDB_SUCCESS; + + /* Estimate how much space this op will take */ + i = m0->mc_db->md_depth; + /* Named DBs also dirty the main DB */ + if (m0->mc_dbi > MAIN_DBI) + i += txn->mt_dbs[MAIN_DBI].md_depth; + /* For puts, roughly factor in the key+data size */ + if (key) + i += (LEAFSIZE(key, data) + txn->mt_env->me_psize) / txn->mt_env->me_psize; + i += i; /* double it for good measure */ + need = i; + + if (txn->mt_dirty_room > i) + return MDB_SUCCESS; + + if (!txn->mt_spill_pgs) { + txn->mt_spill_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX); + if (!txn->mt_spill_pgs) + return ENOMEM; + } else { + /* purge deleted slots */ + MDB_IDL sl = txn->mt_spill_pgs; + unsigned int num = sl[0]; + j=0; + for (i=1; i<=num; i++) { + if (!(sl[i] & 1)) + sl[++j] = sl[i]; + } + sl[0] = j; + } + + /* Preserve pages which may soon be dirtied again */ + if ((rc = mdb_pages_xkeep(m0, P_DIRTY, 1)) != MDB_SUCCESS) + goto done; + + /* Less aggressive spill - we originally spilled the entire dirty list, + * with a few exceptions for cursor pages and DB root pages. But this + * turns out to be a lot of wasted effort because in a large txn many + * of those pages will need to be used again. So now we spill only 1/8th + * of the dirty pages. Testing revealed this to be a good tradeoff, + * better than 1/2, 1/4, or 1/10. + */ + if (need < MDB_IDL_UM_MAX / 8) + need = MDB_IDL_UM_MAX / 8; + + /* Save the page IDs of all the pages we're flushing */ + /* flush from the tail forward, this saves a lot of shifting later on. */ + for (i=dl[0].mid; i && need; i--) { + MDB_ID pn = dl[i].mid << 1; + dp = dl[i].mptr; + if (dp->mp_flags & (P_LOOSE|P_KEEP)) + continue; + /* Can't spill twice, make sure it's not already in a parent's + * spill list. + */ + if (txn->mt_parent) { + MDB_txn *tx2; + for (tx2 = txn->mt_parent; tx2; tx2 = tx2->mt_parent) { + if (tx2->mt_spill_pgs) { + j = mdb_midl_search(tx2->mt_spill_pgs, pn); + if (j <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[j] == pn) { + dp->mp_flags |= P_KEEP; + break; + } + } + } + if (tx2) + continue; + } + if ((rc = mdb_midl_append(&txn->mt_spill_pgs, pn))) + goto done; + need--; + } + mdb_midl_sort(txn->mt_spill_pgs); + + /* Flush the spilled part of dirty list */ + if ((rc = mdb_page_flush(txn, i)) != MDB_SUCCESS) + goto done; + + /* Reset any dirty pages we kept that page_flush didn't see */ + rc = mdb_pages_xkeep(m0, P_DIRTY|P_KEEP, i); + +done: + txn->mt_flags |= rc ? MDB_TXN_ERROR : MDB_TXN_SPILLS; + return rc; +} + +/** Find oldest txnid still referenced. Expects txn->mt_txnid > 0. */ +static txnid_t +mdb_find_oldest(MDB_txn *txn) +{ + int i; + txnid_t mr, oldest = txn->mt_txnid - 1; + if (txn->mt_env->me_txns) { + MDB_reader *r = txn->mt_env->me_txns->mti_readers; + for (i = txn->mt_env->me_txns->mti_numreaders; --i >= 0; ) { + if (r[i].mr_pid) { + mr = r[i].mr_txnid; + if (oldest > mr) + oldest = mr; + } + } + } + return oldest; +} + +/** Add a page to the txn's dirty list */ +static void +mdb_page_dirty(MDB_txn *txn, MDB_page *mp) +{ + MDB_ID2 mid; + int rc, (*insert)(MDB_ID2L, MDB_ID2 *); + + if (txn->mt_env->me_flags & MDB_WRITEMAP) { + insert = mdb_mid2l_append; + } else { + insert = mdb_mid2l_insert; + } + mid.mid = mp->mp_pgno; + mid.mptr = mp; + rc = insert(txn->mt_u.dirty_list, &mid); + mdb_tassert(txn, rc == 0); + txn->mt_dirty_room--; +} + +/** Allocate page numbers and memory for writing. Maintain me_pglast, + * me_pghead and mt_next_pgno. + * + * If there are free pages available from older transactions, they + * are re-used first. Otherwise allocate a new page at mt_next_pgno. + * Do not modify the freedB, just merge freeDB records into me_pghead[] + * and move me_pglast to say which records were consumed. Only this + * function can create me_pghead and move me_pglast/mt_next_pgno. + * @param[in] mc cursor A cursor handle identifying the transaction and + * database for which we are allocating. + * @param[in] num the number of pages to allocate. + * @param[out] mp Address of the allocated page(s). Requests for multiple pages + * will always be satisfied by a single contiguous chunk of memory. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) +{ +#ifdef MDB_PARANOID /* Seems like we can ignore this now */ + /* Get at most <Max_retries> more freeDB records once me_pghead + * has enough pages. If not enough, use new pages from the map. + * If <Paranoid> and mc is updating the freeDB, only get new + * records if me_pghead is empty. Then the freelist cannot play + * catch-up with itself by growing while trying to save it. + */ + enum { Paranoid = 1, Max_retries = 500 }; +#else + enum { Paranoid = 0, Max_retries = INT_MAX /*infinite*/ }; +#endif + int rc, retry = num * 60; + MDB_txn *txn = mc->mc_txn; + MDB_env *env = txn->mt_env; + pgno_t pgno, *mop = env->me_pghead; + unsigned i, j, mop_len = mop ? mop[0] : 0, n2 = num-1; + MDB_page *np; + txnid_t oldest = 0, last; + MDB_cursor_op op; + MDB_cursor m2; + int found_old = 0; + + /* If there are any loose pages, just use them */ + if (num == 1 && txn->mt_loose_pgs) { + np = txn->mt_loose_pgs; + txn->mt_loose_pgs = NEXT_LOOSE_PAGE(np); + txn->mt_loose_count--; + DPRINTF(("db %d use loose page %"Z"u", DDBI(mc), + np->mp_pgno)); + *mp = np; + return MDB_SUCCESS; + } + + *mp = NULL; + + /* If our dirty list is already full, we can't do anything */ + if (txn->mt_dirty_room == 0) { + rc = MDB_TXN_FULL; + goto fail; + } + + for (op = MDB_FIRST;; op = MDB_NEXT) { + MDB_val key, data; + MDB_node *leaf; + pgno_t *idl; + + /* Seek a big enough contiguous page range. Prefer + * pages at the tail, just truncating the list. + */ + if (mop_len > n2) { + i = mop_len; + do { + pgno = mop[i]; + if (mop[i-n2] == pgno+n2) + goto search_done; + } while (--i > n2); + if (--retry < 0) + break; + } + + if (op == MDB_FIRST) { /* 1st iteration */ + /* Prepare to fetch more and coalesce */ + last = env->me_pglast; + oldest = env->me_pgoldest; + mdb_cursor_init(&m2, txn, FREE_DBI, NULL); + if (last) { + op = MDB_SET_RANGE; + key.mv_data = &last; /* will look up last+1 */ + key.mv_size = sizeof(last); + } + if (Paranoid && mc->mc_dbi == FREE_DBI) + retry = -1; + } + if (Paranoid && retry < 0 && mop_len) + break; + + last++; + /* Do not fetch more if the record will be too recent */ + if (oldest <= last) { + if (!found_old) { + oldest = mdb_find_oldest(txn); + env->me_pgoldest = oldest; + found_old = 1; + } + if (oldest <= last) + break; + } + rc = mdb_cursor_get(&m2, &key, NULL, op); + if (rc) { + if (rc == MDB_NOTFOUND) + break; + goto fail; + } + last = *(txnid_t*)key.mv_data; + if (oldest <= last) { + if (!found_old) { + oldest = mdb_find_oldest(txn); + env->me_pgoldest = oldest; + found_old = 1; + } + if (oldest <= last) + break; + } + np = m2.mc_pg[m2.mc_top]; + leaf = NODEPTR(np, m2.mc_ki[m2.mc_top]); + if ((rc = mdb_node_read(txn, leaf, &data)) != MDB_SUCCESS) + return rc; + + idl = (MDB_ID *) data.mv_data; + i = idl[0]; + if (!mop) { + if (!(env->me_pghead = mop = mdb_midl_alloc(i))) { + rc = ENOMEM; + goto fail; + } + } else { + if ((rc = mdb_midl_need(&env->me_pghead, i)) != 0) + goto fail; + mop = env->me_pghead; + } + env->me_pglast = last; +#if (MDB_DEBUG) > 1 + DPRINTF(("IDL read txn %"Z"u root %"Z"u num %u", + last, txn->mt_dbs[FREE_DBI].md_root, i)); + for (j = i; j; j--) + DPRINTF(("IDL %"Z"u", idl[j])); +#endif + /* Merge in descending sorted order */ + mdb_midl_xmerge(mop, idl); + mop_len = mop[0]; + } + + /* Use new pages from the map when nothing suitable in the freeDB */ + i = 0; + pgno = txn->mt_next_pgno; + if (pgno + num >= env->me_maxpg) { + DPUTS("DB size maxed out"); + rc = MDB_MAP_FULL; + goto fail; + } + +search_done: + if (env->me_flags & MDB_WRITEMAP) { + np = (MDB_page *)(env->me_map + env->me_psize * pgno); + } else { + if (!(np = mdb_page_malloc(txn, num))) { + rc = ENOMEM; + goto fail; + } + } + if (i) { + mop[0] = mop_len -= num; + /* Move any stragglers down */ + for (j = i-num; j < mop_len; ) + mop[++j] = mop[++i]; + } else { + txn->mt_next_pgno = pgno + num; + } + np->mp_pgno = pgno; + mdb_page_dirty(txn, np); + *mp = np; + + return MDB_SUCCESS; + +fail: + txn->mt_flags |= MDB_TXN_ERROR; + return rc; +} + +/** Copy the used portions of a non-overflow page. + * @param[in] dst page to copy into + * @param[in] src page to copy from + * @param[in] psize size of a page + */ +static void +mdb_page_copy(MDB_page *dst, MDB_page *src, unsigned int psize) +{ + enum { Align = sizeof(pgno_t) }; + indx_t upper = src->mp_upper, lower = src->mp_lower, unused = upper-lower; + + /* If page isn't full, just copy the used portion. Adjust + * alignment so memcpy may copy words instead of bytes. + */ + if ((unused &= -Align) && !IS_LEAF2(src)) { + upper = (upper + PAGEBASE) & -Align; + memcpy(dst, src, (lower + PAGEBASE + (Align-1)) & -Align); + memcpy((pgno_t *)((char *)dst+upper), (pgno_t *)((char *)src+upper), + psize - upper); + } else { + memcpy(dst, src, psize - unused); + } +} + +/** Pull a page off the txn's spill list, if present. + * If a page being referenced was spilled to disk in this txn, bring + * it back and make it dirty/writable again. + * @param[in] txn the transaction handle. + * @param[in] mp the page being referenced. It must not be dirty. + * @param[out] ret the writable page, if any. ret is unchanged if + * mp wasn't spilled. + */ +static int +mdb_page_unspill(MDB_txn *txn, MDB_page *mp, MDB_page **ret) +{ + MDB_env *env = txn->mt_env; + const MDB_txn *tx2; + unsigned x; + pgno_t pgno = mp->mp_pgno, pn = pgno << 1; + + for (tx2 = txn; tx2; tx2=tx2->mt_parent) { + if (!tx2->mt_spill_pgs) + continue; + x = mdb_midl_search(tx2->mt_spill_pgs, pn); + if (x <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[x] == pn) { + MDB_page *np; + int num; + if (txn->mt_dirty_room == 0) + return MDB_TXN_FULL; + if (IS_OVERFLOW(mp)) + num = mp->mp_pages; + else + num = 1; + if (env->me_flags & MDB_WRITEMAP) { + np = mp; + } else { + np = mdb_page_malloc(txn, num); + if (!np) + return ENOMEM; + if (num > 1) + memcpy(np, mp, num * env->me_psize); + else + mdb_page_copy(np, mp, env->me_psize); + } + if (tx2 == txn) { + /* If in current txn, this page is no longer spilled. + * If it happens to be the last page, truncate the spill list. + * Otherwise mark it as deleted by setting the LSB. + */ + if (x == txn->mt_spill_pgs[0]) + txn->mt_spill_pgs[0]--; + else + txn->mt_spill_pgs[x] |= 1; + } /* otherwise, if belonging to a parent txn, the + * page remains spilled until child commits + */ + + mdb_page_dirty(txn, np); + np->mp_flags |= P_DIRTY; + *ret = np; + break; + } + } + return MDB_SUCCESS; +} + +/** Touch a page: make it dirty and re-insert into tree with updated pgno. + * @param[in] mc cursor pointing to the page to be touched + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_touch(MDB_cursor *mc) +{ + MDB_page *mp = mc->mc_pg[mc->mc_top], *np; + MDB_txn *txn = mc->mc_txn; + MDB_cursor *m2, *m3; + pgno_t pgno; + int rc; + + if (!F_ISSET(mp->mp_flags, P_DIRTY)) { + if (txn->mt_flags & MDB_TXN_SPILLS) { + np = NULL; + rc = mdb_page_unspill(txn, mp, &np); + if (rc) + goto fail; + if (np) + goto done; + } + if ((rc = mdb_midl_need(&txn->mt_free_pgs, 1)) || + (rc = mdb_page_alloc(mc, 1, &np))) + goto fail; + pgno = np->mp_pgno; + DPRINTF(("touched db %d page %"Z"u -> %"Z"u", DDBI(mc), + mp->mp_pgno, pgno)); + mdb_cassert(mc, mp->mp_pgno != pgno); + mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno); + /* Update the parent page, if any, to point to the new page */ + if (mc->mc_top) { + MDB_page *parent = mc->mc_pg[mc->mc_top-1]; + MDB_node *node = NODEPTR(parent, mc->mc_ki[mc->mc_top-1]); + SETPGNO(node, pgno); + } else { + mc->mc_db->md_root = pgno; + } + } else if (txn->mt_parent && !IS_SUBP(mp)) { + MDB_ID2 mid, *dl = txn->mt_u.dirty_list; + pgno = mp->mp_pgno; + /* If txn has a parent, make sure the page is in our + * dirty list. + */ + if (dl[0].mid) { + unsigned x = mdb_mid2l_search(dl, pgno); + if (x <= dl[0].mid && dl[x].mid == pgno) { + if (mp != dl[x].mptr) { /* bad cursor? */ + mc->mc_flags &= ~(C_INITIALIZED|C_EOF); + txn->mt_flags |= MDB_TXN_ERROR; + return MDB_CORRUPTED; + } + return 0; + } + } + mdb_cassert(mc, dl[0].mid < MDB_IDL_UM_MAX); + /* No - copy it */ + np = mdb_page_malloc(txn, 1); + if (!np) + return ENOMEM; + mid.mid = pgno; + mid.mptr = np; + rc = mdb_mid2l_insert(dl, &mid); + mdb_cassert(mc, rc == 0); + } else { + return 0; + } + + mdb_page_copy(np, mp, txn->mt_env->me_psize); + np->mp_pgno = pgno; + np->mp_flags |= P_DIRTY; + +done: + /* Adjust cursors pointing to mp */ + mc->mc_pg[mc->mc_top] = np; + m2 = txn->mt_cursors[mc->mc_dbi]; + if (mc->mc_flags & C_SUB) { + for (; m2; m2=m2->mc_next) { + m3 = &m2->mc_xcursor->mx_cursor; + if (m3->mc_snum < mc->mc_snum) continue; + if (m3->mc_pg[mc->mc_top] == mp) + m3->mc_pg[mc->mc_top] = np; + } + } else { + for (; m2; m2=m2->mc_next) { + if (m2->mc_snum < mc->mc_snum) continue; + if (m2->mc_pg[mc->mc_top] == mp) { + m2->mc_pg[mc->mc_top] = np; + if ((mc->mc_db->md_flags & MDB_DUPSORT) && + IS_LEAF(np) && + m2->mc_ki[mc->mc_top] == mc->mc_ki[mc->mc_top]) + { + MDB_node *leaf = NODEPTR(np, mc->mc_ki[mc->mc_top]); + if (!(leaf->mn_flags & F_SUBDATA)) + m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); + } + } + } + } + return 0; + +fail: + txn->mt_flags |= MDB_TXN_ERROR; + return rc; +} + +int +mdb_env_sync(MDB_env *env, int force) +{ + int rc = 0; + if (env->me_flags & MDB_RDONLY) + return EACCES; + if (force || !F_ISSET(env->me_flags, MDB_NOSYNC)) { + if (env->me_flags & MDB_WRITEMAP) { + int flags = ((env->me_flags & MDB_MAPASYNC) && !force) + ? MS_ASYNC : MS_SYNC; + if (MDB_MSYNC(env->me_map, env->me_mapsize, flags)) + rc = ErrCode(); +#ifdef _WIN32 + else if (flags == MS_SYNC && MDB_FDATASYNC(env->me_fd)) + rc = ErrCode(); +#endif + } else { +#ifdef BROKEN_FDATASYNC + if (env->me_flags & MDB_FSYNCONLY) { + if (fsync(env->me_fd)) + rc = ErrCode(); + } else +#endif + if (MDB_FDATASYNC(env->me_fd)) + rc = ErrCode(); + } + } + return rc; +} + +/** Back up parent txn's cursors, then grab the originals for tracking */ +static int +mdb_cursor_shadow(MDB_txn *src, MDB_txn *dst) +{ + MDB_cursor *mc, *bk; + MDB_xcursor *mx; + size_t size; + int i; + + for (i = src->mt_numdbs; --i >= 0; ) { + if ((mc = src->mt_cursors[i]) != NULL) { + size = sizeof(MDB_cursor); + if (mc->mc_xcursor) + size += sizeof(MDB_xcursor); + for (; mc; mc = bk->mc_next) { + bk = malloc(size); + if (!bk) + return ENOMEM; + *bk = *mc; + mc->mc_backup = bk; + mc->mc_db = &dst->mt_dbs[i]; + /* Kill pointers into src - and dst to reduce abuse: The + * user may not use mc until dst ends. Otherwise we'd... + */ + mc->mc_txn = NULL; /* ...set this to dst */ + mc->mc_dbflag = NULL; /* ...and &dst->mt_dbflags[i] */ + if ((mx = mc->mc_xcursor) != NULL) { + *(MDB_xcursor *)(bk+1) = *mx; + mx->mx_cursor.mc_txn = NULL; /* ...and dst. */ + } + mc->mc_next = dst->mt_cursors[i]; + dst->mt_cursors[i] = mc; + } + } + } + return MDB_SUCCESS; +} + +/** Close this write txn's cursors, give parent txn's cursors back to parent. + * @param[in] txn the transaction handle. + * @param[in] merge true to keep changes to parent cursors, false to revert. + * @return 0 on success, non-zero on failure. + */ +static void +mdb_cursors_close(MDB_txn *txn, unsigned merge) +{ + MDB_cursor **cursors = txn->mt_cursors, *mc, *next, *bk; + MDB_xcursor *mx; + int i; + + for (i = txn->mt_numdbs; --i >= 0; ) { + for (mc = cursors[i]; mc; mc = next) { + next = mc->mc_next; + if ((bk = mc->mc_backup) != NULL) { + if (merge) { + /* Commit changes to parent txn */ + mc->mc_next = bk->mc_next; + mc->mc_backup = bk->mc_backup; + mc->mc_txn = bk->mc_txn; + mc->mc_db = bk->mc_db; + mc->mc_dbflag = bk->mc_dbflag; + if ((mx = mc->mc_xcursor) != NULL) + mx->mx_cursor.mc_txn = bk->mc_txn; + } else { + /* Abort nested txn */ + *mc = *bk; + if ((mx = mc->mc_xcursor) != NULL) + *mx = *(MDB_xcursor *)(bk+1); + } + mc = bk; + } + /* Only malloced cursors are permanently tracked. */ + free(mc); + } + cursors[i] = NULL; + } +} + +#if !(MDB_DEBUG) +#define mdb_txn_reset0(txn, act) mdb_txn_reset0(txn) +#endif +static void +mdb_txn_reset0(MDB_txn *txn, const char *act); + +#if !(MDB_PIDLOCK) /* Currently the same as defined(_WIN32) */ +enum Pidlock_op { + Pidset, Pidcheck +}; +#else +enum Pidlock_op { + Pidset = F_SETLK, Pidcheck = F_GETLK +}; +#endif + +/** Set or check a pid lock. Set returns 0 on success. + * Check returns 0 if the process is certainly dead, nonzero if it may + * be alive (the lock exists or an error happened so we do not know). + * + * On Windows Pidset is a no-op, we merely check for the existence + * of the process with the given pid. On POSIX we use a single byte + * lock on the lockfile, set at an offset equal to the pid. + */ +static int +mdb_reader_pid(MDB_env *env, enum Pidlock_op op, MDB_PID_T pid) +{ +#if !(MDB_PIDLOCK) /* Currently the same as defined(_WIN32) */ + int ret = 0; + HANDLE h; + if (op == Pidcheck) { + h = OpenProcess(env->me_pidquery, FALSE, pid); + /* No documented "no such process" code, but other program use this: */ + if (!h) + return ErrCode() != ERROR_INVALID_PARAMETER; + /* A process exists until all handles to it close. Has it exited? */ + ret = WaitForSingleObject(h, 0) != 0; + CloseHandle(h); + } + return ret; +#else + for (;;) { + int rc; + struct flock lock_info; + memset(&lock_info, 0, sizeof(lock_info)); + lock_info.l_type = F_WRLCK; + lock_info.l_whence = SEEK_SET; + lock_info.l_start = pid; + lock_info.l_len = 1; + if ((rc = fcntl(env->me_lfd, op, &lock_info)) == 0) { + if (op == F_GETLK && lock_info.l_type != F_UNLCK) + rc = -1; + } else if ((rc = ErrCode()) == EINTR) { + continue; + } + return rc; + } +#endif +} + +/** Common code for #mdb_txn_begin() and #mdb_txn_renew(). + * @param[in] txn the transaction handle to initialize + * @return 0 on success, non-zero on failure. + */ +static int +mdb_txn_renew0(MDB_txn *txn) +{ + MDB_env *env = txn->mt_env; + MDB_txninfo *ti = env->me_txns; + MDB_meta *meta; + unsigned int i, nr; + uint16_t x; + int rc, new_notls = 0; + + if (txn->mt_flags & MDB_TXN_RDONLY) { + /* Setup db info */ + txn->mt_numdbs = env->me_numdbs; + txn->mt_dbxs = env->me_dbxs; /* mostly static anyway */ + if (!ti) { + meta = env->me_metas[ mdb_env_pick_meta(env) ]; + txn->mt_txnid = meta->mm_txnid; + txn->mt_u.reader = NULL; + } else { + MDB_reader *r = (env->me_flags & MDB_NOTLS) ? txn->mt_u.reader : + pthread_getspecific(env->me_txkey); + if (r) { + if (r->mr_pid != env->me_pid || r->mr_txnid != (txnid_t)-1) + return MDB_BAD_RSLOT; + } else { + MDB_PID_T pid = env->me_pid; + MDB_THR_T tid = pthread_self(); + + if (!env->me_live_reader) { + rc = mdb_reader_pid(env, Pidset, pid); + if (rc) + return rc; + env->me_live_reader = 1; + } + + LOCK_MUTEX_R(env); + nr = ti->mti_numreaders; + for (i=0; i<nr; i++) + if (ti->mti_readers[i].mr_pid == 0) + break; + if (i == env->me_maxreaders) { + UNLOCK_MUTEX_R(env); + return MDB_READERS_FULL; + } + ti->mti_readers[i].mr_pid = pid; + ti->mti_readers[i].mr_tid = tid; + if (i == nr) + ti->mti_numreaders = ++nr; + /* Save numreaders for un-mutexed mdb_env_close() */ + env->me_numreaders = nr; + UNLOCK_MUTEX_R(env); + + r = &ti->mti_readers[i]; + new_notls = (env->me_flags & MDB_NOTLS); + if (!new_notls && (rc=pthread_setspecific(env->me_txkey, r))) { + r->mr_pid = 0; + return rc; + } + } + do /* LY: Retry on a race, ITS#7970. */ + r->mr_txnid = ti->mti_txnid; + while(r->mr_txnid != ti->mti_txnid); + txn->mt_txnid = r->mr_txnid; + txn->mt_u.reader = r; + meta = env->me_metas[txn->mt_txnid & 1]; + } + } else { + if (ti) { + LOCK_MUTEX_W(env); + + txn->mt_txnid = ti->mti_txnid; + meta = env->me_metas[txn->mt_txnid & 1]; + } else { + meta = env->me_metas[ mdb_env_pick_meta(env) ]; + txn->mt_txnid = meta->mm_txnid; + } + /* Setup db info */ + txn->mt_numdbs = env->me_numdbs; + txn->mt_txnid++; +#if MDB_DEBUG + if (txn->mt_txnid == mdb_debug_start) + mdb_debug = 1; +#endif + txn->mt_flags = 0; + txn->mt_child = NULL; + txn->mt_loose_pgs = NULL; + txn->mt_loose_count = 0; + txn->mt_dirty_room = MDB_IDL_UM_MAX; + txn->mt_u.dirty_list = env->me_dirty_list; + txn->mt_u.dirty_list[0].mid = 0; + txn->mt_free_pgs = env->me_free_pgs; + txn->mt_free_pgs[0] = 0; + txn->mt_spill_pgs = NULL; + env->me_txn = txn; + memcpy(txn->mt_dbiseqs, env->me_dbiseqs, env->me_maxdbs * sizeof(unsigned int)); + } + + /* Copy the DB info and flags */ + memcpy(txn->mt_dbs, meta->mm_dbs, 2 * sizeof(MDB_db)); + + /* Moved to here to avoid a data race in read TXNs */ + txn->mt_next_pgno = meta->mm_last_pg+1; + + for (i=2; i<txn->mt_numdbs; i++) { + x = env->me_dbflags[i]; + txn->mt_dbs[i].md_flags = x & PERSISTENT_FLAGS; + txn->mt_dbflags[i] = (x & MDB_VALID) ? DB_VALID|DB_STALE : 0; + } + txn->mt_dbflags[0] = txn->mt_dbflags[1] = DB_VALID; + + if (env->me_maxpg < txn->mt_next_pgno) { + mdb_txn_reset0(txn, "renew0-mapfail"); + if (new_notls) { + txn->mt_u.reader->mr_pid = 0; + txn->mt_u.reader = NULL; + } + return MDB_MAP_RESIZED; + } + + return MDB_SUCCESS; +} + +int +mdb_txn_renew(MDB_txn *txn) +{ + int rc; + + if (!txn || txn->mt_dbxs) /* A reset txn has mt_dbxs==NULL */ + return EINVAL; + + if (txn->mt_env->me_flags & MDB_FATAL_ERROR) { + DPUTS("environment had fatal error, must shutdown!"); + return MDB_PANIC; + } + + rc = mdb_txn_renew0(txn); + if (rc == MDB_SUCCESS) { + DPRINTF(("renew txn %"Z"u%c %p on mdbenv %p, root page %"Z"u", + txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w', + (void *)txn, (void *)txn->mt_env, txn->mt_dbs[MAIN_DBI].md_root)); + } + return rc; +} + +int +mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret) +{ + MDB_txn *txn; + MDB_ntxn *ntxn; + int rc, size, tsize = sizeof(MDB_txn); + + if (env->me_flags & MDB_FATAL_ERROR) { + DPUTS("environment had fatal error, must shutdown!"); + return MDB_PANIC; + } + if ((env->me_flags & MDB_RDONLY) && !(flags & MDB_RDONLY)) + return EACCES; + if (parent) { + /* Nested transactions: Max 1 child, write txns only, no writemap */ + if (parent->mt_child || + (flags & MDB_RDONLY) || + (parent->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_ERROR)) || + (env->me_flags & MDB_WRITEMAP)) + { + return (parent->mt_flags & MDB_TXN_RDONLY) ? EINVAL : MDB_BAD_TXN; + } + tsize = sizeof(MDB_ntxn); + } + size = tsize; + if (!(flags & MDB_RDONLY)) { + if (!parent) { + txn = env->me_txn0; /* just reuse preallocated write txn */ + goto ok; + } + /* child txns use own copy of cursors */ + size += env->me_maxdbs * sizeof(MDB_cursor *); + } + size += env->me_maxdbs * (sizeof(MDB_db)+1); + + if ((txn = calloc(1, size)) == NULL) { + DPRINTF(("calloc: %s", strerror(errno))); + return ENOMEM; + } + txn->mt_dbs = (MDB_db *) ((char *)txn + tsize); + if (flags & MDB_RDONLY) { + txn->mt_flags |= MDB_TXN_RDONLY; + txn->mt_dbflags = (unsigned char *)(txn->mt_dbs + env->me_maxdbs); + txn->mt_dbiseqs = env->me_dbiseqs; + } else { + txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs); + if (parent) { + txn->mt_dbiseqs = parent->mt_dbiseqs; + txn->mt_dbflags = (unsigned char *)(txn->mt_cursors + env->me_maxdbs); + } else { + txn->mt_dbiseqs = (unsigned int *)(txn->mt_cursors + env->me_maxdbs); + txn->mt_dbflags = (unsigned char *)(txn->mt_dbiseqs + env->me_maxdbs); + } + } + txn->mt_env = env; + +ok: + if (parent) { + unsigned int i; + txn->mt_u.dirty_list = malloc(sizeof(MDB_ID2)*MDB_IDL_UM_SIZE); + if (!txn->mt_u.dirty_list || + !(txn->mt_free_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX))) + { + free(txn->mt_u.dirty_list); + free(txn); + return ENOMEM; + } + txn->mt_txnid = parent->mt_txnid; + txn->mt_dirty_room = parent->mt_dirty_room; + txn->mt_u.dirty_list[0].mid = 0; + txn->mt_spill_pgs = NULL; + txn->mt_next_pgno = parent->mt_next_pgno; + parent->mt_child = txn; + txn->mt_parent = parent; + txn->mt_numdbs = parent->mt_numdbs; + txn->mt_flags = parent->mt_flags; + txn->mt_dbxs = parent->mt_dbxs; + memcpy(txn->mt_dbs, parent->mt_dbs, txn->mt_numdbs * sizeof(MDB_db)); + /* Copy parent's mt_dbflags, but clear DB_NEW */ + for (i=0; i<txn->mt_numdbs; i++) + txn->mt_dbflags[i] = parent->mt_dbflags[i] & ~DB_NEW; + rc = 0; + ntxn = (MDB_ntxn *)txn; + ntxn->mnt_pgstate = env->me_pgstate; /* save parent me_pghead & co */ + if (env->me_pghead) { + size = MDB_IDL_SIZEOF(env->me_pghead); + env->me_pghead = mdb_midl_alloc(env->me_pghead[0]); + if (env->me_pghead) + memcpy(env->me_pghead, ntxn->mnt_pgstate.mf_pghead, size); + else + rc = ENOMEM; + } + if (!rc) + rc = mdb_cursor_shadow(parent, txn); + if (rc) + mdb_txn_reset0(txn, "beginchild-fail"); + } else { + rc = mdb_txn_renew0(txn); + } + if (rc) { + if (txn != env->me_txn0) + free(txn); + } else { + *ret = txn; + DPRINTF(("begin txn %"Z"u%c %p on mdbenv %p, root page %"Z"u", + txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w', + (void *) txn, (void *) env, txn->mt_dbs[MAIN_DBI].md_root)); + } + + return rc; +} + +MDB_env * +mdb_txn_env(MDB_txn *txn) +{ + if(!txn) return NULL; + return txn->mt_env; +} + +/** Export or close DBI handles opened in this txn. */ +static void +mdb_dbis_update(MDB_txn *txn, int keep) +{ + int i; + MDB_dbi n = txn->mt_numdbs; + MDB_env *env = txn->mt_env; + unsigned char *tdbflags = txn->mt_dbflags; + + for (i = n; --i >= 2;) { + if (tdbflags[i] & DB_NEW) { + if (keep) { + env->me_dbflags[i] = txn->mt_dbs[i].md_flags | MDB_VALID; + } else { + char *ptr = env->me_dbxs[i].md_name.mv_data; + if (ptr) { + env->me_dbxs[i].md_name.mv_data = NULL; + env->me_dbxs[i].md_name.mv_size = 0; + env->me_dbflags[i] = 0; + env->me_dbiseqs[i]++; + free(ptr); + } + } + } + } + if (keep && env->me_numdbs < n) + env->me_numdbs = n; +} + +/** Common code for #mdb_txn_reset() and #mdb_txn_abort(). + * May be called twice for readonly txns: First reset it, then abort. + * @param[in] txn the transaction handle to reset + * @param[in] act why the transaction is being reset + */ +static void +mdb_txn_reset0(MDB_txn *txn, const char *act) +{ + MDB_env *env = txn->mt_env; + + /* Close any DBI handles opened in this txn */ + mdb_dbis_update(txn, 0); + + DPRINTF(("%s txn %"Z"u%c %p on mdbenv %p, root page %"Z"u", + act, txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w', + (void *) txn, (void *)env, txn->mt_dbs[MAIN_DBI].md_root)); + + if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) { + if (txn->mt_u.reader) { + txn->mt_u.reader->mr_txnid = (txnid_t)-1; + if (!(env->me_flags & MDB_NOTLS)) + txn->mt_u.reader = NULL; /* txn does not own reader */ + } + txn->mt_numdbs = 0; /* close nothing if called again */ + txn->mt_dbxs = NULL; /* mark txn as reset */ + } else { + pgno_t *pghead = env->me_pghead; + + mdb_cursors_close(txn, 0); + if (!(env->me_flags & MDB_WRITEMAP)) { + mdb_dlist_free(txn); + } + + if (!txn->mt_parent) { + if (mdb_midl_shrink(&txn->mt_free_pgs)) + env->me_free_pgs = txn->mt_free_pgs; + /* me_pgstate: */ + env->me_pghead = NULL; + env->me_pglast = 0; + + env->me_txn = NULL; + /* The writer mutex was locked in mdb_txn_begin. */ + if (env->me_txns) + UNLOCK_MUTEX_W(env); + } else { + txn->mt_parent->mt_child = NULL; + env->me_pgstate = ((MDB_ntxn *)txn)->mnt_pgstate; + mdb_midl_free(txn->mt_free_pgs); + mdb_midl_free(txn->mt_spill_pgs); + free(txn->mt_u.dirty_list); + } + + mdb_midl_free(pghead); + } +} + +void +mdb_txn_reset(MDB_txn *txn) +{ + if (txn == NULL) + return; + + /* This call is only valid for read-only txns */ + if (!(txn->mt_flags & MDB_TXN_RDONLY)) + return; + + mdb_txn_reset0(txn, "reset"); +} + +void +mdb_txn_abort(MDB_txn *txn) +{ + if (txn == NULL) + return; + + if (txn->mt_child) + mdb_txn_abort(txn->mt_child); + + mdb_txn_reset0(txn, "abort"); + /* Free reader slot tied to this txn (if MDB_NOTLS && writable FS) */ + if ((txn->mt_flags & MDB_TXN_RDONLY) && txn->mt_u.reader) + txn->mt_u.reader->mr_pid = 0; + + if (txn != txn->mt_env->me_txn0) + free(txn); +} + +/** Save the freelist as of this transaction to the freeDB. + * This changes the freelist. Keep trying until it stabilizes. + */ +static int +mdb_freelist_save(MDB_txn *txn) +{ + /* env->me_pghead[] can grow and shrink during this call. + * env->me_pglast and txn->mt_free_pgs[] can only grow. + * Page numbers cannot disappear from txn->mt_free_pgs[]. + */ + MDB_cursor mc; + MDB_env *env = txn->mt_env; + int rc, maxfree_1pg = env->me_maxfree_1pg, more = 1; + txnid_t pglast = 0, head_id = 0; + pgno_t freecnt = 0, *free_pgs, *mop; + ssize_t head_room = 0, total_room = 0, mop_len, clean_limit; + + mdb_cursor_init(&mc, txn, FREE_DBI, NULL); + + if (env->me_pghead) { + /* Make sure first page of freeDB is touched and on freelist */ + rc = mdb_page_search(&mc, NULL, MDB_PS_FIRST|MDB_PS_MODIFY); + if (rc && rc != MDB_NOTFOUND) + return rc; + } + + if (!env->me_pghead && txn->mt_loose_pgs) { + /* Put loose page numbers in mt_free_pgs, since + * we may be unable to return them to me_pghead. + */ + MDB_page *mp = txn->mt_loose_pgs; + if ((rc = mdb_midl_need(&txn->mt_free_pgs, txn->mt_loose_count)) != 0) + return rc; + for (; mp; mp = NEXT_LOOSE_PAGE(mp)) + mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno); + txn->mt_loose_pgs = NULL; + txn->mt_loose_count = 0; + } + + /* MDB_RESERVE cancels meminit in ovpage malloc (when no WRITEMAP) */ + clean_limit = (env->me_flags & (MDB_NOMEMINIT|MDB_WRITEMAP)) + ? SSIZE_MAX : maxfree_1pg; + + for (;;) { + /* Come back here after each Put() in case freelist changed */ + MDB_val key, data; + pgno_t *pgs; + ssize_t j; + + /* If using records from freeDB which we have not yet + * deleted, delete them and any we reserved for me_pghead. + */ + while (pglast < env->me_pglast) { + rc = mdb_cursor_first(&mc, &key, NULL); + if (rc) + return rc; + pglast = head_id = *(txnid_t *)key.mv_data; + total_room = head_room = 0; + mdb_tassert(txn, pglast <= env->me_pglast); + rc = mdb_cursor_del(&mc, 0); + if (rc) + return rc; + } + + /* Save the IDL of pages freed by this txn, to a single record */ + if (freecnt < txn->mt_free_pgs[0]) { + if (!freecnt) { + /* Make sure last page of freeDB is touched and on freelist */ + rc = mdb_page_search(&mc, NULL, MDB_PS_LAST|MDB_PS_MODIFY); + if (rc && rc != MDB_NOTFOUND) + return rc; + } + free_pgs = txn->mt_free_pgs; + /* Write to last page of freeDB */ + key.mv_size = sizeof(txn->mt_txnid); + key.mv_data = &txn->mt_txnid; + do { + freecnt = free_pgs[0]; + data.mv_size = MDB_IDL_SIZEOF(free_pgs); + rc = mdb_cursor_put(&mc, &key, &data, MDB_RESERVE); + if (rc) + return rc; + /* Retry if mt_free_pgs[] grew during the Put() */ + free_pgs = txn->mt_free_pgs; + } while (freecnt < free_pgs[0]); + mdb_midl_sort(free_pgs); + memcpy(data.mv_data, free_pgs, data.mv_size); +#if (MDB_DEBUG) > 1 + { + unsigned int i = free_pgs[0]; + DPRINTF(("IDL write txn %"Z"u root %"Z"u num %u", + txn->mt_txnid, txn->mt_dbs[FREE_DBI].md_root, i)); + for (; i; i--) + DPRINTF(("IDL %"Z"u", free_pgs[i])); + } +#endif + continue; + } + + mop = env->me_pghead; + mop_len = (mop ? mop[0] : 0) + txn->mt_loose_count; + + /* Reserve records for me_pghead[]. Split it if multi-page, + * to avoid searching freeDB for a page range. Use keys in + * range [1,me_pglast]: Smaller than txnid of oldest reader. + */ + if (total_room >= mop_len) { + if (total_room == mop_len || --more < 0) + break; + } else if (head_room >= maxfree_1pg && head_id > 1) { + /* Keep current record (overflow page), add a new one */ + head_id--; + head_room = 0; + } + /* (Re)write {key = head_id, IDL length = head_room} */ + total_room -= head_room; + head_room = mop_len - total_room; + if (head_room > maxfree_1pg && head_id > 1) { + /* Overflow multi-page for part of me_pghead */ + head_room /= head_id; /* amortize page sizes */ + head_room += maxfree_1pg - head_room % (maxfree_1pg + 1); + } else if (head_room < 0) { + /* Rare case, not bothering to delete this record */ + head_room = 0; + } + key.mv_size = sizeof(head_id); + key.mv_data = &head_id; + data.mv_size = (head_room + 1) * sizeof(pgno_t); + rc = mdb_cursor_put(&mc, &key, &data, MDB_RESERVE); + if (rc) + return rc; + /* IDL is initially empty, zero out at least the length */ + pgs = (pgno_t *)data.mv_data; + j = head_room > clean_limit ? head_room : 0; + do { + pgs[j] = 0; + } while (--j >= 0); + total_room += head_room; + } + + /* Return loose page numbers to me_pghead, though usually none are + * left at this point. The pages themselves remain in dirty_list. + */ + if (txn->mt_loose_pgs) { + MDB_page *mp = txn->mt_loose_pgs; + unsigned count = txn->mt_loose_count; + MDB_IDL loose; + /* Room for loose pages + temp IDL with same */ + if ((rc = mdb_midl_need(&env->me_pghead, 2*count+1)) != 0) + return rc; + mop = env->me_pghead; + loose = mop + MDB_IDL_ALLOCLEN(mop) - count; + for (count = 0; mp; mp = NEXT_LOOSE_PAGE(mp)) + loose[ ++count ] = mp->mp_pgno; + loose[0] = count; + mdb_midl_sort(loose); + mdb_midl_xmerge(mop, loose); + txn->mt_loose_pgs = NULL; + txn->mt_loose_count = 0; + mop_len = mop[0]; + } + + /* Fill in the reserved me_pghead records */ + rc = MDB_SUCCESS; + if (mop_len) { + MDB_val key, data; + + mop += mop_len; + rc = mdb_cursor_first(&mc, &key, &data); + for (; !rc; rc = mdb_cursor_next(&mc, &key, &data, MDB_NEXT)) { + txnid_t id = *(txnid_t *)key.mv_data; + ssize_t len = (ssize_t)(data.mv_size / sizeof(MDB_ID)) - 1; + MDB_ID save; + + mdb_tassert(txn, len >= 0 && id <= env->me_pglast); + key.mv_data = &id; + if (len > mop_len) { + len = mop_len; + data.mv_size = (len + 1) * sizeof(MDB_ID); + } + data.mv_data = mop -= len; + save = mop[0]; + mop[0] = len; + rc = mdb_cursor_put(&mc, &key, &data, MDB_CURRENT); + mop[0] = save; + if (rc || !(mop_len -= len)) + break; + } + } + return rc; +} + +/** Flush (some) dirty pages to the map, after clearing their dirty flag. + * @param[in] txn the transaction that's being committed + * @param[in] keep number of initial pages in dirty_list to keep dirty. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_flush(MDB_txn *txn, int keep) +{ + MDB_env *env = txn->mt_env; + MDB_ID2L dl = txn->mt_u.dirty_list; + unsigned psize = env->me_psize, j; + int i, pagecount = dl[0].mid, rc; + size_t size = 0, pos = 0; + pgno_t pgno = 0; + MDB_page *dp = NULL; +#ifdef _WIN32 + OVERLAPPED ov; +#else + struct iovec iov[MDB_COMMIT_PAGES]; + ssize_t wpos = 0, wsize = 0, wres; + size_t next_pos = 1; /* impossible pos, so pos != next_pos */ + int n = 0; +#endif + + j = i = keep; + + if (env->me_flags & MDB_WRITEMAP) { + /* Clear dirty flags */ + while (++i <= pagecount) { + dp = dl[i].mptr; + /* Don't flush this page yet */ + if (dp->mp_flags & (P_LOOSE|P_KEEP)) { + dp->mp_flags &= ~P_KEEP; + dl[++j] = dl[i]; + continue; + } + dp->mp_flags &= ~P_DIRTY; + } + goto done; + } + + /* Write the pages */ + for (;;) { + if (++i <= pagecount) { + dp = dl[i].mptr; + /* Don't flush this page yet */ + if (dp->mp_flags & (P_LOOSE|P_KEEP)) { + dp->mp_flags &= ~P_KEEP; + dl[i].mid = 0; + continue; + } + pgno = dl[i].mid; + /* clear dirty flag */ + dp->mp_flags &= ~P_DIRTY; + pos = pgno * psize; + size = psize; + if (IS_OVERFLOW(dp)) size *= dp->mp_pages; + } +#ifdef _WIN32 + else break; + + /* Windows actually supports scatter/gather I/O, but only on + * unbuffered file handles. Since we're relying on the OS page + * cache for all our data, that's self-defeating. So we just + * write pages one at a time. We use the ov structure to set + * the write offset, to at least save the overhead of a Seek + * system call. + */ + DPRINTF(("committing page %"Z"u", pgno)); + memset(&ov, 0, sizeof(ov)); + ov.Offset = pos & 0xffffffff; + ov.OffsetHigh = pos >> 16 >> 16; + if (!WriteFile(env->me_fd, dp, size, NULL, &ov)) { + rc = ErrCode(); + DPRINTF(("WriteFile: %d", rc)); + return rc; + } +#else + /* Write up to MDB_COMMIT_PAGES dirty pages at a time. */ + if (pos!=next_pos || n==MDB_COMMIT_PAGES || wsize+size>MAX_WRITE) { + if (n) { + /* Write previous page(s) */ +#ifdef MDB_USE_PWRITEV + wres = pwritev(env->me_fd, iov, n, wpos); +#else + if (n == 1) { + wres = pwrite(env->me_fd, iov[0].iov_base, wsize, wpos); + } else { + if (lseek(env->me_fd, wpos, SEEK_SET) == -1) { + rc = ErrCode(); + DPRINTF(("lseek: %s", strerror(rc))); + return rc; + } + wres = writev(env->me_fd, iov, n); + } +#endif + if (wres != wsize) { + if (wres < 0) { + rc = ErrCode(); + DPRINTF(("Write error: %s", strerror(rc))); + } else { + rc = EIO; /* TODO: Use which error code? */ + DPUTS("short write, filesystem full?"); + } + return rc; + } + n = 0; + } + if (i > pagecount) + break; + wpos = pos; + wsize = 0; + } + DPRINTF(("committing page %"Z"u", pgno)); + next_pos = pos + size; + iov[n].iov_len = size; + iov[n].iov_base = (char *)dp; + wsize += size; + n++; +#endif /* _WIN32 */ + } + + /* MIPS has cache coherency issues, this is a no-op everywhere else + * Note: for any size >= on-chip cache size, entire on-chip cache is + * flushed. + */ + CACHEFLUSH(env->me_map, txn->mt_next_pgno * env->me_psize, DCACHE); + + for (i = keep; ++i <= pagecount; ) { + dp = dl[i].mptr; + /* This is a page we skipped above */ + if (!dl[i].mid) { + dl[++j] = dl[i]; + dl[j].mid = dp->mp_pgno; + continue; + } + mdb_dpage_free(env, dp); + } + +done: + i--; + txn->mt_dirty_room += i - j; + dl[0].mid = j; + return MDB_SUCCESS; +} + +int +mdb_txn_commit(MDB_txn *txn) +{ + int rc; + unsigned int i; + MDB_env *env; + + if (txn == NULL || txn->mt_env == NULL) + return EINVAL; + + if (txn->mt_child) { + rc = mdb_txn_commit(txn->mt_child); + txn->mt_child = NULL; + if (rc) + goto fail; + } + + env = txn->mt_env; + + if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) { + mdb_dbis_update(txn, 1); + txn->mt_numdbs = 2; /* so txn_abort() doesn't close any new handles */ + mdb_txn_abort(txn); + return MDB_SUCCESS; + } + + if (F_ISSET(txn->mt_flags, MDB_TXN_ERROR)) { + DPUTS("error flag is set, can't commit"); + if (txn->mt_parent) + txn->mt_parent->mt_flags |= MDB_TXN_ERROR; + rc = MDB_BAD_TXN; + goto fail; + } + + if (txn->mt_parent) { + MDB_txn *parent = txn->mt_parent; + MDB_page **lp; + MDB_ID2L dst, src; + MDB_IDL pspill; + unsigned x, y, len, ps_len; + + /* Append our free list to parent's */ + rc = mdb_midl_append_list(&parent->mt_free_pgs, txn->mt_free_pgs); + if (rc) + goto fail; + mdb_midl_free(txn->mt_free_pgs); + /* Failures after this must either undo the changes + * to the parent or set MDB_TXN_ERROR in the parent. + */ + + parent->mt_next_pgno = txn->mt_next_pgno; + parent->mt_flags = txn->mt_flags; + + /* Merge our cursors into parent's and close them */ + mdb_cursors_close(txn, 1); + + /* Update parent's DB table. */ + memcpy(parent->mt_dbs, txn->mt_dbs, txn->mt_numdbs * sizeof(MDB_db)); + parent->mt_numdbs = txn->mt_numdbs; + parent->mt_dbflags[0] = txn->mt_dbflags[0]; + parent->mt_dbflags[1] = txn->mt_dbflags[1]; + for (i=2; i<txn->mt_numdbs; i++) { + /* preserve parent's DB_NEW status */ + x = parent->mt_dbflags[i] & DB_NEW; + parent->mt_dbflags[i] = txn->mt_dbflags[i] | x; + } + + dst = parent->mt_u.dirty_list; + src = txn->mt_u.dirty_list; + /* Remove anything in our dirty list from parent's spill list */ + if ((pspill = parent->mt_spill_pgs) && (ps_len = pspill[0])) { + x = y = ps_len; + pspill[0] = (pgno_t)-1; + /* Mark our dirty pages as deleted in parent spill list */ + for (i=0, len=src[0].mid; ++i <= len; ) { + MDB_ID pn = src[i].mid << 1; + while (pn > pspill[x]) + x--; + if (pn == pspill[x]) { + pspill[x] = 1; + y = --x; + } + } + /* Squash deleted pagenums if we deleted any */ + for (x=y; ++x <= ps_len; ) + if (!(pspill[x] & 1)) + pspill[++y] = pspill[x]; + pspill[0] = y; + } + + /* Find len = length of merging our dirty list with parent's */ + x = dst[0].mid; + dst[0].mid = 0; /* simplify loops */ + if (parent->mt_parent) { + len = x + src[0].mid; + y = mdb_mid2l_search(src, dst[x].mid + 1) - 1; + for (i = x; y && i; y--) { + pgno_t yp = src[y].mid; + while (yp < dst[i].mid) + i--; + if (yp == dst[i].mid) { + i--; + len--; + } + } + } else { /* Simplify the above for single-ancestor case */ + len = MDB_IDL_UM_MAX - txn->mt_dirty_room; + } + /* Merge our dirty list with parent's */ + y = src[0].mid; + for (i = len; y; dst[i--] = src[y--]) { + pgno_t yp = src[y].mid; + while (yp < dst[x].mid) + dst[i--] = dst[x--]; + if (yp == dst[x].mid) + free(dst[x--].mptr); + } + mdb_tassert(txn, i == x); + dst[0].mid = len; + free(txn->mt_u.dirty_list); + parent->mt_dirty_room = txn->mt_dirty_room; + if (txn->mt_spill_pgs) { + if (parent->mt_spill_pgs) { + /* TODO: Prevent failure here, so parent does not fail */ + rc = mdb_midl_append_list(&parent->mt_spill_pgs, txn->mt_spill_pgs); + if (rc) + parent->mt_flags |= MDB_TXN_ERROR; + mdb_midl_free(txn->mt_spill_pgs); + mdb_midl_sort(parent->mt_spill_pgs); + } else { + parent->mt_spill_pgs = txn->mt_spill_pgs; + } + } + + /* Append our loose page list to parent's */ + for (lp = &parent->mt_loose_pgs; *lp; lp = &NEXT_LOOSE_PAGE(lp)) + ; + *lp = txn->mt_loose_pgs; + parent->mt_loose_count += txn->mt_loose_count; + + parent->mt_child = NULL; + mdb_midl_free(((MDB_ntxn *)txn)->mnt_pgstate.mf_pghead); + free(txn); + return rc; + } + + if (txn != env->me_txn) { + DPUTS("attempt to commit unknown transaction"); + rc = EINVAL; + goto fail; + } + + mdb_cursors_close(txn, 0); + + if (!txn->mt_u.dirty_list[0].mid && + !(txn->mt_flags & (MDB_TXN_DIRTY|MDB_TXN_SPILLS))) + goto done; + + DPRINTF(("committing txn %"Z"u %p on mdbenv %p, root page %"Z"u", + txn->mt_txnid, (void*)txn, (void*)env, txn->mt_dbs[MAIN_DBI].md_root)); + + /* Update DB root pointers */ + if (txn->mt_numdbs > 2) { + MDB_cursor mc; + MDB_dbi i; + MDB_val data; + data.mv_size = sizeof(MDB_db); + + mdb_cursor_init(&mc, txn, MAIN_DBI, NULL); + for (i = 2; i < txn->mt_numdbs; i++) { + if (txn->mt_dbflags[i] & DB_DIRTY) { + if (TXN_DBI_CHANGED(txn, i)) { + rc = MDB_BAD_DBI; + goto fail; + } + data.mv_data = &txn->mt_dbs[i]; + rc = mdb_cursor_put(&mc, &txn->mt_dbxs[i].md_name, &data, 0); + if (rc) + goto fail; + } + } + } + + rc = mdb_freelist_save(txn); + if (rc) + goto fail; + + mdb_midl_free(env->me_pghead); + env->me_pghead = NULL; + if (mdb_midl_shrink(&txn->mt_free_pgs)) + env->me_free_pgs = txn->mt_free_pgs; + +#if (MDB_DEBUG) > 2 + mdb_audit(txn); +#endif + + if ((rc = mdb_page_flush(txn, 0)) || + (rc = mdb_env_sync(env, 0)) || + (rc = mdb_env_write_meta(txn))) + goto fail; + + /* Free P_LOOSE pages left behind in dirty_list */ + if (!(env->me_flags & MDB_WRITEMAP)) + mdb_dlist_free(txn); + +done: + env->me_pglast = 0; + env->me_txn = NULL; + mdb_dbis_update(txn, 1); + + if (env->me_txns) + UNLOCK_MUTEX_W(env); + if (txn != env->me_txn0) + free(txn); + + return MDB_SUCCESS; + +fail: + mdb_txn_abort(txn); + return rc; +} + +/** Read the environment parameters of a DB environment before + * mapping it into memory. + * @param[in] env the environment handle + * @param[out] meta address of where to store the meta information + * @return 0 on success, non-zero on failure. + */ +static int ESECT +mdb_env_read_header(MDB_env *env, MDB_meta *meta) +{ + MDB_metabuf pbuf; + MDB_page *p; + MDB_meta *m; + int i, rc, off; + enum { Size = sizeof(pbuf) }; + + /* We don't know the page size yet, so use a minimum value. + * Read both meta pages so we can use the latest one. + */ + + for (i=off=0; i<2; i++, off = meta->mm_psize) { +#ifdef _WIN32 + DWORD len; + OVERLAPPED ov; + memset(&ov, 0, sizeof(ov)); + ov.Offset = off; + rc = ReadFile(env->me_fd, &pbuf, Size, &len, &ov) ? (int)len : -1; + if (rc == -1 && ErrCode() == ERROR_HANDLE_EOF) + rc = 0; +#else + rc = pread(env->me_fd, &pbuf, Size, off); +#endif + if (rc != Size) { + if (rc == 0 && off == 0) + return ENOENT; + rc = rc < 0 ? (int) ErrCode() : MDB_INVALID; + DPRINTF(("read: %s", mdb_strerror(rc))); + return rc; + } + + p = (MDB_page *)&pbuf; + + if (!F_ISSET(p->mp_flags, P_META)) { + DPRINTF(("page %"Z"u not a meta page", p->mp_pgno)); + return MDB_INVALID; + } + + m = METADATA(p); + if (m->mm_magic != MDB_MAGIC) { + DPUTS("meta has invalid magic"); + return MDB_INVALID; + } + + if (m->mm_version != MDB_DATA_VERSION) { + DPRINTF(("database is version %u, expected version %u", + m->mm_version, MDB_DATA_VERSION)); + return MDB_VERSION_MISMATCH; + } + + if (off == 0 || m->mm_txnid > meta->mm_txnid) + *meta = *m; + } + return 0; +} + +static void ESECT +mdb_env_init_meta0(MDB_env *env, MDB_meta *meta) +{ + meta->mm_magic = MDB_MAGIC; + meta->mm_version = MDB_DATA_VERSION; + meta->mm_mapsize = env->me_mapsize; + meta->mm_psize = env->me_psize; + meta->mm_last_pg = 1; + meta->mm_flags = env->me_flags & 0xffff; + meta->mm_flags |= MDB_INTEGERKEY; + meta->mm_dbs[0].md_root = P_INVALID; + meta->mm_dbs[1].md_root = P_INVALID; +} + +/** Write the environment parameters of a freshly created DB environment. + * @param[in] env the environment handle + * @param[out] meta address of where to store the meta information + * @return 0 on success, non-zero on failure. + */ +static int ESECT +mdb_env_init_meta(MDB_env *env, MDB_meta *meta) +{ + MDB_page *p, *q; + int rc; + unsigned int psize; +#ifdef _WIN32 + DWORD len; + OVERLAPPED ov; + memset(&ov, 0, sizeof(ov)); +#define DO_PWRITE(rc, fd, ptr, size, len, pos) do { \ + ov.Offset = pos; \ + rc = WriteFile(fd, ptr, size, &len, &ov); } while(0) +#else + int len; +#define DO_PWRITE(rc, fd, ptr, size, len, pos) do { \ + len = pwrite(fd, ptr, size, pos); \ + rc = (len >= 0); } while(0) +#endif + + DPUTS("writing new meta page"); + + psize = env->me_psize; + + mdb_env_init_meta0(env, meta); + + p = calloc(2, psize); + p->mp_pgno = 0; + p->mp_flags = P_META; + *(MDB_meta *)METADATA(p) = *meta; + + q = (MDB_page *)((char *)p + psize); + q->mp_pgno = 1; + q->mp_flags = P_META; + *(MDB_meta *)METADATA(q) = *meta; + + DO_PWRITE(rc, env->me_fd, p, psize * 2, len, 0); + if (!rc) + rc = ErrCode(); + else if ((unsigned) len == psize * 2) + rc = MDB_SUCCESS; + else + rc = ENOSPC; + free(p); + return rc; +} + +/** Update the environment info to commit a transaction. + * @param[in] txn the transaction that's being committed + * @return 0 on success, non-zero on failure. + */ +static int +mdb_env_write_meta(MDB_txn *txn) +{ + MDB_env *env; + MDB_meta meta, metab, *mp; + size_t mapsize; + off_t off; + int rc, len, toggle; + char *ptr; + HANDLE mfd; +#ifdef _WIN32 + OVERLAPPED ov; +#else + int r2; +#endif + + toggle = txn->mt_txnid & 1; + DPRINTF(("writing meta page %d for root page %"Z"u", + toggle, txn->mt_dbs[MAIN_DBI].md_root)); + + env = txn->mt_env; + mp = env->me_metas[toggle]; + mapsize = env->me_metas[toggle ^ 1]->mm_mapsize; + /* Persist any increases of mapsize config */ + if (mapsize < env->me_mapsize) + mapsize = env->me_mapsize; + + if (env->me_flags & MDB_WRITEMAP) { + mp->mm_mapsize = mapsize; + mp->mm_dbs[0] = txn->mt_dbs[0]; + mp->mm_dbs[1] = txn->mt_dbs[1]; + mp->mm_last_pg = txn->mt_next_pgno - 1; + mp->mm_txnid = txn->mt_txnid; + if (!(env->me_flags & (MDB_NOMETASYNC|MDB_NOSYNC))) { + unsigned meta_size = env->me_psize; + rc = (env->me_flags & MDB_MAPASYNC) ? MS_ASYNC : MS_SYNC; + ptr = env->me_map; + if (toggle) { +#ifndef _WIN32 /* POSIX msync() requires ptr = start of OS page */ + if (meta_size < env->me_os_psize) + meta_size += meta_size; + else +#endif + ptr += meta_size; + } + if (MDB_MSYNC(ptr, meta_size, rc)) { + rc = ErrCode(); + goto fail; + } + } + goto done; + } + metab.mm_txnid = env->me_metas[toggle]->mm_txnid; + metab.mm_last_pg = env->me_metas[toggle]->mm_last_pg; + + meta.mm_mapsize = mapsize; + meta.mm_dbs[0] = txn->mt_dbs[0]; + meta.mm_dbs[1] = txn->mt_dbs[1]; + meta.mm_last_pg = txn->mt_next_pgno - 1; + meta.mm_txnid = txn->mt_txnid; + + off = offsetof(MDB_meta, mm_mapsize); + ptr = (char *)&meta + off; + len = sizeof(MDB_meta) - off; + if (toggle) + off += env->me_psize; + off += PAGEHDRSZ; + + /* Write to the SYNC fd */ + mfd = env->me_flags & (MDB_NOSYNC|MDB_NOMETASYNC) ? + env->me_fd : env->me_mfd; +#ifdef _WIN32 + { + memset(&ov, 0, sizeof(ov)); + ov.Offset = off; + if (!WriteFile(mfd, ptr, len, (DWORD *)&rc, &ov)) + rc = -1; + } +#else + rc = pwrite(mfd, ptr, len, off); +#endif + if (rc != len) { + rc = rc < 0 ? ErrCode() : EIO; + DPUTS("write failed, disk error?"); + /* On a failure, the pagecache still contains the new data. + * Write some old data back, to prevent it from being used. + * Use the non-SYNC fd; we know it will fail anyway. + */ + meta.mm_last_pg = metab.mm_last_pg; + meta.mm_txnid = metab.mm_txnid; +#ifdef _WIN32 + memset(&ov, 0, sizeof(ov)); + ov.Offset = off; + WriteFile(env->me_fd, ptr, len, NULL, &ov); +#else + r2 = pwrite(env->me_fd, ptr, len, off); + (void)r2; /* Silence warnings. We don't care about pwrite's return value */ +#endif +fail: + env->me_flags |= MDB_FATAL_ERROR; + return rc; + } + /* MIPS has cache coherency issues, this is a no-op everywhere else */ + CACHEFLUSH(env->me_map + off, len, DCACHE); +done: + /* Memory ordering issues are irrelevant; since the entire writer + * is wrapped by wmutex, all of these changes will become visible + * after the wmutex is unlocked. Since the DB is multi-version, + * readers will get consistent data regardless of how fresh or + * how stale their view of these values is. + */ + if (env->me_txns) + env->me_txns->mti_txnid = txn->mt_txnid; + + return MDB_SUCCESS; +} + +/** Check both meta pages to see which one is newer. + * @param[in] env the environment handle + * @return meta toggle (0 or 1). + */ +static int +mdb_env_pick_meta(const MDB_env *env) +{ + return (env->me_metas[0]->mm_txnid < env->me_metas[1]->mm_txnid); +} + +int ESECT +mdb_env_create(MDB_env **env) +{ + MDB_env *e; + + e = calloc(1, sizeof(MDB_env)); + if (!e) + return ENOMEM; + + e->me_maxreaders = DEFAULT_READERS; + e->me_maxdbs = e->me_numdbs = 2; + e->me_fd = INVALID_HANDLE_VALUE; + e->me_lfd = INVALID_HANDLE_VALUE; + e->me_mfd = INVALID_HANDLE_VALUE; +#ifdef MDB_USE_POSIX_SEM + e->me_rmutex = SEM_FAILED; + e->me_wmutex = SEM_FAILED; +#endif + e->me_pid = getpid(); + GET_PAGESIZE(e->me_os_psize); + VGMEMP_CREATE(e,0,0); + *env = e; + return MDB_SUCCESS; +} + +static int ESECT +mdb_env_map(MDB_env *env, void *addr) +{ + MDB_page *p; + unsigned int flags = env->me_flags; +#ifdef _WIN32 + int rc; + HANDLE mh; + LONG sizelo, sizehi; + size_t msize; + + if (flags & MDB_RDONLY) { + /* Don't set explicit map size, use whatever exists */ + msize = 0; + sizelo = 0; + sizehi = 0; + } else { + msize = env->me_mapsize; + sizelo = msize & 0xffffffff; + sizehi = msize >> 16 >> 16; /* only needed on Win64 */ + + /* Windows won't create mappings for zero length files. + * and won't map more than the file size. + * Just set the maxsize right now. + */ + if (SetFilePointer(env->me_fd, sizelo, &sizehi, 0) != (DWORD)sizelo + || !SetEndOfFile(env->me_fd) + || SetFilePointer(env->me_fd, 0, NULL, 0) != 0) + return ErrCode(); + } + + mh = CreateFileMapping(env->me_fd, NULL, flags & MDB_WRITEMAP ? + PAGE_READWRITE : PAGE_READONLY, + sizehi, sizelo, NULL); + if (!mh) + return ErrCode(); + env->me_map = MapViewOfFileEx(mh, flags & MDB_WRITEMAP ? + FILE_MAP_WRITE : FILE_MAP_READ, + 0, 0, msize, addr); + rc = env->me_map ? 0 : ErrCode(); + CloseHandle(mh); + if (rc) + return rc; +#else + int prot = PROT_READ; + if (flags & MDB_WRITEMAP) { + prot |= PROT_WRITE; + if (ftruncate(env->me_fd, env->me_mapsize) < 0) + return ErrCode(); + } + env->me_map = mmap(addr, env->me_mapsize, prot, MAP_SHARED, + env->me_fd, 0); + if (env->me_map == MAP_FAILED) { + env->me_map = NULL; + return ErrCode(); + } + + if (flags & MDB_NORDAHEAD) { + /* Turn off readahead. It's harmful when the DB is larger than RAM. */ +#ifdef MADV_RANDOM + madvise(env->me_map, env->me_mapsize, MADV_RANDOM); +#else +#ifdef POSIX_MADV_RANDOM + posix_madvise(env->me_map, env->me_mapsize, POSIX_MADV_RANDOM); +#endif /* POSIX_MADV_RANDOM */ +#endif /* MADV_RANDOM */ + } +#endif /* _WIN32 */ + + /* Can happen because the address argument to mmap() is just a + * hint. mmap() can pick another, e.g. if the range is in use. + * The MAP_FIXED flag would prevent that, but then mmap could + * instead unmap existing pages to make room for the new map. + */ + if (addr && env->me_map != addr) + return EBUSY; /* TODO: Make a new MDB_* error code? */ + + p = (MDB_page *)env->me_map; + env->me_metas[0] = METADATA(p); + env->me_metas[1] = (MDB_meta *)((char *)env->me_metas[0] + env->me_psize); + + return MDB_SUCCESS; +} + +int ESECT +mdb_env_set_mapsize(MDB_env *env, size_t size) +{ + /* If env is already open, caller is responsible for making + * sure there are no active txns. + */ + if (env->me_map) { + int rc; + void *old; + if (env->me_txn) + return EINVAL; + if (!size) + size = env->me_metas[mdb_env_pick_meta(env)]->mm_mapsize; + else if (size < env->me_mapsize) { + /* If the configured size is smaller, make sure it's + * still big enough. Silently round up to minimum if not. + */ + size_t minsize = (env->me_metas[mdb_env_pick_meta(env)]->mm_last_pg + 1) * env->me_psize; + if (size < minsize) + size = minsize; + } + munmap(env->me_map, env->me_mapsize); + env->me_mapsize = size; + old = (env->me_flags & MDB_FIXEDMAP) ? env->me_map : NULL; + rc = mdb_env_map(env, old); + if (rc) + return rc; + } + env->me_mapsize = size; + if (env->me_psize) + env->me_maxpg = env->me_mapsize / env->me_psize; + return MDB_SUCCESS; +} + +int ESECT +mdb_env_set_maxdbs(MDB_env *env, MDB_dbi dbs) +{ + if (env->me_map) + return EINVAL; + env->me_maxdbs = dbs + 2; /* Named databases + main and free DB */ + return MDB_SUCCESS; +} + +int ESECT +mdb_env_set_maxreaders(MDB_env *env, unsigned int readers) +{ + if (env->me_map || readers < 1) + return EINVAL; + env->me_maxreaders = readers; + return MDB_SUCCESS; +} + +int ESECT +mdb_env_get_maxreaders(MDB_env *env, unsigned int *readers) +{ + if (!env || !readers) + return EINVAL; + *readers = env->me_maxreaders; + return MDB_SUCCESS; +} + +static int ESECT +mdb_fsize(HANDLE fd, size_t *size) +{ +#ifdef _WIN32 + LARGE_INTEGER fsize; + + if (!GetFileSizeEx(fd, &fsize)) + return ErrCode(); + + *size = fsize.QuadPart; +#else + struct stat st; + + if (fstat(fd, &st)) + return ErrCode(); + + *size = st.st_size; +#endif + return MDB_SUCCESS; +} + +#ifdef BROKEN_FDATASYNC +#include <sys/utsname.h> +#include <sys/vfs.h> +#endif + +/** Further setup required for opening an LMDB environment + */ +static int ESECT +mdb_env_open2(MDB_env *env) +{ + unsigned int flags = env->me_flags; + int i, newenv = 0, rc; + MDB_meta meta; + +#ifdef _WIN32 + /* See if we should use QueryLimited */ + rc = GetVersion(); + if ((rc & 0xff) > 5) + env->me_pidquery = MDB_PROCESS_QUERY_LIMITED_INFORMATION; + else + env->me_pidquery = PROCESS_QUERY_INFORMATION; +#endif /* _WIN32 */ +#ifdef BROKEN_FDATASYNC + /* ext3/ext4 fdatasync is broken on some older Linux kernels. + * https://lkml.org/lkml/2012/9/3/83 + * Kernels after 3.6-rc6 are known good. + * https://lkml.org/lkml/2012/9/10/556 + * See if the DB is on ext3/ext4, then check for new enough kernel + * Kernels 2.6.32.60, 2.6.34.15, 3.2.30, and 3.5.4 are also known + * to be patched. + */ + { + struct statfs st; + fstatfs(env->me_fd, &st); + while (st.f_type == 0xEF53) { + struct utsname uts; + int i; + uname(&uts); + if (uts.release[0] < '3') { + if (!strncmp(uts.release, "2.6.32.", 7)) { + i = atoi(uts.release+7); + if (i >= 60) + break; /* 2.6.32.60 and newer is OK */ + } else if (!strncmp(uts.release, "2.6.34.", 7)) { + i = atoi(uts.release+7); + if (i >= 15) + break; /* 2.6.34.15 and newer is OK */ + } + } else if (uts.release[0] == '3') { + i = atoi(uts.release+2); + if (i > 5) + break; /* 3.6 and newer is OK */ + if (i == 5) { + i = atoi(uts.release+4); + if (i >= 4) + break; /* 3.5.4 and newer is OK */ + } else if (i == 2) { + i = atoi(uts.release+4); + if (i >= 30) + break; /* 3.2.30 and newer is OK */ + } + } else { /* 4.x and newer is OK */ + break; + } + env->me_flags |= MDB_FSYNCONLY; + break; + } + } +#endif + + memset(&meta, 0, sizeof(meta)); + + if ((i = mdb_env_read_header(env, &meta)) != 0) { + if (i != ENOENT) + return i; + DPUTS("new mdbenv"); + newenv = 1; + env->me_psize = env->me_os_psize; + if (env->me_psize > MAX_PAGESIZE) + env->me_psize = MAX_PAGESIZE; + } else { + env->me_psize = meta.mm_psize; + } + + /* Was a mapsize configured? */ + if (!env->me_mapsize) { + /* If this is a new environment, take the default, + * else use the size recorded in the existing env. + */ + env->me_mapsize = newenv ? DEFAULT_MAPSIZE : meta.mm_mapsize; + } else if (env->me_mapsize < meta.mm_mapsize) { + /* If the configured size is smaller, make sure it's + * still big enough. Silently round up to minimum if not. + */ + size_t minsize = (meta.mm_last_pg + 1) * meta.mm_psize; + if (env->me_mapsize < minsize) + env->me_mapsize = minsize; + } + + rc = mdb_env_map(env, (flags & MDB_FIXEDMAP) ? meta.mm_address : NULL); + if (rc) + return rc; + + if (newenv) { + if (flags & MDB_FIXEDMAP) + meta.mm_address = env->me_map; + i = mdb_env_init_meta(env, &meta); + if (i != MDB_SUCCESS) { + return i; + } + } + + env->me_maxfree_1pg = (env->me_psize - PAGEHDRSZ) / sizeof(pgno_t) - 1; + env->me_nodemax = (((env->me_psize - PAGEHDRSZ) / MDB_MINKEYS) & -2) + - sizeof(indx_t); +#if !(MDB_MAXKEYSIZE) + env->me_maxkey = env->me_nodemax - (NODESIZE + sizeof(MDB_db)); +#endif + env->me_maxpg = env->me_mapsize / env->me_psize; + +#if MDB_DEBUG + { + int toggle = mdb_env_pick_meta(env); + MDB_db *db = &env->me_metas[toggle]->mm_dbs[MAIN_DBI]; + + DPRINTF(("opened database version %u, pagesize %u", + env->me_metas[0]->mm_version, env->me_psize)); + DPRINTF(("using meta page %d", toggle)); + DPRINTF(("depth: %u", db->md_depth)); + DPRINTF(("entries: %"Z"u", db->md_entries)); + DPRINTF(("branch pages: %"Z"u", db->md_branch_pages)); + DPRINTF(("leaf pages: %"Z"u", db->md_leaf_pages)); + DPRINTF(("overflow pages: %"Z"u", db->md_overflow_pages)); + DPRINTF(("root: %"Z"u", db->md_root)); + } +#endif + + return MDB_SUCCESS; +} + + +/** Release a reader thread's slot in the reader lock table. + * This function is called automatically when a thread exits. + * @param[in] ptr This points to the slot in the reader lock table. + */ +static void +mdb_env_reader_dest(void *ptr) +{ + MDB_reader *reader = ptr; + + reader->mr_pid = 0; +} + +#ifdef _WIN32 +/** Junk for arranging thread-specific callbacks on Windows. This is + * necessarily platform and compiler-specific. Windows supports up + * to 1088 keys. Let's assume nobody opens more than 64 environments + * in a single process, for now. They can override this if needed. + */ +#ifndef MAX_TLS_KEYS +#define MAX_TLS_KEYS 64 +#endif +static pthread_key_t mdb_tls_keys[MAX_TLS_KEYS]; +static int mdb_tls_nkeys; + +static void NTAPI mdb_tls_callback(PVOID module, DWORD reason, PVOID ptr) +{ + int i; + switch(reason) { + case DLL_PROCESS_ATTACH: break; + case DLL_THREAD_ATTACH: break; + case DLL_THREAD_DETACH: + for (i=0; i<mdb_tls_nkeys; i++) { + MDB_reader *r = pthread_getspecific(mdb_tls_keys[i]); + if (r) { + mdb_env_reader_dest(r); + } + } + break; + case DLL_PROCESS_DETACH: break; + } +} +#ifdef __GNUC__ +#ifdef _WIN64 +const PIMAGE_TLS_CALLBACK mdb_tls_cbp __attribute__((section (".CRT$XLB"))) = mdb_tls_callback; +#else +PIMAGE_TLS_CALLBACK mdb_tls_cbp __attribute__((section (".CRT$XLB"))) = mdb_tls_callback; +#endif +#else +#ifdef _WIN64 +/* Force some symbol references. + * _tls_used forces the linker to create the TLS directory if not already done + * mdb_tls_cbp prevents whole-program-optimizer from dropping the symbol. + */ +#pragma comment(linker, "/INCLUDE:_tls_used") +#pragma comment(linker, "/INCLUDE:mdb_tls_cbp") +#pragma const_seg(".CRT$XLB") +extern const PIMAGE_TLS_CALLBACK mdb_tls_cbp; +const PIMAGE_TLS_CALLBACK mdb_tls_cbp = mdb_tls_callback; +#pragma const_seg() +#else /* _WIN32 */ +#pragma comment(linker, "/INCLUDE:__tls_used") +#pragma comment(linker, "/INCLUDE:_mdb_tls_cbp") +#pragma data_seg(".CRT$XLB") +PIMAGE_TLS_CALLBACK mdb_tls_cbp = mdb_tls_callback; +#pragma data_seg() +#endif /* WIN 32/64 */ +#endif /* !__GNUC__ */ +#endif + +/** Downgrade the exclusive lock on the region back to shared */ +static int ESECT +mdb_env_share_locks(MDB_env *env, int *excl) +{ + int rc = 0, toggle = mdb_env_pick_meta(env); + + env->me_txns->mti_txnid = env->me_metas[toggle]->mm_txnid; + +#ifdef _WIN32 + { + OVERLAPPED ov; + /* First acquire a shared lock. The Unlock will + * then release the existing exclusive lock. + */ + memset(&ov, 0, sizeof(ov)); + if (!LockFileEx(env->me_lfd, 0, 0, 1, 0, &ov)) { + rc = ErrCode(); + } else { + UnlockFile(env->me_lfd, 0, 0, 1, 0); + *excl = 0; + } + } +#else + { + struct flock lock_info; + /* The shared lock replaces the existing lock */ + memset((void *)&lock_info, 0, sizeof(lock_info)); + lock_info.l_type = F_RDLCK; + lock_info.l_whence = SEEK_SET; + lock_info.l_start = 0; + lock_info.l_len = 1; + while ((rc = fcntl(env->me_lfd, F_SETLK, &lock_info)) && + (rc = ErrCode()) == EINTR) ; + *excl = rc ? -1 : 0; /* error may mean we lost the lock */ + } +#endif + + return rc; +} + +/** Try to get exclusive lock, otherwise shared. + * Maintain *excl = -1: no/unknown lock, 0: shared, 1: exclusive. + */ +static int ESECT +mdb_env_excl_lock(MDB_env *env, int *excl) +{ + int rc = 0; +#ifdef _WIN32 + if (LockFile(env->me_lfd, 0, 0, 1, 0)) { + *excl = 1; + } else { + OVERLAPPED ov; + memset(&ov, 0, sizeof(ov)); + if (LockFileEx(env->me_lfd, 0, 0, 1, 0, &ov)) { + *excl = 0; + } else { + rc = ErrCode(); + } + } +#else + struct flock lock_info; + memset((void *)&lock_info, 0, sizeof(lock_info)); + lock_info.l_type = F_WRLCK; + lock_info.l_whence = SEEK_SET; + lock_info.l_start = 0; + lock_info.l_len = 1; + while ((rc = fcntl(env->me_lfd, F_SETLK, &lock_info)) && + (rc = ErrCode()) == EINTR) ; + if (!rc) { + *excl = 1; + } else +# ifdef MDB_USE_POSIX_SEM + if (*excl < 0) /* always true when !MDB_USE_POSIX_SEM */ +# endif + { + lock_info.l_type = F_RDLCK; + while ((rc = fcntl(env->me_lfd, F_SETLKW, &lock_info)) && + (rc = ErrCode()) == EINTR) ; + if (rc == 0) + *excl = 0; + } +#endif + return rc; +} + +#ifdef MDB_USE_HASH +/* + * hash_64 - 64 bit Fowler/Noll/Vo-0 FNV-1a hash code + * + * @(#) $Revision: 5.1 $ + * @(#) $Id: hash_64a.c,v 5.1 2009/06/30 09:01:38 chongo Exp $ + * @(#) $Source: /usr/local/src/cmd/fnv/RCS/hash_64a.c,v $ + * + * http://www.isthe.com/chongo/tech/comp/fnv/index.html + * + *** + * + * Please do not copyright this code. This code is in the public domain. + * + * LANDON CURT NOLL DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, + * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO + * EVENT SHALL LANDON CURT NOLL BE LIABLE FOR ANY SPECIAL, INDIRECT OR + * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF + * USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR + * OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THIS SOFTWARE. + * + * By: + * chongo <Landon Curt Noll> /\oo/\ + * http://www.isthe.com/chongo/ + * + * Share and Enjoy! :-) + */ + +typedef unsigned long long mdb_hash_t; +#define MDB_HASH_INIT ((mdb_hash_t)0xcbf29ce484222325ULL) + +/** perform a 64 bit Fowler/Noll/Vo FNV-1a hash on a buffer + * @param[in] val value to hash + * @param[in] hval initial value for hash + * @return 64 bit hash + * + * NOTE: To use the recommended 64 bit FNV-1a hash, use MDB_HASH_INIT as the + * hval arg on the first call. + */ +static mdb_hash_t +mdb_hash_val(MDB_val *val, mdb_hash_t hval) +{ + unsigned char *s = (unsigned char *)val->mv_data; /* unsigned string */ + unsigned char *end = s + val->mv_size; + /* + * FNV-1a hash each octet of the string + */ + while (s < end) { + /* xor the bottom with the current octet */ + hval ^= (mdb_hash_t)*s++; + + /* multiply by the 64 bit FNV magic prime mod 2^64 */ + hval += (hval << 1) + (hval << 4) + (hval << 5) + + (hval << 7) + (hval << 8) + (hval << 40); + } + /* return our new hash value */ + return hval; +} + +/** Hash the string and output the encoded hash. + * This uses modified RFC1924 Ascii85 encoding to accommodate systems with + * very short name limits. We don't care about the encoding being reversible, + * we just want to preserve as many bits of the input as possible in a + * small printable string. + * @param[in] str string to hash + * @param[out] encbuf an array of 11 chars to hold the hash + */ +static const char mdb_a85[]= "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~"; + +static void +mdb_pack85(unsigned long l, char *out) +{ + int i; + + for (i=0; i<5; i++) { + *out++ = mdb_a85[l % 85]; + l /= 85; + } +} + +static void +mdb_hash_enc(MDB_val *val, char *encbuf) +{ + mdb_hash_t h = mdb_hash_val(val, MDB_HASH_INIT); + + mdb_pack85(h, encbuf); + mdb_pack85(h>>32, encbuf+5); + encbuf[10] = '\0'; +} +#endif + +/** Open and/or initialize the lock region for the environment. + * @param[in] env The LMDB environment. + * @param[in] lpath The pathname of the file used for the lock region. + * @param[in] mode The Unix permissions for the file, if we create it. + * @param[in,out] excl In -1, out lock type: -1 none, 0 shared, 1 exclusive + * @return 0 on success, non-zero on failure. + */ +static int ESECT +mdb_env_setup_locks(MDB_env *env, char *lpath, int mode, int *excl) +{ +#ifdef _WIN32 +# define MDB_ERRCODE_ROFS ERROR_WRITE_PROTECT +#else +# define MDB_ERRCODE_ROFS EROFS +#ifdef O_CLOEXEC /* Linux: Open file and set FD_CLOEXEC atomically */ +# define MDB_CLOEXEC O_CLOEXEC +#else + int fdflags; +# define MDB_CLOEXEC 0 +#endif +#endif + int rc; + off_t size, rsize; + +#ifdef _WIN32 + env->me_lfd = CreateFile(lpath, GENERIC_READ|GENERIC_WRITE, + FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, NULL); +#else + env->me_lfd = open(lpath, O_RDWR|O_CREAT|MDB_CLOEXEC, mode); +#endif + if (env->me_lfd == INVALID_HANDLE_VALUE) { + rc = ErrCode(); + if (rc == MDB_ERRCODE_ROFS && (env->me_flags & MDB_RDONLY)) { + return MDB_SUCCESS; + } + goto fail_errno; + } +#if ! ((MDB_CLOEXEC) || defined(_WIN32)) + /* Lose record locks when exec*() */ + if ((fdflags = fcntl(env->me_lfd, F_GETFD) | FD_CLOEXEC) >= 0) + fcntl(env->me_lfd, F_SETFD, fdflags); +#endif + + if (!(env->me_flags & MDB_NOTLS)) { + rc = pthread_key_create(&env->me_txkey, mdb_env_reader_dest); + if (rc) + goto fail; + env->me_flags |= MDB_ENV_TXKEY; +#ifdef _WIN32 + /* Windows TLS callbacks need help finding their TLS info. */ + if (mdb_tls_nkeys >= MAX_TLS_KEYS) { + rc = MDB_TLS_FULL; + goto fail; + } + mdb_tls_keys[mdb_tls_nkeys++] = env->me_txkey; +#endif + } + + /* Try to get exclusive lock. If we succeed, then + * nobody is using the lock region and we should initialize it. + */ + if ((rc = mdb_env_excl_lock(env, excl))) goto fail; + +#ifdef _WIN32 + size = GetFileSize(env->me_lfd, NULL); +#else + size = lseek(env->me_lfd, 0, SEEK_END); + if (size == -1) goto fail_errno; +#endif + rsize = (env->me_maxreaders-1) * sizeof(MDB_reader) + sizeof(MDB_txninfo); + if (size < rsize && *excl > 0) { +#ifdef _WIN32 + if (SetFilePointer(env->me_lfd, rsize, NULL, FILE_BEGIN) != (DWORD)rsize + || !SetEndOfFile(env->me_lfd)) + goto fail_errno; +#else + if (ftruncate(env->me_lfd, rsize) != 0) goto fail_errno; +#endif + } else { + rsize = size; + size = rsize - sizeof(MDB_txninfo); + env->me_maxreaders = size/sizeof(MDB_reader) + 1; + } + { +#ifdef _WIN32 + HANDLE mh; + mh = CreateFileMapping(env->me_lfd, NULL, PAGE_READWRITE, + 0, 0, NULL); + if (!mh) goto fail_errno; + env->me_txns = MapViewOfFileEx(mh, FILE_MAP_WRITE, 0, 0, rsize, NULL); + CloseHandle(mh); + if (!env->me_txns) goto fail_errno; +#else + void *m = mmap(NULL, rsize, PROT_READ|PROT_WRITE, MAP_SHARED, + env->me_lfd, 0); + if (m == MAP_FAILED) goto fail_errno; + env->me_txns = m; +#endif + } + if (*excl > 0) { +#ifdef _WIN32 + BY_HANDLE_FILE_INFORMATION stbuf; + struct { + DWORD volume; + DWORD nhigh; + DWORD nlow; + } idbuf; + MDB_val val; + char encbuf[11]; + + if (!mdb_sec_inited) { + InitializeSecurityDescriptor(&mdb_null_sd, + SECURITY_DESCRIPTOR_REVISION); + SetSecurityDescriptorDacl(&mdb_null_sd, TRUE, 0, FALSE); + mdb_all_sa.nLength = sizeof(SECURITY_ATTRIBUTES); + mdb_all_sa.bInheritHandle = FALSE; + mdb_all_sa.lpSecurityDescriptor = &mdb_null_sd; + mdb_sec_inited = 1; + } + if (!GetFileInformationByHandle(env->me_lfd, &stbuf)) goto fail_errno; + idbuf.volume = stbuf.dwVolumeSerialNumber; + idbuf.nhigh = stbuf.nFileIndexHigh; + idbuf.nlow = stbuf.nFileIndexLow; + val.mv_data = &idbuf; + val.mv_size = sizeof(idbuf); + mdb_hash_enc(&val, encbuf); + sprintf(env->me_txns->mti_rmname, "Global\\MDBr%s", encbuf); + sprintf(env->me_txns->mti_wmname, "Global\\MDBw%s", encbuf); + env->me_rmutex = CreateMutex(&mdb_all_sa, FALSE, env->me_txns->mti_rmname); + if (!env->me_rmutex) goto fail_errno; + env->me_wmutex = CreateMutex(&mdb_all_sa, FALSE, env->me_txns->mti_wmname); + if (!env->me_wmutex) goto fail_errno; +#elif defined(MDB_USE_POSIX_SEM) + struct stat stbuf; + struct { + dev_t dev; + ino_t ino; + } idbuf; + MDB_val val; + char encbuf[11]; + +#if defined(__NetBSD__) +#define MDB_SHORT_SEMNAMES 1 /* limited to 14 chars */ +#endif + if (fstat(env->me_lfd, &stbuf)) goto fail_errno; + idbuf.dev = stbuf.st_dev; + idbuf.ino = stbuf.st_ino; + val.mv_data = &idbuf; + val.mv_size = sizeof(idbuf); + mdb_hash_enc(&val, encbuf); +#ifdef MDB_SHORT_SEMNAMES + encbuf[9] = '\0'; /* drop name from 15 chars to 14 chars */ +#endif + sprintf(env->me_txns->mti_rmname, "/MDBr%s", encbuf); + sprintf(env->me_txns->mti_wmname, "/MDBw%s", encbuf); + /* Clean up after a previous run, if needed: Try to + * remove both semaphores before doing anything else. + */ + sem_unlink(env->me_txns->mti_rmname); + sem_unlink(env->me_txns->mti_wmname); + env->me_rmutex = sem_open(env->me_txns->mti_rmname, + O_CREAT|O_EXCL, mode, 1); + if (env->me_rmutex == SEM_FAILED) goto fail_errno; + env->me_wmutex = sem_open(env->me_txns->mti_wmname, + O_CREAT|O_EXCL, mode, 1); + if (env->me_wmutex == SEM_FAILED) goto fail_errno; +#else /* MDB_USE_POSIX_SEM */ + pthread_mutexattr_t mattr; + + if ((rc = pthread_mutexattr_init(&mattr)) + || (rc = pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED)) + || (rc = pthread_mutex_init(&env->me_txns->mti_mutex, &mattr)) + || (rc = pthread_mutex_init(&env->me_txns->mti_wmutex, &mattr))) + goto fail; + pthread_mutexattr_destroy(&mattr); +#endif /* _WIN32 || MDB_USE_POSIX_SEM */ + + env->me_txns->mti_magic = MDB_MAGIC; + env->me_txns->mti_format = MDB_LOCK_FORMAT; + env->me_txns->mti_txnid = 0; + env->me_txns->mti_numreaders = 0; + + } else { + if (env->me_txns->mti_magic != MDB_MAGIC) { + DPUTS("lock region has invalid magic"); + rc = MDB_INVALID; + goto fail; + } + if (env->me_txns->mti_format != MDB_LOCK_FORMAT) { + DPRINTF(("lock region has format+version 0x%x, expected 0x%x", + env->me_txns->mti_format, MDB_LOCK_FORMAT)); + rc = MDB_VERSION_MISMATCH; + goto fail; + } + rc = ErrCode(); + if (rc && rc != EACCES && rc != EAGAIN) { + goto fail; + } +#ifdef _WIN32 + env->me_rmutex = OpenMutex(SYNCHRONIZE, FALSE, env->me_txns->mti_rmname); + if (!env->me_rmutex) goto fail_errno; + env->me_wmutex = OpenMutex(SYNCHRONIZE, FALSE, env->me_txns->mti_wmname); + if (!env->me_wmutex) goto fail_errno; +#elif defined(MDB_USE_POSIX_SEM) + env->me_rmutex = sem_open(env->me_txns->mti_rmname, 0); + if (env->me_rmutex == SEM_FAILED) goto fail_errno; + env->me_wmutex = sem_open(env->me_txns->mti_wmname, 0); + if (env->me_wmutex == SEM_FAILED) goto fail_errno; +#endif + } + return MDB_SUCCESS; + +fail_errno: + rc = ErrCode(); +fail: + return rc; +} + + /** The name of the lock file in the DB environment */ +#define LOCKNAME "/lock.mdb" + /** The name of the data file in the DB environment */ +#define DATANAME "/data.mdb" + /** The suffix of the lock file when no subdir is used */ +#define LOCKSUFF "-lock" + /** Only a subset of the @ref mdb_env flags can be changed + * at runtime. Changing other flags requires closing the + * environment and re-opening it with the new flags. + */ +#define CHANGEABLE (MDB_NOSYNC|MDB_NOMETASYNC|MDB_MAPASYNC|MDB_NOMEMINIT) +#define CHANGELESS (MDB_FIXEDMAP|MDB_NOSUBDIR|MDB_RDONLY|MDB_WRITEMAP| \ + MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD) + +#if VALID_FLAGS & PERSISTENT_FLAGS & (CHANGEABLE|CHANGELESS) +# error "Persistent DB flags & env flags overlap, but both go in mm_flags" +#endif + +int ESECT +mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode) +{ + int oflags, rc, len, excl = -1; + char *lpath, *dpath; + + if (env->me_fd!=INVALID_HANDLE_VALUE || (flags & ~(CHANGEABLE|CHANGELESS))) + return EINVAL; + + len = strlen(path); + if (flags & MDB_NOSUBDIR) { + rc = len + sizeof(LOCKSUFF) + len + 1; + } else { + rc = len + sizeof(LOCKNAME) + len + sizeof(DATANAME); + } + lpath = malloc(rc); + if (!lpath) + return ENOMEM; + if (flags & MDB_NOSUBDIR) { + dpath = lpath + len + sizeof(LOCKSUFF); + sprintf(lpath, "%s" LOCKSUFF, path); + strcpy(dpath, path); + } else { + dpath = lpath + len + sizeof(LOCKNAME); + sprintf(lpath, "%s" LOCKNAME, path); + sprintf(dpath, "%s" DATANAME, path); + } + + rc = MDB_SUCCESS; + flags |= env->me_flags; + if (flags & MDB_RDONLY) { + /* silently ignore WRITEMAP when we're only getting read access */ + flags &= ~MDB_WRITEMAP; + } else { + if (!((env->me_free_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX)) && + (env->me_dirty_list = calloc(MDB_IDL_UM_SIZE, sizeof(MDB_ID2))))) + rc = ENOMEM; + } + env->me_flags = flags |= MDB_ENV_ACTIVE; + if (rc) + goto leave; + + env->me_path = strdup(path); + env->me_dbxs = calloc(env->me_maxdbs, sizeof(MDB_dbx)); + env->me_dbflags = calloc(env->me_maxdbs, sizeof(uint16_t)); + env->me_dbiseqs = calloc(env->me_maxdbs, sizeof(unsigned int)); + if (!(env->me_dbxs && env->me_path && env->me_dbflags && env->me_dbiseqs)) { + rc = ENOMEM; + goto leave; + } + + /* For RDONLY, get lockfile after we know datafile exists */ + if (!(flags & (MDB_RDONLY|MDB_NOLOCK))) { + rc = mdb_env_setup_locks(env, lpath, mode, &excl); + if (rc) + goto leave; + } + +#ifdef _WIN32 + if (F_ISSET(flags, MDB_RDONLY)) { + oflags = GENERIC_READ; + len = OPEN_EXISTING; + } else { + oflags = GENERIC_READ|GENERIC_WRITE; + len = OPEN_ALWAYS; + } + mode = FILE_ATTRIBUTE_NORMAL; + env->me_fd = CreateFile(dpath, oflags, FILE_SHARE_READ|FILE_SHARE_WRITE, + NULL, len, mode, NULL); +#else + if (F_ISSET(flags, MDB_RDONLY)) + oflags = O_RDONLY; + else + oflags = O_RDWR | O_CREAT; + + env->me_fd = open(dpath, oflags, mode); +#endif + if (env->me_fd == INVALID_HANDLE_VALUE) { + rc = ErrCode(); + goto leave; + } + + if ((flags & (MDB_RDONLY|MDB_NOLOCK)) == MDB_RDONLY) { + rc = mdb_env_setup_locks(env, lpath, mode, &excl); + if (rc) + goto leave; + } + + if ((rc = mdb_env_open2(env)) == MDB_SUCCESS) { + if (flags & (MDB_RDONLY|MDB_WRITEMAP)) { + env->me_mfd = env->me_fd; + } else { + /* Synchronous fd for meta writes. Needed even with + * MDB_NOSYNC/MDB_NOMETASYNC, in case these get reset. + */ +#ifdef _WIN32 + len = OPEN_EXISTING; + env->me_mfd = CreateFile(dpath, oflags, + FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, len, + mode | FILE_FLAG_WRITE_THROUGH, NULL); +#else + oflags &= ~O_CREAT; + env->me_mfd = open(dpath, oflags | MDB_DSYNC, mode); +#endif + if (env->me_mfd == INVALID_HANDLE_VALUE) { + rc = ErrCode(); + goto leave; + } + } + DPRINTF(("opened dbenv %p", (void *) env)); + if (excl > 0) { + rc = mdb_env_share_locks(env, &excl); + if (rc) + goto leave; + } + if (!((flags & MDB_RDONLY) || + (env->me_pbuf = calloc(1, env->me_psize)))) + rc = ENOMEM; + if (!(flags & MDB_RDONLY)) { + MDB_txn *txn; + int tsize = sizeof(MDB_txn), size = tsize + env->me_maxdbs * + (sizeof(MDB_db)+sizeof(MDB_cursor *)+sizeof(unsigned int)+1); + txn = calloc(1, size); + if (txn) { + txn->mt_dbs = (MDB_db *)((char *)txn + tsize); + txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs); + txn->mt_dbiseqs = (unsigned int *)(txn->mt_cursors + env->me_maxdbs); + txn->mt_dbflags = (unsigned char *)(txn->mt_dbiseqs + env->me_maxdbs); + txn->mt_env = env; + txn->mt_dbxs = env->me_dbxs; + env->me_txn0 = txn; + } else { + rc = ENOMEM; + } + } + } + +leave: + if (rc) { + mdb_env_close0(env, excl); + } + free(lpath); + return rc; +} + +/** Destroy resources from mdb_env_open(), clear our readers & DBIs */ +static void ESECT +mdb_env_close0(MDB_env *env, int excl) +{ + int i; + + if (!(env->me_flags & MDB_ENV_ACTIVE)) + return; + + /* Doing this here since me_dbxs may not exist during mdb_env_close */ + if (env->me_dbxs) { + for (i = env->me_maxdbs; --i > MAIN_DBI; ) + free(env->me_dbxs[i].md_name.mv_data); + free(env->me_dbxs); + } + + free(env->me_pbuf); + free(env->me_dbiseqs); + free(env->me_dbflags); + free(env->me_path); + free(env->me_dirty_list); + free(env->me_txn0); + mdb_midl_free(env->me_free_pgs); + + if (env->me_flags & MDB_ENV_TXKEY) { + pthread_key_delete(env->me_txkey); +#ifdef _WIN32 + /* Delete our key from the global list */ + for (i=0; i<mdb_tls_nkeys; i++) + if (mdb_tls_keys[i] == env->me_txkey) { + mdb_tls_keys[i] = mdb_tls_keys[mdb_tls_nkeys-1]; + mdb_tls_nkeys--; + break; + } +#endif + } + + if (env->me_map) { + munmap(env->me_map, env->me_mapsize); + } + if (env->me_mfd != env->me_fd && env->me_mfd != INVALID_HANDLE_VALUE) + (void) close(env->me_mfd); + if (env->me_fd != INVALID_HANDLE_VALUE) + (void) close(env->me_fd); + if (env->me_txns) { + MDB_PID_T pid = env->me_pid; + /* Clearing readers is done in this function because + * me_txkey with its destructor must be disabled first. + */ + for (i = env->me_numreaders; --i >= 0; ) + if (env->me_txns->mti_readers[i].mr_pid == pid) + env->me_txns->mti_readers[i].mr_pid = 0; +#ifdef _WIN32 + if (env->me_rmutex) { + CloseHandle(env->me_rmutex); + if (env->me_wmutex) CloseHandle(env->me_wmutex); + } + /* Windows automatically destroys the mutexes when + * the last handle closes. + */ +#elif defined(MDB_USE_POSIX_SEM) + if (env->me_rmutex != SEM_FAILED) { + sem_close(env->me_rmutex); + if (env->me_wmutex != SEM_FAILED) + sem_close(env->me_wmutex); + /* If we have the filelock: If we are the + * only remaining user, clean up semaphores. + */ + if (excl == 0) + mdb_env_excl_lock(env, &excl); + if (excl > 0) { + sem_unlink(env->me_txns->mti_rmname); + sem_unlink(env->me_txns->mti_wmname); + } + } +#endif + munmap((void *)env->me_txns, (env->me_maxreaders-1)*sizeof(MDB_reader)+sizeof(MDB_txninfo)); + } + if (env->me_lfd != INVALID_HANDLE_VALUE) { +#ifdef _WIN32 + if (excl >= 0) { + /* Unlock the lockfile. Windows would have unlocked it + * after closing anyway, but not necessarily at once. + */ + UnlockFile(env->me_lfd, 0, 0, 1, 0); + } +#endif + (void) close(env->me_lfd); + } + + env->me_flags &= ~(MDB_ENV_ACTIVE|MDB_ENV_TXKEY); +} + + +void ESECT +mdb_env_close(MDB_env *env) +{ + MDB_page *dp; + + if (env == NULL) + return; + + VGMEMP_DESTROY(env); + while ((dp = env->me_dpages) != NULL) { + VGMEMP_DEFINED(&dp->mp_next, sizeof(dp->mp_next)); + env->me_dpages = dp->mp_next; + free(dp); + } + + mdb_env_close0(env, 0); + free(env); +} + +/** Compare two items pointing at aligned size_t's */ +static int +mdb_cmp_long(const MDB_val *a, const MDB_val *b) +{ + return (*(size_t *)a->mv_data < *(size_t *)b->mv_data) ? -1 : + *(size_t *)a->mv_data > *(size_t *)b->mv_data; +} + +/** Compare two items pointing at aligned unsigned int's */ +static int +mdb_cmp_int(const MDB_val *a, const MDB_val *b) +{ + return (*(unsigned int *)a->mv_data < *(unsigned int *)b->mv_data) ? -1 : + *(unsigned int *)a->mv_data > *(unsigned int *)b->mv_data; +} + +/** Compare two items pointing at unsigned ints of unknown alignment. + * Nodes and keys are guaranteed to be 2-byte aligned. + */ +static int +mdb_cmp_cint(const MDB_val *a, const MDB_val *b) +{ +#if BYTE_ORDER == LITTLE_ENDIAN + unsigned short *u, *c; + int x; + + u = (unsigned short *) ((char *) a->mv_data + a->mv_size); + c = (unsigned short *) ((char *) b->mv_data + a->mv_size); + do { + x = *--u - *--c; + } while(!x && u > (unsigned short *)a->mv_data); + return x; +#else + unsigned short *u, *c, *end; + int x; + + end = (unsigned short *) ((char *) a->mv_data + a->mv_size); + u = (unsigned short *)a->mv_data; + c = (unsigned short *)b->mv_data; + do { + x = *u++ - *c++; + } while(!x && u < end); + return x; +#endif +} + +/** Compare two items pointing at size_t's of unknown alignment. */ +#ifdef MISALIGNED_OK +# define mdb_cmp_clong mdb_cmp_long +#else +# define mdb_cmp_clong mdb_cmp_cint +#endif + +/** Compare two items lexically */ +static int +mdb_cmp_memn(const MDB_val *a, const MDB_val *b) +{ + int diff; + ssize_t len_diff; + unsigned int len; + + len = a->mv_size; + len_diff = (ssize_t) a->mv_size - (ssize_t) b->mv_size; + if (len_diff > 0) { + len = b->mv_size; + len_diff = 1; + } + + diff = memcmp(a->mv_data, b->mv_data, len); + return diff ? diff : len_diff<0 ? -1 : len_diff; +} + +/** Compare two items in reverse byte order */ +static int +mdb_cmp_memnr(const MDB_val *a, const MDB_val *b) +{ + const unsigned char *p1, *p2, *p1_lim; + ssize_t len_diff; + int diff; + + p1_lim = (const unsigned char *)a->mv_data; + p1 = (const unsigned char *)a->mv_data + a->mv_size; + p2 = (const unsigned char *)b->mv_data + b->mv_size; + + len_diff = (ssize_t) a->mv_size - (ssize_t) b->mv_size; + if (len_diff > 0) { + p1_lim += len_diff; + len_diff = 1; + } + + while (p1 > p1_lim) { + diff = *--p1 - *--p2; + if (diff) + return diff; + } + return len_diff<0 ? -1 : len_diff; +} + +/** Search for key within a page, using binary search. + * Returns the smallest entry larger or equal to the key. + * If exactp is non-null, stores whether the found entry was an exact match + * in *exactp (1 or 0). + * Updates the cursor index with the index of the found entry. + * If no entry larger or equal to the key is found, returns NULL. + */ +static MDB_node * +mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp) +{ + unsigned int i = 0, nkeys; + int low, high; + int rc = 0; + MDB_page *mp = mc->mc_pg[mc->mc_top]; + MDB_node *node = NULL; + MDB_val nodekey; + MDB_cmp_func *cmp; + DKBUF; + + nkeys = NUMKEYS(mp); + + DPRINTF(("searching %u keys in %s %spage %"Z"u", + nkeys, IS_LEAF(mp) ? "leaf" : "branch", IS_SUBP(mp) ? "sub-" : "", + mdb_dbg_pgno(mp))); + + low = IS_LEAF(mp) ? 0 : 1; + high = nkeys - 1; + cmp = mc->mc_dbx->md_cmp; + + /* Branch pages have no data, so if using integer keys, + * alignment is guaranteed. Use faster mdb_cmp_int. + */ + if (cmp == mdb_cmp_cint && IS_BRANCH(mp)) { + if (NODEPTR(mp, 1)->mn_ksize == sizeof(size_t)) + cmp = mdb_cmp_long; + else + cmp = mdb_cmp_int; + } + + if (IS_LEAF2(mp)) { + nodekey.mv_size = mc->mc_db->md_pad; + node = NODEPTR(mp, 0); /* fake */ + while (low <= high) { + i = (low + high) >> 1; + nodekey.mv_data = LEAF2KEY(mp, i, nodekey.mv_size); + rc = cmp(key, &nodekey); + DPRINTF(("found leaf index %u [%s], rc = %i", + i, DKEY(&nodekey), rc)); + if (rc == 0) + break; + if (rc > 0) + low = i + 1; + else + high = i - 1; + } + } else { + while (low <= high) { + i = (low + high) >> 1; + + node = NODEPTR(mp, i); + nodekey.mv_size = NODEKSZ(node); + nodekey.mv_data = NODEKEY(node); + + rc = cmp(key, &nodekey); +#if MDB_DEBUG + if (IS_LEAF(mp)) + DPRINTF(("found leaf index %u [%s], rc = %i", + i, DKEY(&nodekey), rc)); + else + DPRINTF(("found branch index %u [%s -> %"Z"u], rc = %i", + i, DKEY(&nodekey), NODEPGNO(node), rc)); +#endif + if (rc == 0) + break; + if (rc > 0) + low = i + 1; + else + high = i - 1; + } + } + + if (rc > 0) { /* Found entry is less than the key. */ + i++; /* Skip to get the smallest entry larger than key. */ + if (!IS_LEAF2(mp)) + node = NODEPTR(mp, i); + } + if (exactp) + *exactp = (rc == 0 && nkeys > 0); + /* store the key index */ + mc->mc_ki[mc->mc_top] = i; + if (i >= nkeys) + /* There is no entry larger or equal to the key. */ + return NULL; + + /* nodeptr is fake for LEAF2 */ + return node; +} + +#if 0 +static void +mdb_cursor_adjust(MDB_cursor *mc, func) +{ + MDB_cursor *m2; + + for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { + if (m2->mc_pg[m2->mc_top] == mc->mc_pg[mc->mc_top]) { + func(mc, m2); + } + } +} +#endif + +/** Pop a page off the top of the cursor's stack. */ +static void +mdb_cursor_pop(MDB_cursor *mc) +{ + if (mc->mc_snum) { +#if MDB_DEBUG + MDB_page *top = mc->mc_pg[mc->mc_top]; +#endif + mc->mc_snum--; + if (mc->mc_snum) + mc->mc_top--; + + DPRINTF(("popped page %"Z"u off db %d cursor %p", top->mp_pgno, + DDBI(mc), (void *) mc)); + } +} + +/** Push a page onto the top of the cursor's stack. */ +static int +mdb_cursor_push(MDB_cursor *mc, MDB_page *mp) +{ + DPRINTF(("pushing page %"Z"u on db %d cursor %p", mp->mp_pgno, + DDBI(mc), (void *) mc)); + + if (mc->mc_snum >= CURSOR_STACK) { + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return MDB_CURSOR_FULL; + } + + mc->mc_top = mc->mc_snum++; + mc->mc_pg[mc->mc_top] = mp; + mc->mc_ki[mc->mc_top] = 0; + + return MDB_SUCCESS; +} + +/** Find the address of the page corresponding to a given page number. + * @param[in] txn the transaction for this access. + * @param[in] pgno the page number for the page to retrieve. + * @param[out] ret address of a pointer where the page's address will be stored. + * @param[out] lvl dirty_list inheritance level of found page. 1=current txn, 0=mapped page. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_get(MDB_txn *txn, pgno_t pgno, MDB_page **ret, int *lvl) +{ + MDB_env *env = txn->mt_env; + MDB_page *p = NULL; + int level; + + if (!((txn->mt_flags & MDB_TXN_RDONLY) | (env->me_flags & MDB_WRITEMAP))) { + MDB_txn *tx2 = txn; + level = 1; + do { + MDB_ID2L dl = tx2->mt_u.dirty_list; + unsigned x; + /* Spilled pages were dirtied in this txn and flushed + * because the dirty list got full. Bring this page + * back in from the map (but don't unspill it here, + * leave that unless page_touch happens again). + */ + if (tx2->mt_spill_pgs) { + MDB_ID pn = pgno << 1; + x = mdb_midl_search(tx2->mt_spill_pgs, pn); + if (x <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[x] == pn) { + p = (MDB_page *)(env->me_map + env->me_psize * pgno); + goto done; + } + } + if (dl[0].mid) { + unsigned x = mdb_mid2l_search(dl, pgno); + if (x <= dl[0].mid && dl[x].mid == pgno) { + p = dl[x].mptr; + goto done; + } + } + level++; + } while ((tx2 = tx2->mt_parent) != NULL); + } + + if (pgno < txn->mt_next_pgno) { + level = 0; + p = (MDB_page *)(env->me_map + env->me_psize * pgno); + } else { + DPRINTF(("page %"Z"u not found", pgno)); + txn->mt_flags |= MDB_TXN_ERROR; + return MDB_PAGE_NOTFOUND; + } + +done: + *ret = p; + if (lvl) + *lvl = level; + return MDB_SUCCESS; +} + +/** Finish #mdb_page_search() / #mdb_page_search_lowest(). + * The cursor is at the root page, set up the rest of it. + */ +static int +mdb_page_search_root(MDB_cursor *mc, MDB_val *key, int flags) +{ + MDB_page *mp = mc->mc_pg[mc->mc_top]; + int rc; + DKBUF; + + while (IS_BRANCH(mp)) { + MDB_node *node; + indx_t i; + + DPRINTF(("branch page %"Z"u has %u keys", mp->mp_pgno, NUMKEYS(mp))); + mdb_cassert(mc, NUMKEYS(mp) > 1); + DPRINTF(("found index 0 to page %"Z"u", NODEPGNO(NODEPTR(mp, 0)))); + + if (flags & (MDB_PS_FIRST|MDB_PS_LAST)) { + i = 0; + if (flags & MDB_PS_LAST) + i = NUMKEYS(mp) - 1; + } else { + int exact; + node = mdb_node_search(mc, key, &exact); + if (node == NULL) + i = NUMKEYS(mp) - 1; + else { + i = mc->mc_ki[mc->mc_top]; + if (!exact) { + mdb_cassert(mc, i > 0); + i--; + } + } + DPRINTF(("following index %u for key [%s]", i, DKEY(key))); + } + + mdb_cassert(mc, i < NUMKEYS(mp)); + node = NODEPTR(mp, i); + + if ((rc = mdb_page_get(mc->mc_txn, NODEPGNO(node), &mp, NULL)) != 0) + return rc; + + mc->mc_ki[mc->mc_top] = i; + if ((rc = mdb_cursor_push(mc, mp))) + return rc; + + if (flags & MDB_PS_MODIFY) { + if ((rc = mdb_page_touch(mc)) != 0) + return rc; + mp = mc->mc_pg[mc->mc_top]; + } + } + + if (!IS_LEAF(mp)) { + DPRINTF(("internal error, index points to a %02X page!?", + mp->mp_flags)); + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return MDB_CORRUPTED; + } + + DPRINTF(("found leaf page %"Z"u for key [%s]", mp->mp_pgno, + key ? DKEY(key) : "null")); + mc->mc_flags |= C_INITIALIZED; + mc->mc_flags &= ~C_EOF; + + return MDB_SUCCESS; +} + +/** Search for the lowest key under the current branch page. + * This just bypasses a NUMKEYS check in the current page + * before calling mdb_page_search_root(), because the callers + * are all in situations where the current page is known to + * be underfilled. + */ +static int +mdb_page_search_lowest(MDB_cursor *mc) +{ + MDB_page *mp = mc->mc_pg[mc->mc_top]; + MDB_node *node = NODEPTR(mp, 0); + int rc; + + if ((rc = mdb_page_get(mc->mc_txn, NODEPGNO(node), &mp, NULL)) != 0) + return rc; + + mc->mc_ki[mc->mc_top] = 0; + if ((rc = mdb_cursor_push(mc, mp))) + return rc; + return mdb_page_search_root(mc, NULL, MDB_PS_FIRST); +} + +/** Search for the page a given key should be in. + * Push it and its parent pages on the cursor stack. + * @param[in,out] mc the cursor for this operation. + * @param[in] key the key to search for, or NULL for first/last page. + * @param[in] flags If MDB_PS_MODIFY is set, visited pages in the DB + * are touched (updated with new page numbers). + * If MDB_PS_FIRST or MDB_PS_LAST is set, find first or last leaf. + * This is used by #mdb_cursor_first() and #mdb_cursor_last(). + * If MDB_PS_ROOTONLY set, just fetch root node, no further lookups. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_search(MDB_cursor *mc, MDB_val *key, int flags) +{ + int rc; + pgno_t root; + + /* Make sure the txn is still viable, then find the root from + * the txn's db table and set it as the root of the cursor's stack. + */ + if (F_ISSET(mc->mc_txn->mt_flags, MDB_TXN_ERROR)) { + DPUTS("transaction has failed, must abort"); + return MDB_BAD_TXN; + } else { + /* Make sure we're using an up-to-date root */ + if (*mc->mc_dbflag & DB_STALE) { + MDB_cursor mc2; + if (TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbi)) + return MDB_BAD_DBI; + mdb_cursor_init(&mc2, mc->mc_txn, MAIN_DBI, NULL); + rc = mdb_page_search(&mc2, &mc->mc_dbx->md_name, 0); + if (rc) + return rc; + { + MDB_val data; + int exact = 0; + uint16_t flags; + MDB_node *leaf = mdb_node_search(&mc2, + &mc->mc_dbx->md_name, &exact); + if (!exact) + return MDB_NOTFOUND; + rc = mdb_node_read(mc->mc_txn, leaf, &data); + if (rc) + return rc; + memcpy(&flags, ((char *) data.mv_data + offsetof(MDB_db, md_flags)), + sizeof(uint16_t)); + /* The txn may not know this DBI, or another process may + * have dropped and recreated the DB with other flags. + */ + if ((mc->mc_db->md_flags & PERSISTENT_FLAGS) != flags) + return MDB_INCOMPATIBLE; + memcpy(mc->mc_db, data.mv_data, sizeof(MDB_db)); + } + *mc->mc_dbflag &= ~DB_STALE; + } + root = mc->mc_db->md_root; + + if (root == P_INVALID) { /* Tree is empty. */ + DPUTS("tree is empty"); + return MDB_NOTFOUND; + } + } + + mdb_cassert(mc, root > 1); + if (!mc->mc_pg[0] || mc->mc_pg[0]->mp_pgno != root) + if ((rc = mdb_page_get(mc->mc_txn, root, &mc->mc_pg[0], NULL)) != 0) + return rc; + + mc->mc_snum = 1; + mc->mc_top = 0; + + DPRINTF(("db %d root page %"Z"u has flags 0x%X", + DDBI(mc), root, mc->mc_pg[0]->mp_flags)); + + if (flags & MDB_PS_MODIFY) { + if ((rc = mdb_page_touch(mc))) + return rc; + } + + if (flags & MDB_PS_ROOTONLY) + return MDB_SUCCESS; + + return mdb_page_search_root(mc, key, flags); +} + +static int +mdb_ovpage_free(MDB_cursor *mc, MDB_page *mp) +{ + MDB_txn *txn = mc->mc_txn; + pgno_t pg = mp->mp_pgno; + unsigned x = 0, ovpages = mp->mp_pages; + MDB_env *env = txn->mt_env; + MDB_IDL sl = txn->mt_spill_pgs; + MDB_ID pn = pg << 1; + int rc; + + DPRINTF(("free ov page %"Z"u (%d)", pg, ovpages)); + /* If the page is dirty or on the spill list we just acquired it, + * so we should give it back to our current free list, if any. + * Otherwise put it onto the list of pages we freed in this txn. + * + * Won't create me_pghead: me_pglast must be inited along with it. + * Unsupported in nested txns: They would need to hide the page + * range in ancestor txns' dirty and spilled lists. + */ + if (env->me_pghead && + !txn->mt_parent && + ((mp->mp_flags & P_DIRTY) || + (sl && (x = mdb_midl_search(sl, pn)) <= sl[0] && sl[x] == pn))) + { + unsigned i, j; + pgno_t *mop; + MDB_ID2 *dl, ix, iy; + rc = mdb_midl_need(&env->me_pghead, ovpages); + if (rc) + return rc; + if (!(mp->mp_flags & P_DIRTY)) { + /* This page is no longer spilled */ + if (x == sl[0]) + sl[0]--; + else + sl[x] |= 1; + goto release; + } + /* Remove from dirty list */ + dl = txn->mt_u.dirty_list; + x = dl[0].mid--; + for (ix = dl[x]; ix.mptr != mp; ix = iy) { + if (x > 1) { + x--; + iy = dl[x]; + dl[x] = ix; + } else { + mdb_cassert(mc, x > 1); + j = ++(dl[0].mid); + dl[j] = ix; /* Unsorted. OK when MDB_TXN_ERROR. */ + txn->mt_flags |= MDB_TXN_ERROR; + return MDB_CORRUPTED; + } + } + if (!(env->me_flags & MDB_WRITEMAP)) + mdb_dpage_free(env, mp); +release: + /* Insert in me_pghead */ + mop = env->me_pghead; + j = mop[0] + ovpages; + for (i = mop[0]; i && mop[i] < pg; i--) + mop[j--] = mop[i]; + while (j>i) + mop[j--] = pg++; + mop[0] += ovpages; + } else { + rc = mdb_midl_append_range(&txn->mt_free_pgs, pg, ovpages); + if (rc) + return rc; + } + mc->mc_db->md_overflow_pages -= ovpages; + return 0; +} + +/** Return the data associated with a given node. + * @param[in] txn The transaction for this operation. + * @param[in] leaf The node being read. + * @param[out] data Updated to point to the node's data. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_node_read(MDB_txn *txn, MDB_node *leaf, MDB_val *data) +{ + MDB_page *omp; /* overflow page */ + pgno_t pgno; + int rc; + + if (!F_ISSET(leaf->mn_flags, F_BIGDATA)) { + data->mv_size = NODEDSZ(leaf); + data->mv_data = NODEDATA(leaf); + return MDB_SUCCESS; + } + + /* Read overflow data. + */ + data->mv_size = NODEDSZ(leaf); + memcpy(&pgno, NODEDATA(leaf), sizeof(pgno)); + if ((rc = mdb_page_get(txn, pgno, &omp, NULL)) != 0) { + DPRINTF(("read overflow page %"Z"u failed", pgno)); + return rc; + } + data->mv_data = METADATA(omp); + + return MDB_SUCCESS; +} + +int +mdb_get(MDB_txn *txn, MDB_dbi dbi, + MDB_val *key, MDB_val *data) +{ + MDB_cursor mc; + MDB_xcursor mx; + int exact = 0; + DKBUF; + + DPRINTF(("===> get db %u key [%s]", dbi, DKEY(key))); + + if (!key || !data || dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + + if (txn->mt_flags & MDB_TXN_ERROR) + return MDB_BAD_TXN; + + mdb_cursor_init(&mc, txn, dbi, &mx); + return mdb_cursor_set(&mc, key, data, MDB_SET, &exact); +} + +/** Find a sibling for a page. + * Replaces the page at the top of the cursor's stack with the + * specified sibling, if one exists. + * @param[in] mc The cursor for this operation. + * @param[in] move_right Non-zero if the right sibling is requested, + * otherwise the left sibling. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_cursor_sibling(MDB_cursor *mc, int move_right) +{ + int rc; + MDB_node *indx; + MDB_page *mp; + + if (mc->mc_snum < 2) { + return MDB_NOTFOUND; /* root has no siblings */ + } + + mdb_cursor_pop(mc); + DPRINTF(("parent page is page %"Z"u, index %u", + mc->mc_pg[mc->mc_top]->mp_pgno, mc->mc_ki[mc->mc_top])); + + if (move_right ? (mc->mc_ki[mc->mc_top] + 1u >= NUMKEYS(mc->mc_pg[mc->mc_top])) + : (mc->mc_ki[mc->mc_top] == 0)) { + DPRINTF(("no more keys left, moving to %s sibling", + move_right ? "right" : "left")); + if ((rc = mdb_cursor_sibling(mc, move_right)) != MDB_SUCCESS) { + /* undo cursor_pop before returning */ + mc->mc_top++; + mc->mc_snum++; + return rc; + } + } else { + if (move_right) + mc->mc_ki[mc->mc_top]++; + else + mc->mc_ki[mc->mc_top]--; + DPRINTF(("just moving to %s index key %u", + move_right ? "right" : "left", mc->mc_ki[mc->mc_top])); + } + mdb_cassert(mc, IS_BRANCH(mc->mc_pg[mc->mc_top])); + + indx = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if ((rc = mdb_page_get(mc->mc_txn, NODEPGNO(indx), &mp, NULL)) != 0) { + /* mc will be inconsistent if caller does mc_snum++ as above */ + mc->mc_flags &= ~(C_INITIALIZED|C_EOF); + return rc; + } + + mdb_cursor_push(mc, mp); + if (!move_right) + mc->mc_ki[mc->mc_top] = NUMKEYS(mp)-1; + + return MDB_SUCCESS; +} + +/** Move the cursor to the next data item. */ +static int +mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) +{ + MDB_page *mp; + MDB_node *leaf; + int rc; + + if (mc->mc_flags & C_EOF) { + return MDB_NOTFOUND; + } + + mdb_cassert(mc, mc->mc_flags & C_INITIALIZED); + + mp = mc->mc_pg[mc->mc_top]; + + if (mc->mc_db->md_flags & MDB_DUPSORT) { + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + if (op == MDB_NEXT || op == MDB_NEXT_DUP) { + rc = mdb_cursor_next(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_NEXT); + if (op != MDB_NEXT || rc != MDB_NOTFOUND) { + if (rc == MDB_SUCCESS) + MDB_GET_KEY(leaf, key); + return rc; + } + } + } else { + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + if (op == MDB_NEXT_DUP) + return MDB_NOTFOUND; + } + } + + DPRINTF(("cursor_next: top page is %"Z"u in cursor %p", + mdb_dbg_pgno(mp), (void *) mc)); + if (mc->mc_flags & C_DEL) + goto skip; + + if (mc->mc_ki[mc->mc_top] + 1u >= NUMKEYS(mp)) { + DPUTS("=====> move to next sibling page"); + if ((rc = mdb_cursor_sibling(mc, 1)) != MDB_SUCCESS) { + mc->mc_flags |= C_EOF; + return rc; + } + mp = mc->mc_pg[mc->mc_top]; + DPRINTF(("next page is %"Z"u, key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top])); + } else + mc->mc_ki[mc->mc_top]++; + +skip: + DPRINTF(("==> cursor points to page %"Z"u with %u keys, key index %u", + mdb_dbg_pgno(mp), NUMKEYS(mp), mc->mc_ki[mc->mc_top])); + + if (IS_LEAF2(mp)) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); + return MDB_SUCCESS; + } + + mdb_cassert(mc, IS_LEAF(mp)); + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + mdb_xcursor_init1(mc, leaf); + } + if (data) { + if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS) + return rc; + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); + if (rc != MDB_SUCCESS) + return rc; + } + } + + MDB_GET_KEY(leaf, key); + return MDB_SUCCESS; +} + +/** Move the cursor to the previous data item. */ +static int +mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) +{ + MDB_page *mp; + MDB_node *leaf; + int rc; + + mdb_cassert(mc, mc->mc_flags & C_INITIALIZED); + + mp = mc->mc_pg[mc->mc_top]; + + if (mc->mc_db->md_flags & MDB_DUPSORT) { + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + if (op == MDB_PREV || op == MDB_PREV_DUP) { + rc = mdb_cursor_prev(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_PREV); + if (op != MDB_PREV || rc != MDB_NOTFOUND) { + if (rc == MDB_SUCCESS) { + MDB_GET_KEY(leaf, key); + mc->mc_flags &= ~C_EOF; + } + return rc; + } + } + } else { + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + if (op == MDB_PREV_DUP) + return MDB_NOTFOUND; + } + } + + DPRINTF(("cursor_prev: top page is %"Z"u in cursor %p", + mdb_dbg_pgno(mp), (void *) mc)); + + if (mc->mc_ki[mc->mc_top] == 0) { + DPUTS("=====> move to prev sibling page"); + if ((rc = mdb_cursor_sibling(mc, 0)) != MDB_SUCCESS) { + return rc; + } + mp = mc->mc_pg[mc->mc_top]; + mc->mc_ki[mc->mc_top] = NUMKEYS(mp) - 1; + DPRINTF(("prev page is %"Z"u, key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top])); + } else + mc->mc_ki[mc->mc_top]--; + + mc->mc_flags &= ~C_EOF; + + DPRINTF(("==> cursor points to page %"Z"u with %u keys, key index %u", + mdb_dbg_pgno(mp), NUMKEYS(mp), mc->mc_ki[mc->mc_top])); + + if (IS_LEAF2(mp)) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); + return MDB_SUCCESS; + } + + mdb_cassert(mc, IS_LEAF(mp)); + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + mdb_xcursor_init1(mc, leaf); + } + if (data) { + if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS) + return rc; + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + rc = mdb_cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL); + if (rc != MDB_SUCCESS) + return rc; + } + } + + MDB_GET_KEY(leaf, key); + return MDB_SUCCESS; +} + +/** Set the cursor on a specific data item. */ +static int +mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data, + MDB_cursor_op op, int *exactp) +{ + int rc; + MDB_page *mp; + MDB_node *leaf = NULL; + DKBUF; + + if (key->mv_size == 0) + return MDB_BAD_VALSIZE; + + if (mc->mc_xcursor) + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + + /* See if we're already on the right page */ + if (mc->mc_flags & C_INITIALIZED) { + MDB_val nodekey; + + mp = mc->mc_pg[mc->mc_top]; + if (!NUMKEYS(mp)) { + mc->mc_ki[mc->mc_top] = 0; + return MDB_NOTFOUND; + } + if (mp->mp_flags & P_LEAF2) { + nodekey.mv_size = mc->mc_db->md_pad; + nodekey.mv_data = LEAF2KEY(mp, 0, nodekey.mv_size); + } else { + leaf = NODEPTR(mp, 0); + MDB_GET_KEY2(leaf, nodekey); + } + rc = mc->mc_dbx->md_cmp(key, &nodekey); + if (rc == 0) { + /* Probably happens rarely, but first node on the page + * was the one we wanted. + */ + mc->mc_ki[mc->mc_top] = 0; + if (exactp) + *exactp = 1; + goto set1; + } + if (rc > 0) { + unsigned int i; + unsigned int nkeys = NUMKEYS(mp); + if (nkeys > 1) { + if (mp->mp_flags & P_LEAF2) { + nodekey.mv_data = LEAF2KEY(mp, + nkeys-1, nodekey.mv_size); + } else { + leaf = NODEPTR(mp, nkeys-1); + MDB_GET_KEY2(leaf, nodekey); + } + rc = mc->mc_dbx->md_cmp(key, &nodekey); + if (rc == 0) { + /* last node was the one we wanted */ + mc->mc_ki[mc->mc_top] = nkeys-1; + if (exactp) + *exactp = 1; + goto set1; + } + if (rc < 0) { + if (mc->mc_ki[mc->mc_top] < NUMKEYS(mp)) { + /* This is definitely the right page, skip search_page */ + if (mp->mp_flags & P_LEAF2) { + nodekey.mv_data = LEAF2KEY(mp, + mc->mc_ki[mc->mc_top], nodekey.mv_size); + } else { + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + MDB_GET_KEY2(leaf, nodekey); + } + rc = mc->mc_dbx->md_cmp(key, &nodekey); + if (rc == 0) { + /* current node was the one we wanted */ + if (exactp) + *exactp = 1; + goto set1; + } + } + rc = 0; + goto set2; + } + } + /* If any parents have right-sibs, search. + * Otherwise, there's nothing further. + */ + for (i=0; i<mc->mc_top; i++) + if (mc->mc_ki[i] < + NUMKEYS(mc->mc_pg[i])-1) + break; + if (i == mc->mc_top) { + /* There are no other pages */ + mc->mc_ki[mc->mc_top] = nkeys; + return MDB_NOTFOUND; + } + } + if (!mc->mc_top) { + /* There are no other pages */ + mc->mc_ki[mc->mc_top] = 0; + if (op == MDB_SET_RANGE && !exactp) { + rc = 0; + goto set1; + } else + return MDB_NOTFOUND; + } + } + + rc = mdb_page_search(mc, key, 0); + if (rc != MDB_SUCCESS) + return rc; + + mp = mc->mc_pg[mc->mc_top]; + mdb_cassert(mc, IS_LEAF(mp)); + +set2: + leaf = mdb_node_search(mc, key, exactp); + if (exactp != NULL && !*exactp) { + /* MDB_SET specified and not an exact match. */ + return MDB_NOTFOUND; + } + + if (leaf == NULL) { + DPUTS("===> inexact leaf not found, goto sibling"); + if ((rc = mdb_cursor_sibling(mc, 1)) != MDB_SUCCESS) + return rc; /* no entries matched */ + mp = mc->mc_pg[mc->mc_top]; + mdb_cassert(mc, IS_LEAF(mp)); + leaf = NODEPTR(mp, 0); + } + +set1: + mc->mc_flags |= C_INITIALIZED; + mc->mc_flags &= ~C_EOF; + + if (IS_LEAF2(mp)) { + if (op == MDB_SET_RANGE || op == MDB_SET_KEY) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); + } + return MDB_SUCCESS; + } + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + mdb_xcursor_init1(mc, leaf); + } + if (data) { + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + if (op == MDB_SET || op == MDB_SET_KEY || op == MDB_SET_RANGE) { + rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); + } else { + int ex2, *ex2p; + if (op == MDB_GET_BOTH) { + ex2p = &ex2; + ex2 = 0; + } else { + ex2p = NULL; + } + rc = mdb_cursor_set(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_SET_RANGE, ex2p); + if (rc != MDB_SUCCESS) + return rc; + } + } else if (op == MDB_GET_BOTH || op == MDB_GET_BOTH_RANGE) { + MDB_val d2; + if ((rc = mdb_node_read(mc->mc_txn, leaf, &d2)) != MDB_SUCCESS) + return rc; + rc = mc->mc_dbx->md_dcmp(data, &d2); + if (rc) { + if (op == MDB_GET_BOTH || rc > 0) + return MDB_NOTFOUND; + rc = 0; + *data = d2; + } + + } else { + if (mc->mc_xcursor) + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS) + return rc; + } + } + + /* The key already matches in all other cases */ + if (op == MDB_SET_RANGE || op == MDB_SET_KEY) + MDB_GET_KEY(leaf, key); + DPRINTF(("==> cursor placed on key [%s]", DKEY(key))); + + return rc; +} + +/** Move the cursor to the first item in the database. */ +static int +mdb_cursor_first(MDB_cursor *mc, MDB_val *key, MDB_val *data) +{ + int rc; + MDB_node *leaf; + + if (mc->mc_xcursor) + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + + if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) { + rc = mdb_page_search(mc, NULL, MDB_PS_FIRST); + if (rc != MDB_SUCCESS) + return rc; + } + mdb_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top])); + + leaf = NODEPTR(mc->mc_pg[mc->mc_top], 0); + mc->mc_flags |= C_INITIALIZED; + mc->mc_flags &= ~C_EOF; + + mc->mc_ki[mc->mc_top] = 0; + + if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mc->mc_pg[mc->mc_top], 0, key->mv_size); + return MDB_SUCCESS; + } + + if (data) { + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + mdb_xcursor_init1(mc, leaf); + rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); + if (rc) + return rc; + } else { + if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS) + return rc; + } + } + MDB_GET_KEY(leaf, key); + return MDB_SUCCESS; +} + +/** Move the cursor to the last item in the database. */ +static int +mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data) +{ + int rc; + MDB_node *leaf; + + if (mc->mc_xcursor) + mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); + + if (!(mc->mc_flags & C_EOF)) { + + if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) { + rc = mdb_page_search(mc, NULL, MDB_PS_LAST); + if (rc != MDB_SUCCESS) + return rc; + } + mdb_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top])); + + } + mc->mc_ki[mc->mc_top] = NUMKEYS(mc->mc_pg[mc->mc_top]) - 1; + mc->mc_flags |= C_INITIALIZED|C_EOF; + leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + + if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top], key->mv_size); + return MDB_SUCCESS; + } + + if (data) { + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + mdb_xcursor_init1(mc, leaf); + rc = mdb_cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL); + if (rc) + return rc; + } else { + if ((rc = mdb_node_read(mc->mc_txn, leaf, data)) != MDB_SUCCESS) + return rc; + } + } + + MDB_GET_KEY(leaf, key); + return MDB_SUCCESS; +} + +int +mdb_cursor_get(MDB_cursor *mc, MDB_val *key, MDB_val *data, + MDB_cursor_op op) +{ + int rc; + int exact = 0; + int (*mfunc)(MDB_cursor *mc, MDB_val *key, MDB_val *data); + + if (mc == NULL) + return EINVAL; + + if (mc->mc_txn->mt_flags & MDB_TXN_ERROR) + return MDB_BAD_TXN; + + switch (op) { + case MDB_GET_CURRENT: + if (!(mc->mc_flags & C_INITIALIZED)) { + rc = EINVAL; + } else { + MDB_page *mp = mc->mc_pg[mc->mc_top]; + int nkeys = NUMKEYS(mp); + if (!nkeys || mc->mc_ki[mc->mc_top] >= nkeys) { + mc->mc_ki[mc->mc_top] = nkeys; + rc = MDB_NOTFOUND; + break; + } + rc = MDB_SUCCESS; + if (IS_LEAF2(mp)) { + key->mv_size = mc->mc_db->md_pad; + key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); + } else { + MDB_node *leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + MDB_GET_KEY(leaf, key); + if (data) { + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + if (mc->mc_flags & C_DEL) + mdb_xcursor_init1(mc, leaf); + rc = mdb_cursor_get(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_GET_CURRENT); + } else { + rc = mdb_node_read(mc->mc_txn, leaf, data); + } + } + } + } + break; + case MDB_GET_BOTH: + case MDB_GET_BOTH_RANGE: + if (data == NULL) { + rc = EINVAL; + break; + } + if (mc->mc_xcursor == NULL) { + rc = MDB_INCOMPATIBLE; + break; + } + /* FALLTHRU */ + case MDB_SET: + case MDB_SET_KEY: + case MDB_SET_RANGE: + if (key == NULL) { + rc = EINVAL; + } else { + rc = mdb_cursor_set(mc, key, data, op, + op == MDB_SET_RANGE ? NULL : &exact); + } + break; + case MDB_GET_MULTIPLE: + if (data == NULL || !(mc->mc_flags & C_INITIALIZED)) { + rc = EINVAL; + break; + } + if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) { + rc = MDB_INCOMPATIBLE; + break; + } + rc = MDB_SUCCESS; + if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) || + (mc->mc_xcursor->mx_cursor.mc_flags & C_EOF)) + break; + goto fetchm; + case MDB_NEXT_MULTIPLE: + if (data == NULL) { + rc = EINVAL; + break; + } + if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) { + rc = MDB_INCOMPATIBLE; + break; + } + if (!(mc->mc_flags & C_INITIALIZED)) + rc = mdb_cursor_first(mc, key, data); + else + rc = mdb_cursor_next(mc, key, data, MDB_NEXT_DUP); + if (rc == MDB_SUCCESS) { + if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { + MDB_cursor *mx; +fetchm: + mx = &mc->mc_xcursor->mx_cursor; + data->mv_size = NUMKEYS(mx->mc_pg[mx->mc_top]) * + mx->mc_db->md_pad; + data->mv_data = METADATA(mx->mc_pg[mx->mc_top]); + mx->mc_ki[mx->mc_top] = NUMKEYS(mx->mc_pg[mx->mc_top])-1; + } else { + rc = MDB_NOTFOUND; + } + } + break; + case MDB_NEXT: + case MDB_NEXT_DUP: + case MDB_NEXT_NODUP: + if (!(mc->mc_flags & C_INITIALIZED)) + rc = mdb_cursor_first(mc, key, data); + else + rc = mdb_cursor_next(mc, key, data, op); + break; + case MDB_PREV: + case MDB_PREV_DUP: + case MDB_PREV_NODUP: + if (!(mc->mc_flags & C_INITIALIZED)) { + rc = mdb_cursor_last(mc, key, data); + if (rc) + break; + mc->mc_flags |= C_INITIALIZED; + mc->mc_ki[mc->mc_top]++; + } + rc = mdb_cursor_prev(mc, key, data, op); + break; + case MDB_FIRST: + rc = mdb_cursor_first(mc, key, data); + break; + case MDB_FIRST_DUP: + mfunc = mdb_cursor_first; + mmove: + if (data == NULL || !(mc->mc_flags & C_INITIALIZED)) { + rc = EINVAL; + break; + } + if (mc->mc_xcursor == NULL) { + rc = MDB_INCOMPATIBLE; + break; + } + { + MDB_node *leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { + MDB_GET_KEY(leaf, key); + rc = mdb_node_read(mc->mc_txn, leaf, data); + break; + } + } + if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) { + rc = EINVAL; + break; + } + rc = mfunc(&mc->mc_xcursor->mx_cursor, data, NULL); + break; + case MDB_LAST: + rc = mdb_cursor_last(mc, key, data); + break; + case MDB_LAST_DUP: + mfunc = mdb_cursor_last; + goto mmove; + default: + DPRINTF(("unhandled/unimplemented cursor operation %u", op)); + rc = EINVAL; + break; + } + + if (mc->mc_flags & C_DEL) + mc->mc_flags ^= C_DEL; + + return rc; +} + +/** Touch all the pages in the cursor stack. Set mc_top. + * Makes sure all the pages are writable, before attempting a write operation. + * @param[in] mc The cursor to operate on. + */ +static int +mdb_cursor_touch(MDB_cursor *mc) +{ + int rc = MDB_SUCCESS; + + if (mc->mc_dbi > MAIN_DBI && !(*mc->mc_dbflag & DB_DIRTY)) { + MDB_cursor mc2; + MDB_xcursor mcx; + if (TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbi)) + return MDB_BAD_DBI; + mdb_cursor_init(&mc2, mc->mc_txn, MAIN_DBI, &mcx); + rc = mdb_page_search(&mc2, &mc->mc_dbx->md_name, MDB_PS_MODIFY); + if (rc) + return rc; + *mc->mc_dbflag |= DB_DIRTY; + } + mc->mc_top = 0; + if (mc->mc_snum) { + do { + rc = mdb_page_touch(mc); + } while (!rc && ++(mc->mc_top) < mc->mc_snum); + mc->mc_top = mc->mc_snum-1; + } + return rc; +} + +/** Do not spill pages to disk if txn is getting full, may fail instead */ +#define MDB_NOSPILL 0x8000 + +int +mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, + unsigned int flags) +{ + enum { MDB_NO_ROOT = MDB_LAST_ERRCODE+10 }; /* internal code */ + MDB_env *env; + MDB_node *leaf = NULL; + MDB_page *fp, *mp; + uint16_t fp_flags; + MDB_val xdata, *rdata, dkey, olddata; + MDB_db dummy; + int do_sub = 0, insert_key, insert_data; + unsigned int mcount = 0, dcount = 0, nospill; + size_t nsize; + int rc, rc2; + unsigned int nflags; + DKBUF; + + if (mc == NULL || key == NULL) + return EINVAL; + + env = mc->mc_txn->mt_env; + + /* Check this first so counter will always be zero on any + * early failures. + */ + if (flags & MDB_MULTIPLE) { + dcount = data[1].mv_size; + data[1].mv_size = 0; + if (!F_ISSET(mc->mc_db->md_flags, MDB_DUPFIXED)) + return MDB_INCOMPATIBLE; + } + + nospill = flags & MDB_NOSPILL; + flags &= ~MDB_NOSPILL; + + if (mc->mc_txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_ERROR)) + return (mc->mc_txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN; + + if (key->mv_size-1 >= ENV_MAXKEY(env)) + return MDB_BAD_VALSIZE; + +#if SIZE_MAX > MAXDATASIZE + if (data->mv_size > ((mc->mc_db->md_flags & MDB_DUPSORT) ? ENV_MAXKEY(env) : MAXDATASIZE)) + return MDB_BAD_VALSIZE; +#else + if ((mc->mc_db->md_flags & MDB_DUPSORT) && data->mv_size > ENV_MAXKEY(env)) + return MDB_BAD_VALSIZE; +#endif + + DPRINTF(("==> put db %d key [%s], size %"Z"u, data size %"Z"u", + DDBI(mc), DKEY(key), key ? key->mv_size : 0, data->mv_size)); + + dkey.mv_size = 0; + + if (flags == MDB_CURRENT) { + if (!(mc->mc_flags & C_INITIALIZED)) + return EINVAL; + rc = MDB_SUCCESS; + } else if (mc->mc_db->md_root == P_INVALID) { + /* new database, cursor has nothing to point to */ + mc->mc_snum = 0; + mc->mc_top = 0; + mc->mc_flags &= ~C_INITIALIZED; + rc = MDB_NO_ROOT; + } else { + int exact = 0; + MDB_val d2; + if (flags & MDB_APPEND) { + MDB_val k2; + rc = mdb_cursor_last(mc, &k2, &d2); + if (rc == 0) { + rc = mc->mc_dbx->md_cmp(key, &k2); + if (rc > 0) { + rc = MDB_NOTFOUND; + mc->mc_ki[mc->mc_top]++; + } else { + /* new key is <= last key */ + rc = MDB_KEYEXIST; + } + } + } else { + rc = mdb_cursor_set(mc, key, &d2, MDB_SET, &exact); + } + if ((flags & MDB_NOOVERWRITE) && rc == 0) { + DPRINTF(("duplicate key [%s]", DKEY(key))); + *data = d2; + return MDB_KEYEXIST; + } + if (rc && rc != MDB_NOTFOUND) + return rc; + } + + if (mc->mc_flags & C_DEL) + mc->mc_flags ^= C_DEL; + + /* Cursor is positioned, check for room in the dirty list */ + if (!nospill) { + if (flags & MDB_MULTIPLE) { + rdata = &xdata; + xdata.mv_size = data->mv_size * dcount; + } else { + rdata = data; + } + if ((rc2 = mdb_page_spill(mc, key, rdata))) + return rc2; + } + + if (rc == MDB_NO_ROOT) { + MDB_page *np; + /* new database, write a root leaf page */ + DPUTS("allocating new root leaf page"); + if ((rc2 = mdb_page_new(mc, P_LEAF, 1, &np))) { + return rc2; + } + mdb_cursor_push(mc, np); + mc->mc_db->md_root = np->mp_pgno; + mc->mc_db->md_depth++; + *mc->mc_dbflag |= DB_DIRTY; + if ((mc->mc_db->md_flags & (MDB_DUPSORT|MDB_DUPFIXED)) + == MDB_DUPFIXED) + np->mp_flags |= P_LEAF2; + mc->mc_flags |= C_INITIALIZED; + } else { + /* make sure all cursor pages are writable */ + rc2 = mdb_cursor_touch(mc); + if (rc2) + return rc2; + } + + insert_key = insert_data = rc; + if (insert_key) { + /* The key does not exist */ + DPRINTF(("inserting key at index %i", mc->mc_ki[mc->mc_top])); + if ((mc->mc_db->md_flags & MDB_DUPSORT) && + LEAFSIZE(key, data) > env->me_nodemax) + { + /* Too big for a node, insert in sub-DB. Set up an empty + * "old sub-page" for prep_subDB to expand to a full page. + */ + fp_flags = P_LEAF|P_DIRTY; + fp = env->me_pbuf; + fp->mp_pad = data->mv_size; /* used if MDB_DUPFIXED */ + fp->mp_lower = fp->mp_upper = (PAGEHDRSZ-PAGEBASE); + olddata.mv_size = PAGEHDRSZ; + goto prep_subDB; + } + } else { + /* there's only a key anyway, so this is a no-op */ + if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { + char *ptr; + unsigned int ksize = mc->mc_db->md_pad; + if (key->mv_size != ksize) + return MDB_BAD_VALSIZE; + ptr = LEAF2KEY(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top], ksize); + memcpy(ptr, key->mv_data, ksize); +fix_parent: + /* if overwriting slot 0 of leaf, need to + * update branch key if there is a parent page + */ + if (mc->mc_top && !mc->mc_ki[mc->mc_top]) { + unsigned short top = mc->mc_top; + mc->mc_top--; + /* slot 0 is always an empty key, find real slot */ + while (mc->mc_top && !mc->mc_ki[mc->mc_top]) + mc->mc_top--; + if (mc->mc_ki[mc->mc_top]) + rc2 = mdb_update_key(mc, key); + else + rc2 = MDB_SUCCESS; + mc->mc_top = top; + if (rc2) + return rc2; + } + return MDB_SUCCESS; + } + +more: + leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + olddata.mv_size = NODEDSZ(leaf); + olddata.mv_data = NODEDATA(leaf); + + /* DB has dups? */ + if (F_ISSET(mc->mc_db->md_flags, MDB_DUPSORT)) { + /* Prepare (sub-)page/sub-DB to accept the new item, + * if needed. fp: old sub-page or a header faking + * it. mp: new (sub-)page. offset: growth in page + * size. xdata: node data with new page or DB. + */ + unsigned i, offset = 0; + mp = fp = xdata.mv_data = env->me_pbuf; + mp->mp_pgno = mc->mc_pg[mc->mc_top]->mp_pgno; + + /* Was a single item before, must convert now */ + if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { + /* Just overwrite the current item */ + if (flags == MDB_CURRENT) + goto current; + +#if UINT_MAX < SIZE_MAX + if (mc->mc_dbx->md_dcmp == mdb_cmp_int && olddata.mv_size == sizeof(size_t)) + mc->mc_dbx->md_dcmp = mdb_cmp_clong; +#endif + /* does data match? */ + if (!mc->mc_dbx->md_dcmp(data, &olddata)) { + if (flags & MDB_NODUPDATA) + return MDB_KEYEXIST; + /* overwrite it */ + goto current; + } + + /* Back up original data item */ + dkey.mv_size = olddata.mv_size; + dkey.mv_data = memcpy(fp+1, olddata.mv_data, olddata.mv_size); + + /* Make sub-page header for the dup items, with dummy body */ + fp->mp_flags = P_LEAF|P_DIRTY|P_SUBP; + fp->mp_lower = (PAGEHDRSZ-PAGEBASE); + xdata.mv_size = PAGEHDRSZ + dkey.mv_size + data->mv_size; + if (mc->mc_db->md_flags & MDB_DUPFIXED) { + fp->mp_flags |= P_LEAF2; + fp->mp_pad = data->mv_size; + xdata.mv_size += 2 * data->mv_size; /* leave space for 2 more */ + } else { + xdata.mv_size += 2 * (sizeof(indx_t) + NODESIZE) + + (dkey.mv_size & 1) + (data->mv_size & 1); + } + fp->mp_upper = xdata.mv_size - PAGEBASE; + olddata.mv_size = xdata.mv_size; /* pretend olddata is fp */ + } else if (leaf->mn_flags & F_SUBDATA) { + /* Data is on sub-DB, just store it */ + flags |= F_DUPDATA|F_SUBDATA; + goto put_sub; + } else { + /* Data is on sub-page */ + fp = olddata.mv_data; + switch (flags) { + default: + if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) { + offset = EVEN(NODESIZE + sizeof(indx_t) + + data->mv_size); + break; + } + offset = fp->mp_pad; + if (SIZELEFT(fp) < offset) { + offset *= 4; /* space for 4 more */ + break; + } + /* FALLTHRU: Big enough MDB_DUPFIXED sub-page */ + case MDB_CURRENT: + fp->mp_flags |= P_DIRTY; + COPY_PGNO(fp->mp_pgno, mp->mp_pgno); + mc->mc_xcursor->mx_cursor.mc_pg[0] = fp; + flags |= F_DUPDATA; + goto put_sub; + } + xdata.mv_size = olddata.mv_size + offset; + } + + fp_flags = fp->mp_flags; + if (NODESIZE + NODEKSZ(leaf) + xdata.mv_size > env->me_nodemax) { + /* Too big for a sub-page, convert to sub-DB */ + fp_flags &= ~P_SUBP; +prep_subDB: + if (mc->mc_db->md_flags & MDB_DUPFIXED) { + fp_flags |= P_LEAF2; + dummy.md_pad = fp->mp_pad; + dummy.md_flags = MDB_DUPFIXED; + if (mc->mc_db->md_flags & MDB_INTEGERDUP) + dummy.md_flags |= MDB_INTEGERKEY; + } else { + dummy.md_pad = 0; + dummy.md_flags = 0; + } + dummy.md_depth = 1; + dummy.md_branch_pages = 0; + dummy.md_leaf_pages = 1; + dummy.md_overflow_pages = 0; + dummy.md_entries = NUMKEYS(fp); + xdata.mv_size = sizeof(MDB_db); + xdata.mv_data = &dummy; + if ((rc = mdb_page_alloc(mc, 1, &mp))) + return rc; + offset = env->me_psize - olddata.mv_size; + flags |= F_DUPDATA|F_SUBDATA; + dummy.md_root = mp->mp_pgno; + } + if (mp != fp) { + mp->mp_flags = fp_flags | P_DIRTY; + mp->mp_pad = fp->mp_pad; + mp->mp_lower = fp->mp_lower; + mp->mp_upper = fp->mp_upper + offset; + if (fp_flags & P_LEAF2) { + memcpy(METADATA(mp), METADATA(fp), NUMKEYS(fp) * fp->mp_pad); + } else { + memcpy((char *)mp + mp->mp_upper + PAGEBASE, (char *)fp + fp->mp_upper + PAGEBASE, + olddata.mv_size - fp->mp_upper - PAGEBASE); + for (i=0; i<NUMKEYS(fp); i++) + mp->mp_ptrs[i] = fp->mp_ptrs[i] + offset; + } + } + + rdata = &xdata; + flags |= F_DUPDATA; + do_sub = 1; + if (!insert_key) + mdb_node_del(mc, 0); + goto new_sub; + } +current: + /* overflow page overwrites need special handling */ + if (F_ISSET(leaf->mn_flags, F_BIGDATA)) { + MDB_page *omp; + pgno_t pg; + int level, ovpages, dpages = OVPAGES(data->mv_size, env->me_psize); + + memcpy(&pg, olddata.mv_data, sizeof(pg)); + if ((rc2 = mdb_page_get(mc->mc_txn, pg, &omp, &level)) != 0) + return rc2; + ovpages = omp->mp_pages; + + /* Is the ov page large enough? */ + if (ovpages >= dpages) { + if (!(omp->mp_flags & P_DIRTY) && + (level || (env->me_flags & MDB_WRITEMAP))) + { + rc = mdb_page_unspill(mc->mc_txn, omp, &omp); + if (rc) + return rc; + level = 0; /* dirty in this txn or clean */ + } + /* Is it dirty? */ + if (omp->mp_flags & P_DIRTY) { + /* yes, overwrite it. Note in this case we don't + * bother to try shrinking the page if the new data + * is smaller than the overflow threshold. + */ + if (level > 1) { + /* It is writable only in a parent txn */ + size_t sz = (size_t) env->me_psize * ovpages, off; + MDB_page *np = mdb_page_malloc(mc->mc_txn, ovpages); + MDB_ID2 id2; + if (!np) + return ENOMEM; + id2.mid = pg; + id2.mptr = np; + rc2 = mdb_mid2l_insert(mc->mc_txn->mt_u.dirty_list, &id2); + mdb_cassert(mc, rc2 == 0); + if (!(flags & MDB_RESERVE)) { + /* Copy end of page, adjusting alignment so + * compiler may copy words instead of bytes. + */ + off = (PAGEHDRSZ + data->mv_size) & -sizeof(size_t); + memcpy((size_t *)((char *)np + off), + (size_t *)((char *)omp + off), sz - off); + sz = PAGEHDRSZ; + } + memcpy(np, omp, sz); /* Copy beginning of page */ + omp = np; + } + SETDSZ(leaf, data->mv_size); + if (F_ISSET(flags, MDB_RESERVE)) + data->mv_data = METADATA(omp); + else + memcpy(METADATA(omp), data->mv_data, data->mv_size); + return MDB_SUCCESS; + } + } + if ((rc2 = mdb_ovpage_free(mc, omp)) != MDB_SUCCESS) + return rc2; + } else if (data->mv_size == olddata.mv_size) { + /* same size, just replace it. Note that we could + * also reuse this node if the new data is smaller, + * but instead we opt to shrink the node in that case. + */ + if (F_ISSET(flags, MDB_RESERVE)) + data->mv_data = olddata.mv_data; + else if (!(mc->mc_flags & C_SUB)) + memcpy(olddata.mv_data, data->mv_data, data->mv_size); + else { + memcpy(NODEKEY(leaf), key->mv_data, key->mv_size); + goto fix_parent; + } + return MDB_SUCCESS; + } + mdb_node_del(mc, 0); + } + + rdata = data; + +new_sub: + nflags = flags & NODE_ADD_FLAGS; + nsize = IS_LEAF2(mc->mc_pg[mc->mc_top]) ? key->mv_size : mdb_leaf_size(env, key, rdata); + if (SIZELEFT(mc->mc_pg[mc->mc_top]) < nsize) { + if (( flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA ) + nflags &= ~MDB_APPEND; /* sub-page may need room to grow */ + if (!insert_key) + nflags |= MDB_SPLIT_REPLACE; + rc = mdb_page_split(mc, key, rdata, P_INVALID, nflags); + } else { + /* There is room already in this leaf page. */ + rc = mdb_node_add(mc, mc->mc_ki[mc->mc_top], key, rdata, 0, nflags); + if (rc == 0 && insert_key) { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2, *m3; + MDB_dbi dbi = mc->mc_dbi; + unsigned i = mc->mc_top; + MDB_page *mp = mc->mc_pg[i]; + + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (mc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3 == mc || m3->mc_snum < mc->mc_snum) continue; + if (m3->mc_pg[i] == mp && m3->mc_ki[i] >= mc->mc_ki[i]) { + m3->mc_ki[i]++; + } + } + } + } + + if (rc == MDB_SUCCESS) { + /* Now store the actual data in the child DB. Note that we're + * storing the user data in the keys field, so there are strict + * size limits on dupdata. The actual data fields of the child + * DB are all zero size. + */ + if (do_sub) { + int xflags; + size_t ecount; +put_sub: + xdata.mv_size = 0; + xdata.mv_data = ""; + leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if (flags & MDB_CURRENT) { + xflags = MDB_CURRENT|MDB_NOSPILL; + } else { + mdb_xcursor_init1(mc, leaf); + xflags = (flags & MDB_NODUPDATA) ? + MDB_NOOVERWRITE|MDB_NOSPILL : MDB_NOSPILL; + } + /* converted, write the original data first */ + if (dkey.mv_size) { + rc = mdb_cursor_put(&mc->mc_xcursor->mx_cursor, &dkey, &xdata, xflags); + if (rc) + goto bad_sub; + { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2; + unsigned i = mc->mc_top; + MDB_page *mp = mc->mc_pg[i]; + + for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { + if (m2 == mc || m2->mc_snum < mc->mc_snum) continue; + if (!(m2->mc_flags & C_INITIALIZED)) continue; + if (m2->mc_pg[i] == mp && m2->mc_ki[i] == mc->mc_ki[i]) { + mdb_xcursor_init1(m2, leaf); + } + } + } + /* we've done our job */ + dkey.mv_size = 0; + } + ecount = mc->mc_xcursor->mx_db.md_entries; + if (flags & MDB_APPENDDUP) + xflags |= MDB_APPEND; + rc = mdb_cursor_put(&mc->mc_xcursor->mx_cursor, data, &xdata, xflags); + if (flags & F_SUBDATA) { + void *db = NODEDATA(leaf); + memcpy(db, &mc->mc_xcursor->mx_db, sizeof(MDB_db)); + } + insert_data = mc->mc_xcursor->mx_db.md_entries - ecount; + } + /* Increment count unless we just replaced an existing item. */ + if (insert_data) + mc->mc_db->md_entries++; + if (insert_key) { + /* Invalidate txn if we created an empty sub-DB */ + if (rc) + goto bad_sub; + /* If we succeeded and the key didn't exist before, + * make sure the cursor is marked valid. + */ + mc->mc_flags |= C_INITIALIZED; + } + if (flags & MDB_MULTIPLE) { + if (!rc) { + mcount++; + /* let caller know how many succeeded, if any */ + data[1].mv_size = mcount; + if (mcount < dcount) { + data[0].mv_data = (char *)data[0].mv_data + data[0].mv_size; + insert_key = insert_data = 0; + goto more; + } + } + } + return rc; +bad_sub: + if (rc == MDB_KEYEXIST) /* should not happen, we deleted that item */ + rc = MDB_CORRUPTED; + } + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return rc; +} + +int +mdb_cursor_del(MDB_cursor *mc, unsigned int flags) +{ + MDB_node *leaf; + MDB_page *mp; + int rc; + + if (mc->mc_txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_ERROR)) + return (mc->mc_txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN; + + if (!(mc->mc_flags & C_INITIALIZED)) + return EINVAL; + + if (mc->mc_ki[mc->mc_top] >= NUMKEYS(mc->mc_pg[mc->mc_top])) + return MDB_NOTFOUND; + + if (!(flags & MDB_NOSPILL) && (rc = mdb_page_spill(mc, NULL, NULL))) + return rc; + + rc = mdb_cursor_touch(mc); + if (rc) + return rc; + + mp = mc->mc_pg[mc->mc_top]; + if (IS_LEAF2(mp)) + goto del_key; + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + + if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { + if (flags & MDB_NODUPDATA) { + /* mdb_cursor_del0() will subtract the final entry */ + mc->mc_db->md_entries -= mc->mc_xcursor->mx_db.md_entries - 1; + } else { + if (!F_ISSET(leaf->mn_flags, F_SUBDATA)) { + mc->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); + } + rc = mdb_cursor_del(&mc->mc_xcursor->mx_cursor, MDB_NOSPILL); + if (rc) + return rc; + /* If sub-DB still has entries, we're done */ + if (mc->mc_xcursor->mx_db.md_entries) { + if (leaf->mn_flags & F_SUBDATA) { + /* update subDB info */ + void *db = NODEDATA(leaf); + memcpy(db, &mc->mc_xcursor->mx_db, sizeof(MDB_db)); + } else { + MDB_cursor *m2; + /* shrink fake page */ + mdb_node_shrink(mp, mc->mc_ki[mc->mc_top]); + leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + mc->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); + /* fix other sub-DB cursors pointed at this fake page */ + for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { + if (m2 == mc || m2->mc_snum < mc->mc_snum) continue; + if (m2->mc_pg[mc->mc_top] == mp && + m2->mc_ki[mc->mc_top] == mc->mc_ki[mc->mc_top]) + m2->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); + } + } + mc->mc_db->md_entries--; + mc->mc_flags |= C_DEL; + return rc; + } + /* otherwise fall thru and delete the sub-DB */ + } + + if (leaf->mn_flags & F_SUBDATA) { + /* add all the child DB's pages to the free list */ + rc = mdb_drop0(&mc->mc_xcursor->mx_cursor, 0); + if (rc) + goto fail; + } + } + + /* add overflow pages to free list */ + if (F_ISSET(leaf->mn_flags, F_BIGDATA)) { + MDB_page *omp; + pgno_t pg; + + memcpy(&pg, NODEDATA(leaf), sizeof(pg)); + if ((rc = mdb_page_get(mc->mc_txn, pg, &omp, NULL)) || + (rc = mdb_ovpage_free(mc, omp))) + goto fail; + } + +del_key: + return mdb_cursor_del0(mc); + +fail: + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return rc; +} + +/** Allocate and initialize new pages for a database. + * @param[in] mc a cursor on the database being added to. + * @param[in] flags flags defining what type of page is being allocated. + * @param[in] num the number of pages to allocate. This is usually 1, + * unless allocating overflow pages for a large record. + * @param[out] mp Address of a page, or NULL on failure. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp) +{ + MDB_page *np; + int rc; + + if ((rc = mdb_page_alloc(mc, num, &np))) + return rc; + DPRINTF(("allocated new mpage %"Z"u, page size %u", + np->mp_pgno, mc->mc_txn->mt_env->me_psize)); + np->mp_flags = flags | P_DIRTY; + np->mp_lower = (PAGEHDRSZ-PAGEBASE); + np->mp_upper = mc->mc_txn->mt_env->me_psize - PAGEBASE; + + if (IS_BRANCH(np)) + mc->mc_db->md_branch_pages++; + else if (IS_LEAF(np)) + mc->mc_db->md_leaf_pages++; + else if (IS_OVERFLOW(np)) { + mc->mc_db->md_overflow_pages += num; + np->mp_pages = num; + } + *mp = np; + + return 0; +} + +/** Calculate the size of a leaf node. + * The size depends on the environment's page size; if a data item + * is too large it will be put onto an overflow page and the node + * size will only include the key and not the data. Sizes are always + * rounded up to an even number of bytes, to guarantee 2-byte alignment + * of the #MDB_node headers. + * @param[in] env The environment handle. + * @param[in] key The key for the node. + * @param[in] data The data for the node. + * @return The number of bytes needed to store the node. + */ +static size_t +mdb_leaf_size(MDB_env *env, MDB_val *key, MDB_val *data) +{ + size_t sz; + + sz = LEAFSIZE(key, data); + if (sz > env->me_nodemax) { + /* put on overflow page */ + sz -= data->mv_size - sizeof(pgno_t); + } + + return EVEN(sz + sizeof(indx_t)); +} + +/** Calculate the size of a branch node. + * The size should depend on the environment's page size but since + * we currently don't support spilling large keys onto overflow + * pages, it's simply the size of the #MDB_node header plus the + * size of the key. Sizes are always rounded up to an even number + * of bytes, to guarantee 2-byte alignment of the #MDB_node headers. + * @param[in] env The environment handle. + * @param[in] key The key for the node. + * @return The number of bytes needed to store the node. + */ +static size_t +mdb_branch_size(MDB_env *env, MDB_val *key) +{ + size_t sz; + + sz = INDXSIZE(key); + if (sz > env->me_nodemax) { + /* put on overflow page */ + /* not implemented */ + /* sz -= key->size - sizeof(pgno_t); */ + } + + return sz + sizeof(indx_t); +} + +/** Add a node to the page pointed to by the cursor. + * @param[in] mc The cursor for this operation. + * @param[in] indx The index on the page where the new node should be added. + * @param[in] key The key for the new node. + * @param[in] data The data for the new node, if any. + * @param[in] pgno The page number, if adding a branch node. + * @param[in] flags Flags for the node. + * @return 0 on success, non-zero on failure. Possible errors are: + * <ul> + * <li>ENOMEM - failed to allocate overflow pages for the node. + * <li>MDB_PAGE_FULL - there is insufficient room in the page. This error + * should never happen since all callers already calculate the + * page's free space before calling this function. + * </ul> + */ +static int +mdb_node_add(MDB_cursor *mc, indx_t indx, + MDB_val *key, MDB_val *data, pgno_t pgno, unsigned int flags) +{ + unsigned int i; + size_t node_size = NODESIZE; + ssize_t room; + indx_t ofs; + MDB_node *node; + MDB_page *mp = mc->mc_pg[mc->mc_top]; + MDB_page *ofp = NULL; /* overflow page */ + DKBUF; + + mdb_cassert(mc, mp->mp_upper >= mp->mp_lower); + + DPRINTF(("add to %s %spage %"Z"u index %i, data size %"Z"u key size %"Z"u [%s]", + IS_LEAF(mp) ? "leaf" : "branch", + IS_SUBP(mp) ? "sub-" : "", + mdb_dbg_pgno(mp), indx, data ? data->mv_size : 0, + key ? key->mv_size : 0, key ? DKEY(key) : "null")); + + if (IS_LEAF2(mp)) { + /* Move higher keys up one slot. */ + int ksize = mc->mc_db->md_pad, dif; + char *ptr = LEAF2KEY(mp, indx, ksize); + dif = NUMKEYS(mp) - indx; + if (dif > 0) + memmove(ptr+ksize, ptr, dif*ksize); + /* insert new key */ + memcpy(ptr, key->mv_data, ksize); + + /* Just using these for counting */ + mp->mp_lower += sizeof(indx_t); + mp->mp_upper -= ksize - sizeof(indx_t); + return MDB_SUCCESS; + } + + room = (ssize_t)SIZELEFT(mp) - (ssize_t)sizeof(indx_t); + if (key != NULL) + node_size += key->mv_size; + if (IS_LEAF(mp)) { + mdb_cassert(mc, data); + if (F_ISSET(flags, F_BIGDATA)) { + /* Data already on overflow page. */ + node_size += sizeof(pgno_t); + } else if (node_size + data->mv_size > mc->mc_txn->mt_env->me_nodemax) { + int ovpages = OVPAGES(data->mv_size, mc->mc_txn->mt_env->me_psize); + int rc; + /* Put data on overflow page. */ + DPRINTF(("data size is %"Z"u, node would be %"Z"u, put data on overflow page", + data->mv_size, node_size+data->mv_size)); + node_size = EVEN(node_size + sizeof(pgno_t)); + if ((ssize_t)node_size > room) + goto full; + if ((rc = mdb_page_new(mc, P_OVERFLOW, ovpages, &ofp))) + return rc; + DPRINTF(("allocated overflow page %"Z"u", ofp->mp_pgno)); + flags |= F_BIGDATA; + goto update; + } else { + node_size += data->mv_size; + } + } + node_size = EVEN(node_size); + if ((ssize_t)node_size > room) + goto full; + +update: + /* Move higher pointers up one slot. */ + for (i = NUMKEYS(mp); i > indx; i--) + mp->mp_ptrs[i] = mp->mp_ptrs[i - 1]; + + /* Adjust free space offsets. */ + ofs = mp->mp_upper - node_size; + mdb_cassert(mc, ofs >= mp->mp_lower + sizeof(indx_t)); + mp->mp_ptrs[indx] = ofs; + mp->mp_upper = ofs; + mp->mp_lower += sizeof(indx_t); + + /* Write the node data. */ + node = NODEPTR(mp, indx); + node->mn_ksize = (key == NULL) ? 0 : key->mv_size; + node->mn_flags = flags; + if (IS_LEAF(mp)) + SETDSZ(node,data->mv_size); + else + SETPGNO(node,pgno); + + if (key) + memcpy(NODEKEY(node), key->mv_data, key->mv_size); + + if (IS_LEAF(mp)) { + mdb_cassert(mc, key); + if (ofp == NULL) { + if (F_ISSET(flags, F_BIGDATA)) + memcpy(node->mn_data + key->mv_size, data->mv_data, + sizeof(pgno_t)); + else if (F_ISSET(flags, MDB_RESERVE)) + data->mv_data = node->mn_data + key->mv_size; + else + memcpy(node->mn_data + key->mv_size, data->mv_data, + data->mv_size); + } else { + memcpy(node->mn_data + key->mv_size, &ofp->mp_pgno, + sizeof(pgno_t)); + if (F_ISSET(flags, MDB_RESERVE)) + data->mv_data = METADATA(ofp); + else + memcpy(METADATA(ofp), data->mv_data, data->mv_size); + } + } + + return MDB_SUCCESS; + +full: + DPRINTF(("not enough room in page %"Z"u, got %u ptrs", + mdb_dbg_pgno(mp), NUMKEYS(mp))); + DPRINTF(("upper-lower = %u - %u = %"Z"d", mp->mp_upper,mp->mp_lower,room)); + DPRINTF(("node size = %"Z"u", node_size)); + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return MDB_PAGE_FULL; +} + +/** Delete the specified node from a page. + * @param[in] mc Cursor pointing to the node to delete. + * @param[in] ksize The size of a node. Only used if the page is + * part of a #MDB_DUPFIXED database. + */ +static void +mdb_node_del(MDB_cursor *mc, int ksize) +{ + MDB_page *mp = mc->mc_pg[mc->mc_top]; + indx_t indx = mc->mc_ki[mc->mc_top]; + unsigned int sz; + indx_t i, j, numkeys, ptr; + MDB_node *node; + char *base; + + DPRINTF(("delete node %u on %s page %"Z"u", indx, + IS_LEAF(mp) ? "leaf" : "branch", mdb_dbg_pgno(mp))); + numkeys = NUMKEYS(mp); + mdb_cassert(mc, indx < numkeys); + + if (IS_LEAF2(mp)) { + int x = numkeys - 1 - indx; + base = LEAF2KEY(mp, indx, ksize); + if (x) + memmove(base, base + ksize, x * ksize); + mp->mp_lower -= sizeof(indx_t); + mp->mp_upper += ksize - sizeof(indx_t); + return; + } + + node = NODEPTR(mp, indx); + sz = NODESIZE + node->mn_ksize; + if (IS_LEAF(mp)) { + if (F_ISSET(node->mn_flags, F_BIGDATA)) + sz += sizeof(pgno_t); + else + sz += NODEDSZ(node); + } + sz = EVEN(sz); + + ptr = mp->mp_ptrs[indx]; + for (i = j = 0; i < numkeys; i++) { + if (i != indx) { + mp->mp_ptrs[j] = mp->mp_ptrs[i]; + if (mp->mp_ptrs[i] < ptr) + mp->mp_ptrs[j] += sz; + j++; + } + } + + base = (char *)mp + mp->mp_upper + PAGEBASE; + memmove(base + sz, base, ptr - mp->mp_upper); + + mp->mp_lower -= sizeof(indx_t); + mp->mp_upper += sz; +} + +/** Compact the main page after deleting a node on a subpage. + * @param[in] mp The main page to operate on. + * @param[in] indx The index of the subpage on the main page. + */ +static void +mdb_node_shrink(MDB_page *mp, indx_t indx) +{ + MDB_node *node; + MDB_page *sp, *xp; + char *base; + int nsize, delta; + indx_t i, numkeys, ptr; + + node = NODEPTR(mp, indx); + sp = (MDB_page *)NODEDATA(node); + delta = SIZELEFT(sp); + xp = (MDB_page *)((char *)sp + delta); + + /* shift subpage upward */ + if (IS_LEAF2(sp)) { + nsize = NUMKEYS(sp) * sp->mp_pad; + if (nsize & 1) + return; /* do not make the node uneven-sized */ + memmove(METADATA(xp), METADATA(sp), nsize); + } else { + int i; + numkeys = NUMKEYS(sp); + for (i=numkeys-1; i>=0; i--) + xp->mp_ptrs[i] = sp->mp_ptrs[i] - delta; + } + xp->mp_upper = sp->mp_lower; + xp->mp_lower = sp->mp_lower; + xp->mp_flags = sp->mp_flags; + xp->mp_pad = sp->mp_pad; + COPY_PGNO(xp->mp_pgno, mp->mp_pgno); + + nsize = NODEDSZ(node) - delta; + SETDSZ(node, nsize); + + /* shift lower nodes upward */ + ptr = mp->mp_ptrs[indx]; + numkeys = NUMKEYS(mp); + for (i = 0; i < numkeys; i++) { + if (mp->mp_ptrs[i] <= ptr) + mp->mp_ptrs[i] += delta; + } + + base = (char *)mp + mp->mp_upper + PAGEBASE; + memmove(base + delta, base, ptr - mp->mp_upper + NODESIZE + NODEKSZ(node)); + mp->mp_upper += delta; +} + +/** Initial setup of a sorted-dups cursor. + * Sorted duplicates are implemented as a sub-database for the given key. + * The duplicate data items are actually keys of the sub-database. + * Operations on the duplicate data items are performed using a sub-cursor + * initialized when the sub-database is first accessed. This function does + * the preliminary setup of the sub-cursor, filling in the fields that + * depend only on the parent DB. + * @param[in] mc The main cursor whose sorted-dups cursor is to be initialized. + */ +static void +mdb_xcursor_init0(MDB_cursor *mc) +{ + MDB_xcursor *mx = mc->mc_xcursor; + + mx->mx_cursor.mc_xcursor = NULL; + mx->mx_cursor.mc_txn = mc->mc_txn; + mx->mx_cursor.mc_db = &mx->mx_db; + mx->mx_cursor.mc_dbx = &mx->mx_dbx; + mx->mx_cursor.mc_dbi = mc->mc_dbi; + mx->mx_cursor.mc_dbflag = &mx->mx_dbflag; + mx->mx_cursor.mc_snum = 0; + mx->mx_cursor.mc_top = 0; + mx->mx_cursor.mc_flags = C_SUB; + mx->mx_dbx.md_name.mv_size = 0; + mx->mx_dbx.md_name.mv_data = NULL; + mx->mx_dbx.md_cmp = mc->mc_dbx->md_dcmp; + mx->mx_dbx.md_dcmp = NULL; + mx->mx_dbx.md_rel = mc->mc_dbx->md_rel; +} + +/** Final setup of a sorted-dups cursor. + * Sets up the fields that depend on the data from the main cursor. + * @param[in] mc The main cursor whose sorted-dups cursor is to be initialized. + * @param[in] node The data containing the #MDB_db record for the + * sorted-dup database. + */ +static void +mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node) +{ + MDB_xcursor *mx = mc->mc_xcursor; + + if (node->mn_flags & F_SUBDATA) { + memcpy(&mx->mx_db, NODEDATA(node), sizeof(MDB_db)); + mx->mx_cursor.mc_pg[0] = 0; + mx->mx_cursor.mc_snum = 0; + mx->mx_cursor.mc_top = 0; + mx->mx_cursor.mc_flags = C_SUB; + } else { + MDB_page *fp = NODEDATA(node); + mx->mx_db.md_pad = mc->mc_pg[mc->mc_top]->mp_pad; + mx->mx_db.md_flags = 0; + mx->mx_db.md_depth = 1; + mx->mx_db.md_branch_pages = 0; + mx->mx_db.md_leaf_pages = 1; + mx->mx_db.md_overflow_pages = 0; + mx->mx_db.md_entries = NUMKEYS(fp); + COPY_PGNO(mx->mx_db.md_root, fp->mp_pgno); + mx->mx_cursor.mc_snum = 1; + mx->mx_cursor.mc_top = 0; + mx->mx_cursor.mc_flags = C_INITIALIZED|C_SUB; + mx->mx_cursor.mc_pg[0] = fp; + mx->mx_cursor.mc_ki[0] = 0; + if (mc->mc_db->md_flags & MDB_DUPFIXED) { + mx->mx_db.md_flags = MDB_DUPFIXED; + mx->mx_db.md_pad = fp->mp_pad; + if (mc->mc_db->md_flags & MDB_INTEGERDUP) + mx->mx_db.md_flags |= MDB_INTEGERKEY; + } + } + DPRINTF(("Sub-db -%u root page %"Z"u", mx->mx_cursor.mc_dbi, + mx->mx_db.md_root)); + mx->mx_dbflag = DB_VALID|DB_DIRTY; /* DB_DIRTY guides mdb_cursor_touch */ +#if UINT_MAX < SIZE_MAX + if (mx->mx_dbx.md_cmp == mdb_cmp_int && mx->mx_db.md_pad == sizeof(size_t)) + mx->mx_dbx.md_cmp = mdb_cmp_clong; +#endif +} + +/** Initialize a cursor for a given transaction and database. */ +static void +mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx) +{ + mc->mc_next = NULL; + mc->mc_backup = NULL; + mc->mc_dbi = dbi; + mc->mc_txn = txn; + mc->mc_db = &txn->mt_dbs[dbi]; + mc->mc_dbx = &txn->mt_dbxs[dbi]; + mc->mc_dbflag = &txn->mt_dbflags[dbi]; + mc->mc_snum = 0; + mc->mc_top = 0; + mc->mc_pg[0] = 0; + mc->mc_flags = 0; + if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) { + mdb_tassert(txn, mx != NULL); + mc->mc_xcursor = mx; + mdb_xcursor_init0(mc); + } else { + mc->mc_xcursor = NULL; + } + if (*mc->mc_dbflag & DB_STALE) { + mdb_page_search(mc, NULL, MDB_PS_ROOTONLY); + } +} + +int +mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **ret) +{ + MDB_cursor *mc; + size_t size = sizeof(MDB_cursor); + + if (!ret || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + + if (txn->mt_flags & MDB_TXN_ERROR) + return MDB_BAD_TXN; + + /* Allow read access to the freelist */ + if (!dbi && !F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) + return EINVAL; + + if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) + size += sizeof(MDB_xcursor); + + if ((mc = malloc(size)) != NULL) { + mdb_cursor_init(mc, txn, dbi, (MDB_xcursor *)(mc + 1)); + if (txn->mt_cursors) { + mc->mc_next = txn->mt_cursors[dbi]; + txn->mt_cursors[dbi] = mc; + mc->mc_flags |= C_UNTRACK; + } + } else { + return ENOMEM; + } + + *ret = mc; + + return MDB_SUCCESS; +} + +int +mdb_cursor_renew(MDB_txn *txn, MDB_cursor *mc) +{ + if (!mc || !TXN_DBI_EXIST(txn, mc->mc_dbi)) + return EINVAL; + + if ((mc->mc_flags & C_UNTRACK) || txn->mt_cursors) + return EINVAL; + + if (txn->mt_flags & MDB_TXN_ERROR) + return MDB_BAD_TXN; + + mdb_cursor_init(mc, txn, mc->mc_dbi, mc->mc_xcursor); + return MDB_SUCCESS; +} + +/* Return the count of duplicate data items for the current key */ +int +mdb_cursor_count(MDB_cursor *mc, size_t *countp) +{ + MDB_node *leaf; + + if (mc == NULL || countp == NULL) + return EINVAL; + + if (mc->mc_xcursor == NULL) + return MDB_INCOMPATIBLE; + + if (mc->mc_txn->mt_flags & MDB_TXN_ERROR) + return MDB_BAD_TXN; + + if (!(mc->mc_flags & C_INITIALIZED)) + return EINVAL; + + if (!mc->mc_snum || (mc->mc_flags & C_EOF)) + return MDB_NOTFOUND; + + leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); + if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { + *countp = 1; + } else { + if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) + return EINVAL; + + *countp = mc->mc_xcursor->mx_db.md_entries; + } + return MDB_SUCCESS; +} + +void +mdb_cursor_close(MDB_cursor *mc) +{ + if (mc && !mc->mc_backup) { + /* remove from txn, if tracked */ + if ((mc->mc_flags & C_UNTRACK) && mc->mc_txn->mt_cursors) { + MDB_cursor **prev = &mc->mc_txn->mt_cursors[mc->mc_dbi]; + while (*prev && *prev != mc) prev = &(*prev)->mc_next; + if (*prev == mc) + *prev = mc->mc_next; + } + free(mc); + } +} + +MDB_txn * +mdb_cursor_txn(MDB_cursor *mc) +{ + if (!mc) return NULL; + return mc->mc_txn; +} + +MDB_dbi +mdb_cursor_dbi(MDB_cursor *mc) +{ + return mc->mc_dbi; +} + +/** Replace the key for a branch node with a new key. + * @param[in] mc Cursor pointing to the node to operate on. + * @param[in] key The new key to use. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_update_key(MDB_cursor *mc, MDB_val *key) +{ + MDB_page *mp; + MDB_node *node; + char *base; + size_t len; + int delta, ksize, oksize; + indx_t ptr, i, numkeys, indx; + DKBUF; + + indx = mc->mc_ki[mc->mc_top]; + mp = mc->mc_pg[mc->mc_top]; + node = NODEPTR(mp, indx); + ptr = mp->mp_ptrs[indx]; +#if MDB_DEBUG + { + MDB_val k2; + char kbuf2[DKBUF_MAXKEYSIZE*2+1]; + k2.mv_data = NODEKEY(node); + k2.mv_size = node->mn_ksize; + DPRINTF(("update key %u (ofs %u) [%s] to [%s] on page %"Z"u", + indx, ptr, + mdb_dkey(&k2, kbuf2), + DKEY(key), + mp->mp_pgno)); + } +#endif + + /* Sizes must be 2-byte aligned. */ + ksize = EVEN(key->mv_size); + oksize = EVEN(node->mn_ksize); + delta = ksize - oksize; + + /* Shift node contents if EVEN(key length) changed. */ + if (delta) { + if (delta > 0 && SIZELEFT(mp) < delta) { + pgno_t pgno; + /* not enough space left, do a delete and split */ + DPRINTF(("Not enough room, delta = %d, splitting...", delta)); + pgno = NODEPGNO(node); + mdb_node_del(mc, 0); + return mdb_page_split(mc, key, NULL, pgno, MDB_SPLIT_REPLACE); + } + + numkeys = NUMKEYS(mp); + for (i = 0; i < numkeys; i++) { + if (mp->mp_ptrs[i] <= ptr) + mp->mp_ptrs[i] -= delta; + } + + base = (char *)mp + mp->mp_upper + PAGEBASE; + len = ptr - mp->mp_upper + NODESIZE; + memmove(base - delta, base, len); + mp->mp_upper -= delta; + + node = NODEPTR(mp, indx); + } + + /* But even if no shift was needed, update ksize */ + if (node->mn_ksize != key->mv_size) + node->mn_ksize = key->mv_size; + + if (key->mv_size) + memcpy(NODEKEY(node), key->mv_data, key->mv_size); + + return MDB_SUCCESS; +} + +static void +mdb_cursor_copy(const MDB_cursor *csrc, MDB_cursor *cdst); + +/** Move a node from csrc to cdst. + */ +static int +mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst) +{ + MDB_node *srcnode; + MDB_val key, data; + pgno_t srcpg; + MDB_cursor mn; + int rc; + unsigned short flags; + + DKBUF; + + /* Mark src and dst as dirty. */ + if ((rc = mdb_page_touch(csrc)) || + (rc = mdb_page_touch(cdst))) + return rc; + + if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { + key.mv_size = csrc->mc_db->md_pad; + key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], csrc->mc_ki[csrc->mc_top], key.mv_size); + data.mv_size = 0; + data.mv_data = NULL; + srcpg = 0; + flags = 0; + } else { + srcnode = NODEPTR(csrc->mc_pg[csrc->mc_top], csrc->mc_ki[csrc->mc_top]); + mdb_cassert(csrc, !((size_t)srcnode & 1)); + srcpg = NODEPGNO(srcnode); + flags = srcnode->mn_flags; + if (csrc->mc_ki[csrc->mc_top] == 0 && IS_BRANCH(csrc->mc_pg[csrc->mc_top])) { + unsigned int snum = csrc->mc_snum; + MDB_node *s2; + /* must find the lowest key below src */ + rc = mdb_page_search_lowest(csrc); + if (rc) + return rc; + if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { + key.mv_size = csrc->mc_db->md_pad; + key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], 0, key.mv_size); + } else { + s2 = NODEPTR(csrc->mc_pg[csrc->mc_top], 0); + key.mv_size = NODEKSZ(s2); + key.mv_data = NODEKEY(s2); + } + csrc->mc_snum = snum--; + csrc->mc_top = snum; + } else { + key.mv_size = NODEKSZ(srcnode); + key.mv_data = NODEKEY(srcnode); + } + data.mv_size = NODEDSZ(srcnode); + data.mv_data = NODEDATA(srcnode); + } + if (IS_BRANCH(cdst->mc_pg[cdst->mc_top]) && cdst->mc_ki[cdst->mc_top] == 0) { + unsigned int snum = cdst->mc_snum; + MDB_node *s2; + MDB_val bkey; + /* must find the lowest key below dst */ + mdb_cursor_copy(cdst, &mn); + rc = mdb_page_search_lowest(&mn); + if (rc) + return rc; + if (IS_LEAF2(mn.mc_pg[mn.mc_top])) { + bkey.mv_size = mn.mc_db->md_pad; + bkey.mv_data = LEAF2KEY(mn.mc_pg[mn.mc_top], 0, bkey.mv_size); + } else { + s2 = NODEPTR(mn.mc_pg[mn.mc_top], 0); + bkey.mv_size = NODEKSZ(s2); + bkey.mv_data = NODEKEY(s2); + } + mn.mc_snum = snum--; + mn.mc_top = snum; + mn.mc_ki[snum] = 0; + rc = mdb_update_key(&mn, &bkey); + if (rc) + return rc; + } + + DPRINTF(("moving %s node %u [%s] on page %"Z"u to node %u on page %"Z"u", + IS_LEAF(csrc->mc_pg[csrc->mc_top]) ? "leaf" : "branch", + csrc->mc_ki[csrc->mc_top], + DKEY(&key), + csrc->mc_pg[csrc->mc_top]->mp_pgno, + cdst->mc_ki[cdst->mc_top], cdst->mc_pg[cdst->mc_top]->mp_pgno)); + + /* Add the node to the destination page. + */ + rc = mdb_node_add(cdst, cdst->mc_ki[cdst->mc_top], &key, &data, srcpg, flags); + if (rc != MDB_SUCCESS) + return rc; + + /* Delete the node from the source page. + */ + mdb_node_del(csrc, key.mv_size); + + { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2, *m3; + MDB_dbi dbi = csrc->mc_dbi; + MDB_page *mp = csrc->mc_pg[csrc->mc_top]; + + for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (csrc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3 == csrc) continue; + if (m3->mc_pg[csrc->mc_top] == mp && m3->mc_ki[csrc->mc_top] == + csrc->mc_ki[csrc->mc_top]) { + m3->mc_pg[csrc->mc_top] = cdst->mc_pg[cdst->mc_top]; + m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top]; + } + } + } + + /* Update the parent separators. + */ + if (csrc->mc_ki[csrc->mc_top] == 0) { + if (csrc->mc_ki[csrc->mc_top-1] != 0) { + if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { + key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], 0, key.mv_size); + } else { + srcnode = NODEPTR(csrc->mc_pg[csrc->mc_top], 0); + key.mv_size = NODEKSZ(srcnode); + key.mv_data = NODEKEY(srcnode); + } + DPRINTF(("update separator for source page %"Z"u to [%s]", + csrc->mc_pg[csrc->mc_top]->mp_pgno, DKEY(&key))); + mdb_cursor_copy(csrc, &mn); + mn.mc_snum--; + mn.mc_top--; + if ((rc = mdb_update_key(&mn, &key)) != MDB_SUCCESS) + return rc; + } + if (IS_BRANCH(csrc->mc_pg[csrc->mc_top])) { + MDB_val nullkey; + indx_t ix = csrc->mc_ki[csrc->mc_top]; + nullkey.mv_size = 0; + csrc->mc_ki[csrc->mc_top] = 0; + rc = mdb_update_key(csrc, &nullkey); + csrc->mc_ki[csrc->mc_top] = ix; + mdb_cassert(csrc, rc == MDB_SUCCESS); + } + } + + if (cdst->mc_ki[cdst->mc_top] == 0) { + if (cdst->mc_ki[cdst->mc_top-1] != 0) { + if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { + key.mv_data = LEAF2KEY(cdst->mc_pg[cdst->mc_top], 0, key.mv_size); + } else { + srcnode = NODEPTR(cdst->mc_pg[cdst->mc_top], 0); + key.mv_size = NODEKSZ(srcnode); + key.mv_data = NODEKEY(srcnode); + } + DPRINTF(("update separator for destination page %"Z"u to [%s]", + cdst->mc_pg[cdst->mc_top]->mp_pgno, DKEY(&key))); + mdb_cursor_copy(cdst, &mn); + mn.mc_snum--; + mn.mc_top--; + if ((rc = mdb_update_key(&mn, &key)) != MDB_SUCCESS) + return rc; + } + if (IS_BRANCH(cdst->mc_pg[cdst->mc_top])) { + MDB_val nullkey; + indx_t ix = cdst->mc_ki[cdst->mc_top]; + nullkey.mv_size = 0; + cdst->mc_ki[cdst->mc_top] = 0; + rc = mdb_update_key(cdst, &nullkey); + cdst->mc_ki[cdst->mc_top] = ix; + mdb_cassert(cdst, rc == MDB_SUCCESS); + } + } + + return MDB_SUCCESS; +} + +/** Merge one page into another. + * The nodes from the page pointed to by \b csrc will + * be copied to the page pointed to by \b cdst and then + * the \b csrc page will be freed. + * @param[in] csrc Cursor pointing to the source page. + * @param[in] cdst Cursor pointing to the destination page. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst) +{ + MDB_page *psrc, *pdst; + MDB_node *srcnode; + MDB_val key, data; + unsigned nkeys; + int rc; + indx_t i, j; + + psrc = csrc->mc_pg[csrc->mc_top]; + pdst = cdst->mc_pg[cdst->mc_top]; + + DPRINTF(("merging page %"Z"u into %"Z"u", psrc->mp_pgno, pdst->mp_pgno)); + + mdb_cassert(csrc, csrc->mc_snum > 1); /* can't merge root page */ + mdb_cassert(csrc, cdst->mc_snum > 1); + + /* Mark dst as dirty. */ + if ((rc = mdb_page_touch(cdst))) + return rc; + + /* Move all nodes from src to dst. + */ + j = nkeys = NUMKEYS(pdst); + if (IS_LEAF2(psrc)) { + key.mv_size = csrc->mc_db->md_pad; + key.mv_data = METADATA(psrc); + for (i = 0; i < NUMKEYS(psrc); i++, j++) { + rc = mdb_node_add(cdst, j, &key, NULL, 0, 0); + if (rc != MDB_SUCCESS) + return rc; + key.mv_data = (char *)key.mv_data + key.mv_size; + } + } else { + for (i = 0; i < NUMKEYS(psrc); i++, j++) { + srcnode = NODEPTR(psrc, i); + if (i == 0 && IS_BRANCH(psrc)) { + MDB_cursor mn; + MDB_node *s2; + mdb_cursor_copy(csrc, &mn); + /* must find the lowest key below src */ + rc = mdb_page_search_lowest(&mn); + if (rc) + return rc; + if (IS_LEAF2(mn.mc_pg[mn.mc_top])) { + key.mv_size = mn.mc_db->md_pad; + key.mv_data = LEAF2KEY(mn.mc_pg[mn.mc_top], 0, key.mv_size); + } else { + s2 = NODEPTR(mn.mc_pg[mn.mc_top], 0); + key.mv_size = NODEKSZ(s2); + key.mv_data = NODEKEY(s2); + } + } else { + key.mv_size = srcnode->mn_ksize; + key.mv_data = NODEKEY(srcnode); + } + + data.mv_size = NODEDSZ(srcnode); + data.mv_data = NODEDATA(srcnode); + rc = mdb_node_add(cdst, j, &key, &data, NODEPGNO(srcnode), srcnode->mn_flags); + if (rc != MDB_SUCCESS) + return rc; + } + } + + DPRINTF(("dst page %"Z"u now has %u keys (%.1f%% filled)", + pdst->mp_pgno, NUMKEYS(pdst), + (float)PAGEFILL(cdst->mc_txn->mt_env, pdst) / 10)); + + /* Unlink the src page from parent and add to free list. + */ + csrc->mc_top--; + mdb_node_del(csrc, 0); + if (csrc->mc_ki[csrc->mc_top] == 0) { + key.mv_size = 0; + rc = mdb_update_key(csrc, &key); + if (rc) { + csrc->mc_top++; + return rc; + } + } + csrc->mc_top++; + + psrc = csrc->mc_pg[csrc->mc_top]; + /* If not operating on FreeDB, allow this page to be reused + * in this txn. Otherwise just add to free list. + */ + rc = mdb_page_loose(csrc, psrc); + if (rc) + return rc; + if (IS_LEAF(psrc)) + csrc->mc_db->md_leaf_pages--; + else + csrc->mc_db->md_branch_pages--; + { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2, *m3; + MDB_dbi dbi = csrc->mc_dbi; + + for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (csrc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3 == csrc) continue; + if (m3->mc_snum < csrc->mc_snum) continue; + if (m3->mc_pg[csrc->mc_top] == psrc) { + m3->mc_pg[csrc->mc_top] = pdst; + m3->mc_ki[csrc->mc_top] += nkeys; + } + } + } + { + unsigned int snum = cdst->mc_snum; + uint16_t depth = cdst->mc_db->md_depth; + mdb_cursor_pop(cdst); + rc = mdb_rebalance(cdst); + /* Did the tree shrink? */ + if (depth > cdst->mc_db->md_depth) + snum--; + cdst->mc_snum = snum; + cdst->mc_top = snum-1; + } + return rc; +} + +/** Copy the contents of a cursor. + * @param[in] csrc The cursor to copy from. + * @param[out] cdst The cursor to copy to. + */ +static void +mdb_cursor_copy(const MDB_cursor *csrc, MDB_cursor *cdst) +{ + unsigned int i; + + cdst->mc_txn = csrc->mc_txn; + cdst->mc_dbi = csrc->mc_dbi; + cdst->mc_db = csrc->mc_db; + cdst->mc_dbx = csrc->mc_dbx; + cdst->mc_snum = csrc->mc_snum; + cdst->mc_top = csrc->mc_top; + cdst->mc_flags = csrc->mc_flags; + + for (i=0; i<csrc->mc_snum; i++) { + cdst->mc_pg[i] = csrc->mc_pg[i]; + cdst->mc_ki[i] = csrc->mc_ki[i]; + } +} + +/** Rebalance the tree after a delete operation. + * @param[in] mc Cursor pointing to the page where rebalancing + * should begin. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_rebalance(MDB_cursor *mc) +{ + MDB_node *node; + int rc; + unsigned int ptop, minkeys; + MDB_cursor mn; + indx_t oldki; + + minkeys = 1 + (IS_BRANCH(mc->mc_pg[mc->mc_top])); + DPRINTF(("rebalancing %s page %"Z"u (has %u keys, %.1f%% full)", + IS_LEAF(mc->mc_pg[mc->mc_top]) ? "leaf" : "branch", + mdb_dbg_pgno(mc->mc_pg[mc->mc_top]), NUMKEYS(mc->mc_pg[mc->mc_top]), + (float)PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) / 10)); + + if (PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) >= FILL_THRESHOLD && + NUMKEYS(mc->mc_pg[mc->mc_top]) >= minkeys) { + DPRINTF(("no need to rebalance page %"Z"u, above fill threshold", + mdb_dbg_pgno(mc->mc_pg[mc->mc_top]))); + return MDB_SUCCESS; + } + + if (mc->mc_snum < 2) { + MDB_page *mp = mc->mc_pg[0]; + if (IS_SUBP(mp)) { + DPUTS("Can't rebalance a subpage, ignoring"); + return MDB_SUCCESS; + } + if (NUMKEYS(mp) == 0) { + DPUTS("tree is completely empty"); + mc->mc_db->md_root = P_INVALID; + mc->mc_db->md_depth = 0; + mc->mc_db->md_leaf_pages = 0; + rc = mdb_midl_append(&mc->mc_txn->mt_free_pgs, mp->mp_pgno); + if (rc) + return rc; + /* Adjust cursors pointing to mp */ + mc->mc_snum = 0; + mc->mc_top = 0; + mc->mc_flags &= ~C_INITIALIZED; + { + MDB_cursor *m2, *m3; + MDB_dbi dbi = mc->mc_dbi; + + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (mc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3->mc_snum < mc->mc_snum) continue; + if (m3->mc_pg[0] == mp) { + m3->mc_snum = 0; + m3->mc_top = 0; + m3->mc_flags &= ~C_INITIALIZED; + } + } + } + } else if (IS_BRANCH(mp) && NUMKEYS(mp) == 1) { + int i; + DPUTS("collapsing root page!"); + rc = mdb_midl_append(&mc->mc_txn->mt_free_pgs, mp->mp_pgno); + if (rc) + return rc; + mc->mc_db->md_root = NODEPGNO(NODEPTR(mp, 0)); + rc = mdb_page_get(mc->mc_txn,mc->mc_db->md_root,&mc->mc_pg[0],NULL); + if (rc) + return rc; + mc->mc_db->md_depth--; + mc->mc_db->md_branch_pages--; + mc->mc_ki[0] = mc->mc_ki[1]; + for (i = 1; i<mc->mc_db->md_depth; i++) { + mc->mc_pg[i] = mc->mc_pg[i+1]; + mc->mc_ki[i] = mc->mc_ki[i+1]; + } + { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2, *m3; + MDB_dbi dbi = mc->mc_dbi; + + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (mc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3 == mc || m3->mc_snum < mc->mc_snum) continue; + if (m3->mc_pg[0] == mp) { + m3->mc_snum--; + m3->mc_top--; + for (i=0; i<m3->mc_snum; i++) { + m3->mc_pg[i] = m3->mc_pg[i+1]; + m3->mc_ki[i] = m3->mc_ki[i+1]; + } + } + } + } + } else + DPUTS("root page doesn't need rebalancing"); + return MDB_SUCCESS; + } + + /* The parent (branch page) must have at least 2 pointers, + * otherwise the tree is invalid. + */ + ptop = mc->mc_top-1; + mdb_cassert(mc, NUMKEYS(mc->mc_pg[ptop]) > 1); + + /* Leaf page fill factor is below the threshold. + * Try to move keys from left or right neighbor, or + * merge with a neighbor page. + */ + + /* Find neighbors. + */ + mdb_cursor_copy(mc, &mn); + mn.mc_xcursor = NULL; + + oldki = mc->mc_ki[mc->mc_top]; + if (mc->mc_ki[ptop] == 0) { + /* We're the leftmost leaf in our parent. + */ + DPUTS("reading right neighbor"); + mn.mc_ki[ptop]++; + node = NODEPTR(mc->mc_pg[ptop], mn.mc_ki[ptop]); + rc = mdb_page_get(mc->mc_txn,NODEPGNO(node),&mn.mc_pg[mn.mc_top],NULL); + if (rc) + return rc; + mn.mc_ki[mn.mc_top] = 0; + mc->mc_ki[mc->mc_top] = NUMKEYS(mc->mc_pg[mc->mc_top]); + } else { + /* There is at least one neighbor to the left. + */ + DPUTS("reading left neighbor"); + mn.mc_ki[ptop]--; + node = NODEPTR(mc->mc_pg[ptop], mn.mc_ki[ptop]); + rc = mdb_page_get(mc->mc_txn,NODEPGNO(node),&mn.mc_pg[mn.mc_top],NULL); + if (rc) + return rc; + mn.mc_ki[mn.mc_top] = NUMKEYS(mn.mc_pg[mn.mc_top]) - 1; + mc->mc_ki[mc->mc_top] = 0; + } + + DPRINTF(("found neighbor page %"Z"u (%u keys, %.1f%% full)", + mn.mc_pg[mn.mc_top]->mp_pgno, NUMKEYS(mn.mc_pg[mn.mc_top]), + (float)PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) / 10)); + + /* If the neighbor page is above threshold and has enough keys, + * move one key from it. Otherwise we should try to merge them. + * (A branch page must never have less than 2 keys.) + */ + minkeys = 1 + (IS_BRANCH(mn.mc_pg[mn.mc_top])); + if (PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) >= FILL_THRESHOLD && NUMKEYS(mn.mc_pg[mn.mc_top]) > minkeys) { + rc = mdb_node_move(&mn, mc); + if (mc->mc_ki[ptop]) { + oldki++; + } + } else { + if (mc->mc_ki[ptop] == 0) { + rc = mdb_page_merge(&mn, mc); + } else { + oldki += NUMKEYS(mn.mc_pg[mn.mc_top]); + mn.mc_ki[mn.mc_top] += mc->mc_ki[mn.mc_top] + 1; + rc = mdb_page_merge(mc, &mn); + mdb_cursor_copy(&mn, mc); + } + mc->mc_flags &= ~C_EOF; + } + mc->mc_ki[mc->mc_top] = oldki; + return rc; +} + +/** Complete a delete operation started by #mdb_cursor_del(). */ +static int +mdb_cursor_del0(MDB_cursor *mc) +{ + int rc; + MDB_page *mp; + indx_t ki; + unsigned int nkeys; + + ki = mc->mc_ki[mc->mc_top]; + mdb_node_del(mc, mc->mc_db->md_pad); + mc->mc_db->md_entries--; + rc = mdb_rebalance(mc); + + if (rc == MDB_SUCCESS) { + MDB_cursor *m2, *m3; + MDB_dbi dbi = mc->mc_dbi; + + mp = mc->mc_pg[mc->mc_top]; + nkeys = NUMKEYS(mp); + + /* if mc points past last node in page, find next sibling */ + if (mc->mc_ki[mc->mc_top] >= nkeys) { + rc = mdb_cursor_sibling(mc, 1); + if (rc == MDB_NOTFOUND) { + mc->mc_flags |= C_EOF; + rc = MDB_SUCCESS; + } + } + + /* Adjust other cursors pointing to mp */ + for (m2 = mc->mc_txn->mt_cursors[dbi]; !rc && m2; m2=m2->mc_next) { + m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; + if (! (m2->mc_flags & m3->mc_flags & C_INITIALIZED)) + continue; + if (m3 == mc || m3->mc_snum < mc->mc_snum) + continue; + if (m3->mc_pg[mc->mc_top] == mp) { + if (m3->mc_ki[mc->mc_top] >= ki) { + m3->mc_flags |= C_DEL; + if (m3->mc_ki[mc->mc_top] > ki) + m3->mc_ki[mc->mc_top]--; + else if (mc->mc_db->md_flags & MDB_DUPSORT) + m3->mc_xcursor->mx_cursor.mc_flags |= C_EOF; + } + if (m3->mc_ki[mc->mc_top] >= nkeys) { + rc = mdb_cursor_sibling(m3, 1); + if (rc == MDB_NOTFOUND) { + m3->mc_flags |= C_EOF; + rc = MDB_SUCCESS; + } + } + } + } + mc->mc_flags |= C_DEL; + } + + if (rc) + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return rc; +} + +int +mdb_del(MDB_txn *txn, MDB_dbi dbi, + MDB_val *key, MDB_val *data) +{ + if (!key || dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + + if (txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_ERROR)) + return (txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN; + + if (!F_ISSET(txn->mt_dbs[dbi].md_flags, MDB_DUPSORT)) { + /* must ignore any data */ + data = NULL; + } + + return mdb_del0(txn, dbi, key, data, 0); +} + +static int +mdb_del0(MDB_txn *txn, MDB_dbi dbi, + MDB_val *key, MDB_val *data, unsigned flags) +{ + MDB_cursor mc; + MDB_xcursor mx; + MDB_cursor_op op; + MDB_val rdata, *xdata; + int rc, exact = 0; + DKBUF; + + DPRINTF(("====> delete db %u key [%s]", dbi, DKEY(key))); + + mdb_cursor_init(&mc, txn, dbi, &mx); + + if (data) { + op = MDB_GET_BOTH; + rdata = *data; + xdata = &rdata; + } else { + op = MDB_SET; + xdata = NULL; + flags |= MDB_NODUPDATA; + } + rc = mdb_cursor_set(&mc, key, xdata, op, &exact); + if (rc == 0) { + /* let mdb_page_split know about this cursor if needed: + * delete will trigger a rebalance; if it needs to move + * a node from one page to another, it will have to + * update the parent's separator key(s). If the new sepkey + * is larger than the current one, the parent page may + * run out of space, triggering a split. We need this + * cursor to be consistent until the end of the rebalance. + */ + mc.mc_flags |= C_UNTRACK; + mc.mc_next = txn->mt_cursors[dbi]; + txn->mt_cursors[dbi] = &mc; + rc = mdb_cursor_del(&mc, flags); + txn->mt_cursors[dbi] = mc.mc_next; + } + return rc; +} + +/** Split a page and insert a new node. + * @param[in,out] mc Cursor pointing to the page and desired insertion index. + * The cursor will be updated to point to the actual page and index where + * the node got inserted after the split. + * @param[in] newkey The key for the newly inserted node. + * @param[in] newdata The data for the newly inserted node. + * @param[in] newpgno The page number, if the new node is a branch node. + * @param[in] nflags The #NODE_ADD_FLAGS for the new node. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno, + unsigned int nflags) +{ + unsigned int flags; + int rc = MDB_SUCCESS, new_root = 0, did_split = 0; + indx_t newindx; + pgno_t pgno = 0; + int i, j, split_indx, nkeys, pmax; + MDB_env *env = mc->mc_txn->mt_env; + MDB_node *node; + MDB_val sepkey, rkey, xdata, *rdata = &xdata; + MDB_page *copy = NULL; + MDB_page *mp, *rp, *pp; + int ptop; + MDB_cursor mn; + DKBUF; + + mp = mc->mc_pg[mc->mc_top]; + newindx = mc->mc_ki[mc->mc_top]; + nkeys = NUMKEYS(mp); + + DPRINTF(("-----> splitting %s page %"Z"u and adding [%s] at index %i/%i", + IS_LEAF(mp) ? "leaf" : "branch", mp->mp_pgno, + DKEY(newkey), mc->mc_ki[mc->mc_top], nkeys)); + + /* Create a right sibling. */ + if ((rc = mdb_page_new(mc, mp->mp_flags, 1, &rp))) + return rc; + DPRINTF(("new right sibling: page %"Z"u", rp->mp_pgno)); + + if (mc->mc_snum < 2) { + if ((rc = mdb_page_new(mc, P_BRANCH, 1, &pp))) + goto done; + /* shift current top to make room for new parent */ + mc->mc_pg[1] = mc->mc_pg[0]; + mc->mc_ki[1] = mc->mc_ki[0]; + mc->mc_pg[0] = pp; + mc->mc_ki[0] = 0; + mc->mc_db->md_root = pp->mp_pgno; + DPRINTF(("root split! new root = %"Z"u", pp->mp_pgno)); + mc->mc_db->md_depth++; + new_root = 1; + + /* Add left (implicit) pointer. */ + if ((rc = mdb_node_add(mc, 0, NULL, NULL, mp->mp_pgno, 0)) != MDB_SUCCESS) { + /* undo the pre-push */ + mc->mc_pg[0] = mc->mc_pg[1]; + mc->mc_ki[0] = mc->mc_ki[1]; + mc->mc_db->md_root = mp->mp_pgno; + mc->mc_db->md_depth--; + goto done; + } + mc->mc_snum = 2; + mc->mc_top = 1; + ptop = 0; + } else { + ptop = mc->mc_top-1; + DPRINTF(("parent branch page is %"Z"u", mc->mc_pg[ptop]->mp_pgno)); + } + + mc->mc_flags |= C_SPLITTING; + mdb_cursor_copy(mc, &mn); + mn.mc_pg[mn.mc_top] = rp; + mn.mc_ki[ptop] = mc->mc_ki[ptop]+1; + + if (nflags & MDB_APPEND) { + mn.mc_ki[mn.mc_top] = 0; + sepkey = *newkey; + split_indx = newindx; + nkeys = 0; + } else { + + split_indx = (nkeys+1) / 2; + + if (IS_LEAF2(rp)) { + char *split, *ins; + int x; + unsigned int lsize, rsize, ksize; + /* Move half of the keys to the right sibling */ + x = mc->mc_ki[mc->mc_top] - split_indx; + ksize = mc->mc_db->md_pad; + split = LEAF2KEY(mp, split_indx, ksize); + rsize = (nkeys - split_indx) * ksize; + lsize = (nkeys - split_indx) * sizeof(indx_t); + mp->mp_lower -= lsize; + rp->mp_lower += lsize; + mp->mp_upper += rsize - lsize; + rp->mp_upper -= rsize - lsize; + sepkey.mv_size = ksize; + if (newindx == split_indx) { + sepkey.mv_data = newkey->mv_data; + } else { + sepkey.mv_data = split; + } + if (x<0) { + ins = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], ksize); + memcpy(rp->mp_ptrs, split, rsize); + sepkey.mv_data = rp->mp_ptrs; + memmove(ins+ksize, ins, (split_indx - mc->mc_ki[mc->mc_top]) * ksize); + memcpy(ins, newkey->mv_data, ksize); + mp->mp_lower += sizeof(indx_t); + mp->mp_upper -= ksize - sizeof(indx_t); + } else { + if (x) + memcpy(rp->mp_ptrs, split, x * ksize); + ins = LEAF2KEY(rp, x, ksize); + memcpy(ins, newkey->mv_data, ksize); + memcpy(ins+ksize, split + x * ksize, rsize - x * ksize); + rp->mp_lower += sizeof(indx_t); + rp->mp_upper -= ksize - sizeof(indx_t); + mc->mc_ki[mc->mc_top] = x; + mc->mc_pg[mc->mc_top] = rp; + } + } else { + int psize, nsize, k; + /* Maximum free space in an empty page */ + pmax = env->me_psize - PAGEHDRSZ; + if (IS_LEAF(mp)) + nsize = mdb_leaf_size(env, newkey, newdata); + else + nsize = mdb_branch_size(env, newkey); + nsize = EVEN(nsize); + + /* grab a page to hold a temporary copy */ + copy = mdb_page_malloc(mc->mc_txn, 1); + if (copy == NULL) { + rc = ENOMEM; + goto done; + } + copy->mp_pgno = mp->mp_pgno; + copy->mp_flags = mp->mp_flags; + copy->mp_lower = (PAGEHDRSZ-PAGEBASE); + copy->mp_upper = env->me_psize - PAGEBASE; + + /* prepare to insert */ + for (i=0, j=0; i<nkeys; i++) { + if (i == newindx) { + copy->mp_ptrs[j++] = 0; + } + copy->mp_ptrs[j++] = mp->mp_ptrs[i]; + } + + /* When items are relatively large the split point needs + * to be checked, because being off-by-one will make the + * difference between success or failure in mdb_node_add. + * + * It's also relevant if a page happens to be laid out + * such that one half of its nodes are all "small" and + * the other half of its nodes are "large." If the new + * item is also "large" and falls on the half with + * "large" nodes, it also may not fit. + * + * As a final tweak, if the new item goes on the last + * spot on the page (and thus, onto the new page), bias + * the split so the new page is emptier than the old page. + * This yields better packing during sequential inserts. + */ + if (nkeys < 20 || nsize > pmax/16 || newindx >= nkeys) { + /* Find split point */ + psize = 0; + if (newindx <= split_indx || newindx >= nkeys) { + i = 0; j = 1; + k = newindx >= nkeys ? nkeys : split_indx+2; + } else { + i = nkeys; j = -1; + k = split_indx-1; + } + for (; i!=k; i+=j) { + if (i == newindx) { + psize += nsize; + node = NULL; + } else { + node = (MDB_node *)((char *)mp + copy->mp_ptrs[i] + PAGEBASE); + psize += NODESIZE + NODEKSZ(node) + sizeof(indx_t); + if (IS_LEAF(mp)) { + if (F_ISSET(node->mn_flags, F_BIGDATA)) + psize += sizeof(pgno_t); + else + psize += NODEDSZ(node); + } + psize = EVEN(psize); + } + if (psize > pmax || i == k-j) { + split_indx = i + (j<0); + break; + } + } + } + if (split_indx == newindx) { + sepkey.mv_size = newkey->mv_size; + sepkey.mv_data = newkey->mv_data; + } else { + node = (MDB_node *)((char *)mp + copy->mp_ptrs[split_indx] + PAGEBASE); + sepkey.mv_size = node->mn_ksize; + sepkey.mv_data = NODEKEY(node); + } + } + } + + DPRINTF(("separator is %d [%s]", split_indx, DKEY(&sepkey))); + + /* Copy separator key to the parent. + */ + if (SIZELEFT(mn.mc_pg[ptop]) < mdb_branch_size(env, &sepkey)) { + mn.mc_snum--; + mn.mc_top--; + did_split = 1; + rc = mdb_page_split(&mn, &sepkey, NULL, rp->mp_pgno, 0); + if (rc) + goto done; + + /* root split? */ + if (mn.mc_snum == mc->mc_snum) { + mc->mc_pg[mc->mc_snum] = mc->mc_pg[mc->mc_top]; + mc->mc_ki[mc->mc_snum] = mc->mc_ki[mc->mc_top]; + mc->mc_pg[mc->mc_top] = mc->mc_pg[ptop]; + mc->mc_ki[mc->mc_top] = mc->mc_ki[ptop]; + mc->mc_snum++; + mc->mc_top++; + ptop++; + } + /* Right page might now have changed parent. + * Check if left page also changed parent. + */ + if (mn.mc_pg[ptop] != mc->mc_pg[ptop] && + mc->mc_ki[ptop] >= NUMKEYS(mc->mc_pg[ptop])) { + for (i=0; i<ptop; i++) { + mc->mc_pg[i] = mn.mc_pg[i]; + mc->mc_ki[i] = mn.mc_ki[i]; + } + mc->mc_pg[ptop] = mn.mc_pg[ptop]; + if (mn.mc_ki[ptop]) { + mc->mc_ki[ptop] = mn.mc_ki[ptop] - 1; + } else { + /* find right page's left sibling */ + mc->mc_ki[ptop] = mn.mc_ki[ptop]; + mdb_cursor_sibling(mc, 0); + } + } + } else { + mn.mc_top--; + rc = mdb_node_add(&mn, mn.mc_ki[ptop], &sepkey, NULL, rp->mp_pgno, 0); + mn.mc_top++; + } + mc->mc_flags ^= C_SPLITTING; + if (rc != MDB_SUCCESS) { + goto done; + } + if (nflags & MDB_APPEND) { + mc->mc_pg[mc->mc_top] = rp; + mc->mc_ki[mc->mc_top] = 0; + rc = mdb_node_add(mc, 0, newkey, newdata, newpgno, nflags); + if (rc) + goto done; + for (i=0; i<mc->mc_top; i++) + mc->mc_ki[i] = mn.mc_ki[i]; + } else if (!IS_LEAF2(mp)) { + /* Move nodes */ + mc->mc_pg[mc->mc_top] = rp; + i = split_indx; + j = 0; + do { + if (i == newindx) { + rkey.mv_data = newkey->mv_data; + rkey.mv_size = newkey->mv_size; + if (IS_LEAF(mp)) { + rdata = newdata; + } else + pgno = newpgno; + flags = nflags; + /* Update index for the new key. */ + mc->mc_ki[mc->mc_top] = j; + } else { + node = (MDB_node *)((char *)mp + copy->mp_ptrs[i] + PAGEBASE); + rkey.mv_data = NODEKEY(node); + rkey.mv_size = node->mn_ksize; + if (IS_LEAF(mp)) { + xdata.mv_data = NODEDATA(node); + xdata.mv_size = NODEDSZ(node); + rdata = &xdata; + } else + pgno = NODEPGNO(node); + flags = node->mn_flags; + } + + if (!IS_LEAF(mp) && j == 0) { + /* First branch index doesn't need key data. */ + rkey.mv_size = 0; + } + + rc = mdb_node_add(mc, j, &rkey, rdata, pgno, flags); + if (rc) + goto done; + if (i == nkeys) { + i = 0; + j = 0; + mc->mc_pg[mc->mc_top] = copy; + } else { + i++; + j++; + } + } while (i != split_indx); + + nkeys = NUMKEYS(copy); + for (i=0; i<nkeys; i++) + mp->mp_ptrs[i] = copy->mp_ptrs[i]; + mp->mp_lower = copy->mp_lower; + mp->mp_upper = copy->mp_upper; + memcpy(NODEPTR(mp, nkeys-1), NODEPTR(copy, nkeys-1), + env->me_psize - copy->mp_upper - PAGEBASE); + + /* reset back to original page */ + if (newindx < split_indx) { + mc->mc_pg[mc->mc_top] = mp; + if (nflags & MDB_RESERVE) { + node = NODEPTR(mp, mc->mc_ki[mc->mc_top]); + if (!(node->mn_flags & F_BIGDATA)) + newdata->mv_data = NODEDATA(node); + } + } else { + mc->mc_pg[mc->mc_top] = rp; + mc->mc_ki[ptop]++; + /* Make sure mc_ki is still valid. + */ + if (mn.mc_pg[ptop] != mc->mc_pg[ptop] && + mc->mc_ki[ptop] >= NUMKEYS(mc->mc_pg[ptop])) { + for (i=0; i<=ptop; i++) { + mc->mc_pg[i] = mn.mc_pg[i]; + mc->mc_ki[i] = mn.mc_ki[i]; + } + } + } + } + + { + /* Adjust other cursors pointing to mp */ + MDB_cursor *m2, *m3; + MDB_dbi dbi = mc->mc_dbi; + int fixup = NUMKEYS(mp); + + for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { + if (mc->mc_flags & C_SUB) + m3 = &m2->mc_xcursor->mx_cursor; + else + m3 = m2; + if (m3 == mc) + continue; + if (!(m2->mc_flags & m3->mc_flags & C_INITIALIZED)) + continue; + if (m3->mc_flags & C_SPLITTING) + continue; + if (new_root) { + int k; + /* root split */ + for (k=m3->mc_top; k>=0; k--) { + m3->mc_ki[k+1] = m3->mc_ki[k]; + m3->mc_pg[k+1] = m3->mc_pg[k]; + } + if (m3->mc_ki[0] >= split_indx) { + m3->mc_ki[0] = 1; + } else { + m3->mc_ki[0] = 0; + } + m3->mc_pg[0] = mc->mc_pg[0]; + m3->mc_snum++; + m3->mc_top++; + } + if (m3->mc_top >= mc->mc_top && m3->mc_pg[mc->mc_top] == mp) { + if (m3->mc_ki[mc->mc_top] >= newindx && !(nflags & MDB_SPLIT_REPLACE)) + m3->mc_ki[mc->mc_top]++; + if (m3->mc_ki[mc->mc_top] >= fixup) { + m3->mc_pg[mc->mc_top] = rp; + m3->mc_ki[mc->mc_top] -= fixup; + m3->mc_ki[ptop] = mn.mc_ki[ptop]; + } + } else if (!did_split && m3->mc_top >= ptop && m3->mc_pg[ptop] == mc->mc_pg[ptop] && + m3->mc_ki[ptop] >= mc->mc_ki[ptop]) { + m3->mc_ki[ptop]++; + } + } + } + DPRINTF(("mp left: %d, rp left: %d", SIZELEFT(mp), SIZELEFT(rp))); + +done: + if (copy) /* tmp page */ + mdb_page_free(env, copy); + if (rc) + mc->mc_txn->mt_flags |= MDB_TXN_ERROR; + return rc; +} + +int +mdb_put(MDB_txn *txn, MDB_dbi dbi, + MDB_val *key, MDB_val *data, unsigned int flags) +{ + MDB_cursor mc; + MDB_xcursor mx; + + if (!key || !data || dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + + if ((flags & (MDB_NOOVERWRITE|MDB_NODUPDATA|MDB_RESERVE|MDB_APPEND|MDB_APPENDDUP)) != flags) + return EINVAL; + + mdb_cursor_init(&mc, txn, dbi, &mx); + return mdb_cursor_put(&mc, key, data, flags); +} + +#ifndef MDB_WBUF +#define MDB_WBUF (1024*1024) +#endif + + /** State needed for a compacting copy. */ +typedef struct mdb_copy { + pthread_mutex_t mc_mutex; + pthread_cond_t mc_cond; + char *mc_wbuf[2]; + char *mc_over[2]; + MDB_env *mc_env; + MDB_txn *mc_txn; + int mc_wlen[2]; + int mc_olen[2]; + pgno_t mc_next_pgno; + HANDLE mc_fd; + int mc_status; + volatile int mc_new; + int mc_toggle; + +} mdb_copy; + + /** Dedicated writer thread for compacting copy. */ +static THREAD_RET ESECT +mdb_env_copythr(void *arg) +{ + mdb_copy *my = arg; + char *ptr; + int toggle = 0, wsize, rc; +#ifdef _WIN32 + DWORD len; +#define DO_WRITE(rc, fd, ptr, w2, len) rc = WriteFile(fd, ptr, w2, &len, NULL) +#else + int len; +#define DO_WRITE(rc, fd, ptr, w2, len) len = write(fd, ptr, w2); rc = (len >= 0) +#endif + + pthread_mutex_lock(&my->mc_mutex); + my->mc_new = 0; + pthread_cond_signal(&my->mc_cond); + for(;;) { + while (!my->mc_new) + pthread_cond_wait(&my->mc_cond, &my->mc_mutex); + if (my->mc_new < 0) { + my->mc_new = 0; + break; + } + my->mc_new = 0; + wsize = my->mc_wlen[toggle]; + ptr = my->mc_wbuf[toggle]; +again: + while (wsize > 0) { + DO_WRITE(rc, my->mc_fd, ptr, wsize, len); + if (!rc) { + rc = ErrCode(); + break; + } else if (len > 0) { + rc = MDB_SUCCESS; + ptr += len; + wsize -= len; + continue; + } else { + rc = EIO; + break; + } + } + if (rc) { + my->mc_status = rc; + break; + } + /* If there's an overflow page tail, write it too */ + if (my->mc_olen[toggle]) { + wsize = my->mc_olen[toggle]; + ptr = my->mc_over[toggle]; + my->mc_olen[toggle] = 0; + goto again; + } + my->mc_wlen[toggle] = 0; + toggle ^= 1; + pthread_cond_signal(&my->mc_cond); + } + pthread_cond_signal(&my->mc_cond); + pthread_mutex_unlock(&my->mc_mutex); + return (THREAD_RET)0; +#undef DO_WRITE +} + + /** Tell the writer thread there's a buffer ready to write */ +static int ESECT +mdb_env_cthr_toggle(mdb_copy *my, int st) +{ + int toggle = my->mc_toggle ^ 1; + pthread_mutex_lock(&my->mc_mutex); + if (my->mc_status) { + pthread_mutex_unlock(&my->mc_mutex); + return my->mc_status; + } + while (my->mc_new == 1) + pthread_cond_wait(&my->mc_cond, &my->mc_mutex); + my->mc_new = st; + my->mc_toggle = toggle; + pthread_cond_signal(&my->mc_cond); + pthread_mutex_unlock(&my->mc_mutex); + return 0; +} + + /** Depth-first tree traversal for compacting copy. */ +static int ESECT +mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags) +{ + MDB_cursor mc; + MDB_txn *txn = my->mc_txn; + MDB_node *ni; + MDB_page *mo, *mp, *leaf; + char *buf, *ptr; + int rc, toggle; + unsigned int i; + + /* Empty DB, nothing to do */ + if (*pg == P_INVALID) + return MDB_SUCCESS; + + mc.mc_snum = 1; + mc.mc_top = 0; + mc.mc_txn = txn; + + rc = mdb_page_get(my->mc_txn, *pg, &mc.mc_pg[0], NULL); + if (rc) + return rc; + rc = mdb_page_search_root(&mc, NULL, MDB_PS_FIRST); + if (rc) + return rc; + + /* Make cursor pages writable */ + buf = ptr = malloc(my->mc_env->me_psize * mc.mc_snum); + if (buf == NULL) + return ENOMEM; + + for (i=0; i<mc.mc_top; i++) { + mdb_page_copy((MDB_page *)ptr, mc.mc_pg[i], my->mc_env->me_psize); + mc.mc_pg[i] = (MDB_page *)ptr; + ptr += my->mc_env->me_psize; + } + + /* This is writable space for a leaf page. Usually not needed. */ + leaf = (MDB_page *)ptr; + + toggle = my->mc_toggle; + while (mc.mc_snum > 0) { + unsigned n; + mp = mc.mc_pg[mc.mc_top]; + n = NUMKEYS(mp); + + if (IS_LEAF(mp)) { + if (!IS_LEAF2(mp) && !(flags & F_DUPDATA)) { + for (i=0; i<n; i++) { + ni = NODEPTR(mp, i); + if (ni->mn_flags & F_BIGDATA) { + MDB_page *omp; + pgno_t pg; + + /* Need writable leaf */ + if (mp != leaf) { + mc.mc_pg[mc.mc_top] = leaf; + mdb_page_copy(leaf, mp, my->mc_env->me_psize); + mp = leaf; + ni = NODEPTR(mp, i); + } + + memcpy(&pg, NODEDATA(ni), sizeof(pg)); + rc = mdb_page_get(txn, pg, &omp, NULL); + if (rc) + goto done; + if (my->mc_wlen[toggle] >= MDB_WBUF) { + rc = mdb_env_cthr_toggle(my, 1); + if (rc) + goto done; + toggle = my->mc_toggle; + } + mo = (MDB_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]); + memcpy(mo, omp, my->mc_env->me_psize); + mo->mp_pgno = my->mc_next_pgno; + my->mc_next_pgno += omp->mp_pages; + my->mc_wlen[toggle] += my->mc_env->me_psize; + if (omp->mp_pages > 1) { + my->mc_olen[toggle] = my->mc_env->me_psize * (omp->mp_pages - 1); + my->mc_over[toggle] = (char *)omp + my->mc_env->me_psize; + rc = mdb_env_cthr_toggle(my, 1); + if (rc) + goto done; + toggle = my->mc_toggle; + } + memcpy(NODEDATA(ni), &mo->mp_pgno, sizeof(pgno_t)); + } else if (ni->mn_flags & F_SUBDATA) { + MDB_db db; + + /* Need writable leaf */ + if (mp != leaf) { + mc.mc_pg[mc.mc_top] = leaf; + mdb_page_copy(leaf, mp, my->mc_env->me_psize); + mp = leaf; + ni = NODEPTR(mp, i); + } + + memcpy(&db, NODEDATA(ni), sizeof(db)); + my->mc_toggle = toggle; + rc = mdb_env_cwalk(my, &db.md_root, ni->mn_flags & F_DUPDATA); + if (rc) + goto done; + toggle = my->mc_toggle; + memcpy(NODEDATA(ni), &db, sizeof(db)); + } + } + } + } else { + mc.mc_ki[mc.mc_top]++; + if (mc.mc_ki[mc.mc_top] < n) { + pgno_t pg; +again: + ni = NODEPTR(mp, mc.mc_ki[mc.mc_top]); + pg = NODEPGNO(ni); + rc = mdb_page_get(txn, pg, &mp, NULL); + if (rc) + goto done; + mc.mc_top++; + mc.mc_snum++; + mc.mc_ki[mc.mc_top] = 0; + if (IS_BRANCH(mp)) { + /* Whenever we advance to a sibling branch page, + * we must proceed all the way down to its first leaf. + */ + mdb_page_copy(mc.mc_pg[mc.mc_top], mp, my->mc_env->me_psize); + goto again; + } else + mc.mc_pg[mc.mc_top] = mp; + continue; + } + } + if (my->mc_wlen[toggle] >= MDB_WBUF) { + rc = mdb_env_cthr_toggle(my, 1); + if (rc) + goto done; + toggle = my->mc_toggle; + } + mo = (MDB_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]); + mdb_page_copy(mo, mp, my->mc_env->me_psize); + mo->mp_pgno = my->mc_next_pgno++; + my->mc_wlen[toggle] += my->mc_env->me_psize; + if (mc.mc_top) { + /* Update parent if there is one */ + ni = NODEPTR(mc.mc_pg[mc.mc_top-1], mc.mc_ki[mc.mc_top-1]); + SETPGNO(ni, mo->mp_pgno); + mdb_cursor_pop(&mc); + } else { + /* Otherwise we're done */ + *pg = mo->mp_pgno; + break; + } + } +done: + free(buf); + return rc; +} + + /** Copy environment with compaction. */ +static int ESECT +mdb_env_copyfd1(MDB_env *env, HANDLE fd) +{ + MDB_meta *mm; + MDB_page *mp; + mdb_copy my; + MDB_txn *txn = NULL; + pthread_t thr; + int rc; + +#ifdef _WIN32 + my.mc_mutex = CreateMutex(NULL, FALSE, NULL); + my.mc_cond = CreateEvent(NULL, FALSE, FALSE, NULL); + my.mc_wbuf[0] = _aligned_malloc(MDB_WBUF*2, env->me_os_psize); + if (my.mc_wbuf[0] == NULL) + return errno; +#else + pthread_mutex_init(&my.mc_mutex, NULL); + pthread_cond_init(&my.mc_cond, NULL); +#ifdef HAVE_MEMALIGN + my.mc_wbuf[0] = memalign(env->me_os_psize, MDB_WBUF*2); + if (my.mc_wbuf[0] == NULL) + return errno; +#else + rc = posix_memalign((void **)&my.mc_wbuf[0], env->me_os_psize, MDB_WBUF*2); + if (rc) + return rc; +#endif +#endif + memset(my.mc_wbuf[0], 0, MDB_WBUF*2); + my.mc_wbuf[1] = my.mc_wbuf[0] + MDB_WBUF; + my.mc_wlen[0] = 0; + my.mc_wlen[1] = 0; + my.mc_olen[0] = 0; + my.mc_olen[1] = 0; + my.mc_next_pgno = 2; + my.mc_status = 0; + my.mc_new = 1; + my.mc_toggle = 0; + my.mc_env = env; + my.mc_fd = fd; + THREAD_CREATE(thr, mdb_env_copythr, &my); + + rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + if (rc) + return rc; + + mp = (MDB_page *)my.mc_wbuf[0]; + memset(mp, 0, 2*env->me_psize); + mp->mp_pgno = 0; + mp->mp_flags = P_META; + mm = (MDB_meta *)METADATA(mp); + mdb_env_init_meta0(env, mm); + mm->mm_address = env->me_metas[0]->mm_address; + + mp = (MDB_page *)(my.mc_wbuf[0] + env->me_psize); + mp->mp_pgno = 1; + mp->mp_flags = P_META; + *(MDB_meta *)METADATA(mp) = *mm; + mm = (MDB_meta *)METADATA(mp); + + /* Count the number of free pages, subtract from lastpg to find + * number of active pages + */ + { + MDB_ID freecount = 0; + MDB_cursor mc; + MDB_val key, data; + mdb_cursor_init(&mc, txn, FREE_DBI, NULL); + while ((rc = mdb_cursor_get(&mc, &key, &data, MDB_NEXT)) == 0) + freecount += *(MDB_ID *)data.mv_data; + freecount += txn->mt_dbs[0].md_branch_pages + + txn->mt_dbs[0].md_leaf_pages + + txn->mt_dbs[0].md_overflow_pages; + + /* Set metapage 1 */ + mm->mm_last_pg = txn->mt_next_pgno - freecount - 1; + mm->mm_dbs[1] = txn->mt_dbs[1]; + if (mm->mm_last_pg > 1) { + mm->mm_dbs[1].md_root = mm->mm_last_pg; + mm->mm_txnid = 1; + } else { + mm->mm_dbs[1].md_root = P_INVALID; + } + } + my.mc_wlen[0] = env->me_psize * 2; + my.mc_txn = txn; + pthread_mutex_lock(&my.mc_mutex); + while(my.mc_new) + pthread_cond_wait(&my.mc_cond, &my.mc_mutex); + pthread_mutex_unlock(&my.mc_mutex); + rc = mdb_env_cwalk(&my, &txn->mt_dbs[1].md_root, 0); + if (rc == MDB_SUCCESS && my.mc_wlen[my.mc_toggle]) + rc = mdb_env_cthr_toggle(&my, 1); + mdb_env_cthr_toggle(&my, -1); + pthread_mutex_lock(&my.mc_mutex); + while(my.mc_new) + pthread_cond_wait(&my.mc_cond, &my.mc_mutex); + pthread_mutex_unlock(&my.mc_mutex); + THREAD_FINISH(thr); + + mdb_txn_abort(txn); +#ifdef _WIN32 + CloseHandle(my.mc_cond); + CloseHandle(my.mc_mutex); + _aligned_free(my.mc_wbuf[0]); +#else + pthread_cond_destroy(&my.mc_cond); + pthread_mutex_destroy(&my.mc_mutex); + free(my.mc_wbuf[0]); +#endif + return rc; +} + + /** Copy environment as-is. */ +static int ESECT +mdb_env_copyfd0(MDB_env *env, HANDLE fd) +{ + MDB_txn *txn = NULL; + int rc; + size_t wsize; + char *ptr; +#ifdef _WIN32 + DWORD len, w2; +#define DO_WRITE(rc, fd, ptr, w2, len) rc = WriteFile(fd, ptr, w2, &len, NULL) +#else + ssize_t len; + size_t w2; +#define DO_WRITE(rc, fd, ptr, w2, len) len = write(fd, ptr, w2); rc = (len >= 0) +#endif + + /* Do the lock/unlock of the reader mutex before starting the + * write txn. Otherwise other read txns could block writers. + */ + rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + if (rc) + return rc; + + if (env->me_txns) { + /* We must start the actual read txn after blocking writers */ + mdb_txn_reset0(txn, "reset-stage1"); + + /* Temporarily block writers until we snapshot the meta pages */ + LOCK_MUTEX_W(env); + + rc = mdb_txn_renew0(txn); + if (rc) { + UNLOCK_MUTEX_W(env); + goto leave; + } + } + + wsize = env->me_psize * 2; + ptr = env->me_map; + w2 = wsize; + while (w2 > 0) { + DO_WRITE(rc, fd, ptr, w2, len); + if (!rc) { + rc = ErrCode(); + break; + } else if (len > 0) { + rc = MDB_SUCCESS; + ptr += len; + w2 -= len; + continue; + } else { + /* Non-blocking or async handles are not supported */ + rc = EIO; + break; + } + } + if (env->me_txns) + UNLOCK_MUTEX_W(env); + + if (rc) + goto leave; + + w2 = txn->mt_next_pgno * env->me_psize; + { + size_t fsize = 0; + if ((rc = mdb_fsize(env->me_fd, &fsize))) + goto leave; + if (w2 > fsize) + w2 = fsize; + } + wsize = w2 - wsize; + while (wsize > 0) { + if (wsize > MAX_WRITE) + w2 = MAX_WRITE; + else + w2 = wsize; + DO_WRITE(rc, fd, ptr, w2, len); + if (!rc) { + rc = ErrCode(); + break; + } else if (len > 0) { + rc = MDB_SUCCESS; + ptr += len; + wsize -= len; + continue; + } else { + rc = EIO; + break; + } + } + +leave: + mdb_txn_abort(txn); + return rc; +} + +int ESECT +mdb_env_copyfd2(MDB_env *env, HANDLE fd, unsigned int flags) +{ + if (flags & MDB_CP_COMPACT) + return mdb_env_copyfd1(env, fd); + else + return mdb_env_copyfd0(env, fd); +} + +int ESECT +mdb_env_copyfd(MDB_env *env, HANDLE fd) +{ + return mdb_env_copyfd2(env, fd, 0); +} + +int ESECT +mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags) +{ + int rc, len; + char *lpath; + HANDLE newfd = INVALID_HANDLE_VALUE; + + if (env->me_flags & MDB_NOSUBDIR) { + lpath = (char *)path; + } else { + len = strlen(path); + len += sizeof(DATANAME); + lpath = malloc(len); + if (!lpath) + return ENOMEM; + sprintf(lpath, "%s" DATANAME, path); + } + + /* The destination path must exist, but the destination file must not. + * We don't want the OS to cache the writes, since the source data is + * already in the OS cache. + */ +#ifdef _WIN32 + newfd = CreateFile(lpath, GENERIC_WRITE, 0, NULL, CREATE_NEW, + FILE_FLAG_NO_BUFFERING|FILE_FLAG_WRITE_THROUGH, NULL); +#else + newfd = open(lpath, O_WRONLY|O_CREAT|O_EXCL, 0666); +#endif + if (newfd == INVALID_HANDLE_VALUE) { + rc = ErrCode(); + goto leave; + } + + if (env->me_psize >= env->me_os_psize) { +#ifdef O_DIRECT + /* Set O_DIRECT if the file system supports it */ + if ((rc = fcntl(newfd, F_GETFL)) != -1) + (void) fcntl(newfd, F_SETFL, rc | O_DIRECT); +#endif +#ifdef F_NOCACHE /* __APPLE__ */ + rc = fcntl(newfd, F_NOCACHE, 1); + if (rc) { + rc = ErrCode(); + goto leave; + } +#endif + } + + rc = mdb_env_copyfd2(env, newfd, flags); + +leave: + if (!(env->me_flags & MDB_NOSUBDIR)) + free(lpath); + if (newfd != INVALID_HANDLE_VALUE) + if (close(newfd) < 0 && rc == MDB_SUCCESS) + rc = ErrCode(); + + return rc; +} + +int ESECT +mdb_env_copy(MDB_env *env, const char *path) +{ + return mdb_env_copy2(env, path, 0); +} + +int ESECT +mdb_env_set_flags(MDB_env *env, unsigned int flag, int onoff) +{ + if ((flag & CHANGEABLE) != flag) + return EINVAL; + if (onoff) + env->me_flags |= flag; + else + env->me_flags &= ~flag; + return MDB_SUCCESS; +} + +int ESECT +mdb_env_get_flags(MDB_env *env, unsigned int *arg) +{ + if (!env || !arg) + return EINVAL; + + *arg = env->me_flags; + return MDB_SUCCESS; +} + +int ESECT +mdb_env_set_userctx(MDB_env *env, void *ctx) +{ + if (!env) + return EINVAL; + env->me_userctx = ctx; + return MDB_SUCCESS; +} + +void * ESECT +mdb_env_get_userctx(MDB_env *env) +{ + return env ? env->me_userctx : NULL; +} + +int ESECT +mdb_env_set_assert(MDB_env *env, MDB_assert_func *func) +{ + if (!env) + return EINVAL; +#ifndef NDEBUG + env->me_assert_func = func; +#endif + return MDB_SUCCESS; +} + +int ESECT +mdb_env_get_path(MDB_env *env, const char **arg) +{ + if (!env || !arg) + return EINVAL; + + *arg = env->me_path; + return MDB_SUCCESS; +} + +int ESECT +mdb_env_get_fd(MDB_env *env, mdb_filehandle_t *arg) +{ + if (!env || !arg) + return EINVAL; + + *arg = env->me_fd; + return MDB_SUCCESS; +} + +/** Common code for #mdb_stat() and #mdb_env_stat(). + * @param[in] env the environment to operate in. + * @param[in] db the #MDB_db record containing the stats to return. + * @param[out] arg the address of an #MDB_stat structure to receive the stats. + * @return 0, this function always succeeds. + */ +static int ESECT +mdb_stat0(MDB_env *env, MDB_db *db, MDB_stat *arg) +{ + arg->ms_psize = env->me_psize; + arg->ms_depth = db->md_depth; + arg->ms_branch_pages = db->md_branch_pages; + arg->ms_leaf_pages = db->md_leaf_pages; + arg->ms_overflow_pages = db->md_overflow_pages; + arg->ms_entries = db->md_entries; + + return MDB_SUCCESS; +} + +int ESECT +mdb_env_stat(MDB_env *env, MDB_stat *arg) +{ + int toggle; + + if (env == NULL || arg == NULL) + return EINVAL; + + toggle = mdb_env_pick_meta(env); + + return mdb_stat0(env, &env->me_metas[toggle]->mm_dbs[MAIN_DBI], arg); +} + +int ESECT +mdb_env_info(MDB_env *env, MDB_envinfo *arg) +{ + int toggle; + + if (env == NULL || arg == NULL) + return EINVAL; + + toggle = mdb_env_pick_meta(env); + arg->me_mapaddr = env->me_metas[toggle]->mm_address; + arg->me_mapsize = env->me_mapsize; + arg->me_maxreaders = env->me_maxreaders; + + /* me_numreaders may be zero if this process never used any readers. Use + * the shared numreader count if it exists. + */ + arg->me_numreaders = env->me_txns ? env->me_txns->mti_numreaders : env->me_numreaders; + + arg->me_last_pgno = env->me_metas[toggle]->mm_last_pg; + arg->me_last_txnid = env->me_metas[toggle]->mm_txnid; + return MDB_SUCCESS; +} + +/** Set the default comparison functions for a database. + * Called immediately after a database is opened to set the defaults. + * The user can then override them with #mdb_set_compare() or + * #mdb_set_dupsort(). + * @param[in] txn A transaction handle returned by #mdb_txn_begin() + * @param[in] dbi A database handle returned by #mdb_dbi_open() + */ +static void +mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi) +{ + uint16_t f = txn->mt_dbs[dbi].md_flags; + + txn->mt_dbxs[dbi].md_cmp = + (f & MDB_REVERSEKEY) ? mdb_cmp_memnr : + (f & MDB_INTEGERKEY) ? mdb_cmp_cint : mdb_cmp_memn; + + txn->mt_dbxs[dbi].md_dcmp = + !(f & MDB_DUPSORT) ? 0 : + ((f & MDB_INTEGERDUP) + ? ((f & MDB_DUPFIXED) ? mdb_cmp_int : mdb_cmp_cint) + : ((f & MDB_REVERSEDUP) ? mdb_cmp_memnr : mdb_cmp_memn)); +} + +int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *dbi) +{ + MDB_val key, data; + MDB_dbi i; + MDB_cursor mc; + MDB_db dummy; + int rc, dbflag, exact; + unsigned int unused = 0, seq; + size_t len; + + if (txn->mt_dbxs[FREE_DBI].md_cmp == NULL) { + mdb_default_cmp(txn, FREE_DBI); + } + + if ((flags & VALID_FLAGS) != flags) + return EINVAL; + if (txn->mt_flags & MDB_TXN_ERROR) + return MDB_BAD_TXN; + + /* main DB? */ + if (!name) { + *dbi = MAIN_DBI; + if (flags & PERSISTENT_FLAGS) { + uint16_t f2 = flags & PERSISTENT_FLAGS; + /* make sure flag changes get committed */ + if ((txn->mt_dbs[MAIN_DBI].md_flags | f2) != txn->mt_dbs[MAIN_DBI].md_flags) { + txn->mt_dbs[MAIN_DBI].md_flags |= f2; + txn->mt_flags |= MDB_TXN_DIRTY; + } + } + mdb_default_cmp(txn, MAIN_DBI); + return MDB_SUCCESS; + } + + if (txn->mt_dbxs[MAIN_DBI].md_cmp == NULL) { + mdb_default_cmp(txn, MAIN_DBI); + } + + /* Is the DB already open? */ + len = strlen(name); + for (i=2; i<txn->mt_numdbs; i++) { + if (!txn->mt_dbxs[i].md_name.mv_size) { + /* Remember this free slot */ + if (!unused) unused = i; + continue; + } + if (len == txn->mt_dbxs[i].md_name.mv_size && + !strncmp(name, txn->mt_dbxs[i].md_name.mv_data, len)) { + *dbi = i; + return MDB_SUCCESS; + } + } + + /* If no free slot and max hit, fail */ + if (!unused && txn->mt_numdbs >= txn->mt_env->me_maxdbs) + return MDB_DBS_FULL; + + /* Cannot mix named databases with some mainDB flags */ + if (txn->mt_dbs[MAIN_DBI].md_flags & (MDB_DUPSORT|MDB_INTEGERKEY)) + return (flags & MDB_CREATE) ? MDB_INCOMPATIBLE : MDB_NOTFOUND; + + /* Find the DB info */ + dbflag = DB_NEW|DB_VALID; + exact = 0; + key.mv_size = len; + key.mv_data = (void *)name; + mdb_cursor_init(&mc, txn, MAIN_DBI, NULL); + rc = mdb_cursor_set(&mc, &key, &data, MDB_SET, &exact); + if (rc == MDB_SUCCESS) { + /* make sure this is actually a DB */ + MDB_node *node = NODEPTR(mc.mc_pg[mc.mc_top], mc.mc_ki[mc.mc_top]); + if (!(node->mn_flags & F_SUBDATA)) + return MDB_INCOMPATIBLE; + } else if (rc == MDB_NOTFOUND && (flags & MDB_CREATE)) { + /* Create if requested */ + data.mv_size = sizeof(MDB_db); + data.mv_data = &dummy; + memset(&dummy, 0, sizeof(dummy)); + dummy.md_root = P_INVALID; + dummy.md_flags = flags & PERSISTENT_FLAGS; + rc = mdb_cursor_put(&mc, &key, &data, F_SUBDATA); + dbflag |= DB_DIRTY; + } + + /* OK, got info, add to table */ + if (rc == MDB_SUCCESS) { + unsigned int slot = unused ? unused : txn->mt_numdbs; + txn->mt_dbxs[slot].md_name.mv_data = strdup(name); + txn->mt_dbxs[slot].md_name.mv_size = len; + txn->mt_dbxs[slot].md_rel = NULL; + txn->mt_dbflags[slot] = dbflag; + /* txn-> and env-> are the same in read txns, use + * tmp variable to avoid undefined assignment + */ + seq = ++txn->mt_env->me_dbiseqs[slot]; + txn->mt_dbiseqs[slot] = seq; + + memcpy(&txn->mt_dbs[slot], data.mv_data, sizeof(MDB_db)); + *dbi = slot; + mdb_default_cmp(txn, slot); + if (!unused) { + txn->mt_numdbs++; + } + } + + return rc; +} + +int mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *arg) +{ + if (!arg || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + + if (txn->mt_flags & MDB_TXN_ERROR) + return MDB_BAD_TXN; + + if (txn->mt_dbflags[dbi] & DB_STALE) { + MDB_cursor mc; + MDB_xcursor mx; + /* Stale, must read the DB's root. cursor_init does it for us. */ + mdb_cursor_init(&mc, txn, dbi, &mx); + } + return mdb_stat0(txn->mt_env, &txn->mt_dbs[dbi], arg); +} + +void mdb_dbi_close(MDB_env *env, MDB_dbi dbi) +{ + char *ptr; + if (dbi <= MAIN_DBI || dbi >= env->me_maxdbs) + return; + ptr = env->me_dbxs[dbi].md_name.mv_data; + /* If there was no name, this was already closed */ + if (ptr) { + env->me_dbxs[dbi].md_name.mv_data = NULL; + env->me_dbxs[dbi].md_name.mv_size = 0; + env->me_dbflags[dbi] = 0; + env->me_dbiseqs[dbi]++; + free(ptr); + } +} + +int mdb_dbi_flags(MDB_txn *txn, MDB_dbi dbi, unsigned int *flags) +{ + /* We could return the flags for the FREE_DBI too but what's the point? */ + if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + *flags = txn->mt_dbs[dbi].md_flags & PERSISTENT_FLAGS; + return MDB_SUCCESS; +} + +/** Add all the DB's pages to the free list. + * @param[in] mc Cursor on the DB to free. + * @param[in] subs non-Zero to check for sub-DBs in this DB. + * @return 0 on success, non-zero on failure. + */ +static int +mdb_drop0(MDB_cursor *mc, int subs) +{ + int rc; + + rc = mdb_page_search(mc, NULL, MDB_PS_FIRST); + if (rc == MDB_SUCCESS) { + MDB_txn *txn = mc->mc_txn; + MDB_node *ni; + MDB_cursor mx; + unsigned int i; + + /* LEAF2 pages have no nodes, cannot have sub-DBs */ + if (IS_LEAF2(mc->mc_pg[mc->mc_top])) + mdb_cursor_pop(mc); + + mdb_cursor_copy(mc, &mx); + while (mc->mc_snum > 0) { + MDB_page *mp = mc->mc_pg[mc->mc_top]; + unsigned n = NUMKEYS(mp); + if (IS_LEAF(mp)) { + for (i=0; i<n; i++) { + ni = NODEPTR(mp, i); + if (ni->mn_flags & F_BIGDATA) { + MDB_page *omp; + pgno_t pg; + memcpy(&pg, NODEDATA(ni), sizeof(pg)); + rc = mdb_page_get(txn, pg, &omp, NULL); + if (rc != 0) + goto done; + mdb_cassert(mc, IS_OVERFLOW(omp)); + rc = mdb_midl_append_range(&txn->mt_free_pgs, + pg, omp->mp_pages); + if (rc) + goto done; + } else if (subs && (ni->mn_flags & F_SUBDATA)) { + mdb_xcursor_init1(mc, ni); + rc = mdb_drop0(&mc->mc_xcursor->mx_cursor, 0); + if (rc) + goto done; + } + } + } else { + if ((rc = mdb_midl_need(&txn->mt_free_pgs, n)) != 0) + goto done; + for (i=0; i<n; i++) { + pgno_t pg; + ni = NODEPTR(mp, i); + pg = NODEPGNO(ni); + /* free it */ + mdb_midl_xappend(txn->mt_free_pgs, pg); + } + } + if (!mc->mc_top) + break; + mc->mc_ki[mc->mc_top] = i; + rc = mdb_cursor_sibling(mc, 1); + if (rc) { + if (rc != MDB_NOTFOUND) + goto done; + /* no more siblings, go back to beginning + * of previous level. + */ + mdb_cursor_pop(mc); + mc->mc_ki[0] = 0; + for (i=1; i<mc->mc_snum; i++) { + mc->mc_ki[i] = 0; + mc->mc_pg[i] = mx.mc_pg[i]; + } + } + } + /* free it */ + rc = mdb_midl_append(&txn->mt_free_pgs, mc->mc_db->md_root); +done: + if (rc) + txn->mt_flags |= MDB_TXN_ERROR; + } else if (rc == MDB_NOTFOUND) { + rc = MDB_SUCCESS; + } + return rc; +} + +int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del) +{ + MDB_cursor *mc, *m2; + int rc; + + if ((unsigned)del > 1 || dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + + if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) + return EACCES; + + if (dbi > MAIN_DBI && TXN_DBI_CHANGED(txn, dbi)) + return MDB_BAD_DBI; + + rc = mdb_cursor_open(txn, dbi, &mc); + if (rc) + return rc; + + rc = mdb_drop0(mc, mc->mc_db->md_flags & MDB_DUPSORT); + /* Invalidate the dropped DB's cursors */ + for (m2 = txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) + m2->mc_flags &= ~(C_INITIALIZED|C_EOF); + if (rc) + goto leave; + + /* Can't delete the main DB */ + if (del && dbi > MAIN_DBI) { + rc = mdb_del0(txn, MAIN_DBI, &mc->mc_dbx->md_name, NULL, 0); + if (!rc) { + txn->mt_dbflags[dbi] = DB_STALE; + mdb_dbi_close(txn->mt_env, dbi); + } else { + txn->mt_flags |= MDB_TXN_ERROR; + } + } else { + /* reset the DB record, mark it dirty */ + txn->mt_dbflags[dbi] |= DB_DIRTY; + txn->mt_dbs[dbi].md_depth = 0; + txn->mt_dbs[dbi].md_branch_pages = 0; + txn->mt_dbs[dbi].md_leaf_pages = 0; + txn->mt_dbs[dbi].md_overflow_pages = 0; + txn->mt_dbs[dbi].md_entries = 0; + txn->mt_dbs[dbi].md_root = P_INVALID; + + txn->mt_flags |= MDB_TXN_DIRTY; + } +leave: + mdb_cursor_close(mc); + return rc; +} + +int mdb_set_compare(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp) +{ + if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + + txn->mt_dbxs[dbi].md_cmp = cmp; + return MDB_SUCCESS; +} + +int mdb_set_dupsort(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp) +{ + if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + + txn->mt_dbxs[dbi].md_dcmp = cmp; + return MDB_SUCCESS; +} + +int mdb_set_relfunc(MDB_txn *txn, MDB_dbi dbi, MDB_rel_func *rel) +{ + if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + + txn->mt_dbxs[dbi].md_rel = rel; + return MDB_SUCCESS; +} + +int mdb_set_relctx(MDB_txn *txn, MDB_dbi dbi, void *ctx) +{ + if (dbi == FREE_DBI || !TXN_DBI_EXIST(txn, dbi)) + return EINVAL; + + txn->mt_dbxs[dbi].md_relctx = ctx; + return MDB_SUCCESS; +} + +int ESECT +mdb_env_get_maxkeysize(MDB_env *env) +{ + return ENV_MAXKEY(env); +} + +int ESECT +mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx) +{ + unsigned int i, rdrs; + MDB_reader *mr; + char buf[64]; + int rc = 0, first = 1; + + if (!env || !func) + return -1; + if (!env->me_txns) { + return func("(no reader locks)\n", ctx); + } + rdrs = env->me_txns->mti_numreaders; + mr = env->me_txns->mti_readers; + for (i=0; i<rdrs; i++) { + if (mr[i].mr_pid) { + txnid_t txnid = mr[i].mr_txnid; + sprintf(buf, txnid == (txnid_t)-1 ? + "%10d %"Z"x -\n" : "%10d %"Z"x %"Z"u\n", + (int)mr[i].mr_pid, (size_t)mr[i].mr_tid, txnid); + if (first) { + first = 0; + rc = func(" pid thread txnid\n", ctx); + if (rc < 0) + break; + } + rc = func(buf, ctx); + if (rc < 0) + break; + } + } + if (first) { + rc = func("(no active readers)\n", ctx); + } + return rc; +} + +/** Insert pid into list if not already present. + * return -1 if already present. + */ +static int ESECT +mdb_pid_insert(MDB_PID_T *ids, MDB_PID_T pid) +{ + /* binary search of pid in list */ + unsigned base = 0; + unsigned cursor = 1; + int val = 0; + unsigned n = ids[0]; + + while( 0 < n ) { + unsigned pivot = n >> 1; + cursor = base + pivot + 1; + val = pid - ids[cursor]; + + if( val < 0 ) { + n = pivot; + + } else if ( val > 0 ) { + base = cursor; + n -= pivot + 1; + + } else { + /* found, so it's a duplicate */ + return -1; + } + } + + if( val > 0 ) { + ++cursor; + } + ids[0]++; + for (n = ids[0]; n > cursor; n--) + ids[n] = ids[n-1]; + ids[n] = pid; + return 0; +} + +int ESECT +mdb_reader_check(MDB_env *env, int *dead) +{ + unsigned int i, j, rdrs; + MDB_reader *mr; + MDB_PID_T *pids, pid; + int count = 0; + + if (!env) + return EINVAL; + if (dead) + *dead = 0; + if (!env->me_txns) + return MDB_SUCCESS; + rdrs = env->me_txns->mti_numreaders; + pids = malloc((rdrs+1) * sizeof(MDB_PID_T)); + if (!pids) + return ENOMEM; + pids[0] = 0; + mr = env->me_txns->mti_readers; + for (i=0; i<rdrs; i++) { + if (mr[i].mr_pid && mr[i].mr_pid != env->me_pid) { + pid = mr[i].mr_pid; + if (mdb_pid_insert(pids, pid) == 0) { + if (!mdb_reader_pid(env, Pidcheck, pid)) { + LOCK_MUTEX_R(env); + /* Recheck, a new process may have reused pid */ + if (!mdb_reader_pid(env, Pidcheck, pid)) { + for (j=i; j<rdrs; j++) + if (mr[j].mr_pid == pid) { + DPRINTF(("clear stale reader pid %u txn %"Z"d", + (unsigned) pid, mr[j].mr_txnid)); + mr[j].mr_pid = 0; + count++; + } + } + UNLOCK_MUTEX_R(env); + } + } + } + } + free(pids); + if (dead) + *dead = count; + return MDB_SUCCESS; +} +/** @} */ diff --git a/external/db_drivers/liblmdb64/mdb_copy.1 b/external/db_drivers/liblmdb64/mdb_copy.1 new file mode 100644 index 000000000..1e2a97694 --- /dev/null +++ b/external/db_drivers/liblmdb64/mdb_copy.1 @@ -0,0 +1,54 @@ +.TH MDB_COPY 1 "2014/06/20" "LMDB 0.9.14" +.\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved. +.\" Copying restrictions apply. See COPYRIGHT/LICENSE. +.SH NAME +mdb_copy \- LMDB environment copy tool +.SH SYNOPSIS +.B mdb_copy +[\c +.BR \-V ] +[\c +.BR \-c ] +[\c +.BR \-n ] +.B srcpath +[\c +.BR dstpath ] +.SH DESCRIPTION +The +.B mdb_copy +utility copies an LMDB environment. The environment can +be copied regardless of whether it is currently in use. +No lockfile is created, since it gets recreated at need. + +If +.I dstpath +is specified it must be the path of an empty directory +for storing the backup. Otherwise, the backup will be +written to stdout. + +.SH OPTIONS +.TP +.BR \-V +Write the library version number to the standard output, and exit. +.TP +.BR \-c +Compact while copying. Only current data pages will be copied; freed +or unused pages will be omitted from the copy. This option will +slow down the backup process as it is more CPU-intensive. +.TP +.BR \-n +Open LDMB environment(s) which do not use subdirectories. + +.SH DIAGNOSTICS +Exit status is zero if no errors occur. +Errors result in a non-zero exit status and +a diagnostic message being written to standard error. +.SH CAVEATS +This utility can trigger significant file size growth if run +in parallel with write transactions, because pages which they +free during copying cannot be reused until the copy is done. +.SH "SEE ALSO" +.BR mdb_stat (1) +.SH AUTHOR +Howard Chu of Symas Corporation <http://www.symas.com> diff --git a/external/db_drivers/liblmdb64/mdb_copy.c b/external/db_drivers/liblmdb64/mdb_copy.c new file mode 100644 index 000000000..f37ccbcc2 --- /dev/null +++ b/external/db_drivers/liblmdb64/mdb_copy.c @@ -0,0 +1,82 @@ +/* mdb_copy.c - memory-mapped database backup tool */ +/* + * Copyright 2012-2015 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ +#ifdef _WIN32 +#include <windows.h> +#define MDB_STDOUT GetStdHandle(STD_OUTPUT_HANDLE) +#else +#define MDB_STDOUT 1 +#endif +#include <stdio.h> +#include <stdlib.h> +#include <signal.h> +#include "lmdb.h" + +static void +sighandle(int sig) +{ +} + +int main(int argc,char * argv[]) +{ + int rc; + MDB_env *env; + const char *progname = argv[0], *act; + unsigned flags = MDB_RDONLY; + unsigned cpflags = 0; + + for (; argc > 1 && argv[1][0] == '-'; argc--, argv++) { + if (argv[1][1] == 'n' && argv[1][2] == '\0') + flags |= MDB_NOSUBDIR; + else if (argv[1][1] == 'c' && argv[1][2] == '\0') + cpflags |= MDB_CP_COMPACT; + else if (argv[1][1] == 'V' && argv[1][2] == '\0') { + printf("%s\n", MDB_VERSION_STRING); + exit(0); + } else + argc = 0; + } + + if (argc<2 || argc>3) { + fprintf(stderr, "usage: %s [-V] [-c] [-n] srcpath [dstpath]\n", progname); + exit(EXIT_FAILURE); + } + +#ifdef SIGPIPE + signal(SIGPIPE, sighandle); +#endif +#ifdef SIGHUP + signal(SIGHUP, sighandle); +#endif + signal(SIGINT, sighandle); + signal(SIGTERM, sighandle); + + act = "opening environment"; + rc = mdb_env_create(&env); + if (rc == MDB_SUCCESS) { + rc = mdb_env_open(env, argv[1], flags, 0600); + } + if (rc == MDB_SUCCESS) { + act = "copying"; + if (argc == 2) + rc = mdb_env_copyfd2(env, MDB_STDOUT, cpflags); + else + rc = mdb_env_copy2(env, argv[2], cpflags); + } + if (rc) + fprintf(stderr, "%s: %s failed, error %d (%s)\n", + progname, act, rc, mdb_strerror(rc)); + mdb_env_close(env); + + return rc ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/external/db_drivers/liblmdb64/mdb_dump.1 b/external/db_drivers/liblmdb64/mdb_dump.1 new file mode 100644 index 000000000..dd545049b --- /dev/null +++ b/external/db_drivers/liblmdb64/mdb_dump.1 @@ -0,0 +1,75 @@ +.TH MDB_DUMP 1 "2014/06/20" "LMDB 0.9.14" +.\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved. +.\" Copying restrictions apply. See COPYRIGHT/LICENSE. +.SH NAME +mdb_dump \- LMDB environment export tool +.SH SYNOPSIS +.B mdb_dump +.BR \ envpath +[\c +.BR \-V ] +[\c +.BI \-f \ file\fR] +[\c +.BR \-l ] +[\c +.BR \-n ] +[\c +.BR \-p ] +[\c +.BR \-a \ | +.BI \-s \ subdb\fR] +.SH DESCRIPTION +The +.B mdb_dump +utility reads a database and writes its contents to the +standard output using a portable flat-text format +understood by the +.BR mdb_load (1) +utility. +.SH OPTIONS +.TP +.BR \-V +Write the library version number to the standard output, and exit. +.TP +.BR \-f \ file +Write to the specified file instead of to the standard output. +.TP +.BR \-l +List the databases stored in the environment. Just the +names will be listed, no data will be output. +.TP +.BR \-n +Dump an LMDB database which does not use subdirectories. +.TP +.BR \-p +If characters in either the key or data items are printing characters (as +defined by isprint(3)), output them directly. This option permits users to +use standard text editors and tools to modify the contents of databases. + +Note: different systems may have different notions about what characters +are considered printing characters, and databases dumped in this manner may +be less portable to external systems. +.TP +.BR \-a +Dump all of the subdatabases in the environment. +.TP +.BR \-s \ subdb +Dump a specific subdatabase. If no database is specified, only the main database is dumped. +.SH DIAGNOSTICS +Exit status is zero if no errors occur. +Errors result in a non-zero exit status and +a diagnostic message being written to standard error. + +Dumping and reloading databases that use user-defined comparison functions +will result in new databases that use the default comparison functions. +\fBIn this case it is quite likely that the reloaded database will be +damaged beyond repair permitting neither record storage nor retrieval.\fP + +The only available workaround is to modify the source for the +.BR mdb_load (1) +utility to load the database using the correct comparison functions. +.SH "SEE ALSO" +.BR mdb_load (1) +.SH AUTHOR +Howard Chu of Symas Corporation <http://www.symas.com> diff --git a/external/db_drivers/liblmdb64/mdb_dump.c b/external/db_drivers/liblmdb64/mdb_dump.c new file mode 100644 index 000000000..7202d865b --- /dev/null +++ b/external/db_drivers/liblmdb64/mdb_dump.c @@ -0,0 +1,317 @@ +/* mdb_dump.c - memory-mapped database dump tool */ +/* + * Copyright 2011-2015 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ +#include <stdio.h> +#include <errno.h> +#include <stdlib.h> +#include <string.h> +#include <ctype.h> +#include <unistd.h> +#include <signal.h> +#include "lmdb.h" + +#ifdef _WIN32 +#define Z "I" +#else +#define Z "z" +#endif + +#define PRINT 1 +static int mode; + +typedef struct flagbit { + int bit; + char *name; +} flagbit; + +flagbit dbflags[] = { + { MDB_REVERSEKEY, "reversekey" }, + { MDB_DUPSORT, "dupsort" }, + { MDB_INTEGERKEY, "integerkey" }, + { MDB_DUPFIXED, "dupfixed" }, + { MDB_INTEGERDUP, "integerdup" }, + { MDB_REVERSEDUP, "reversedup" }, + { 0, NULL } +}; + +static volatile sig_atomic_t gotsig; + +static void dumpsig( int sig ) +{ + gotsig=1; +} + +static const char hexc[] = "0123456789abcdef"; + +static void hex(unsigned char c) +{ + putchar(hexc[c >> 4]); + putchar(hexc[c & 0xf]); +} + +static void text(MDB_val *v) +{ + unsigned char *c, *end; + + putchar(' '); + c = v->mv_data; + end = c + v->mv_size; + while (c < end) { + if (isprint(*c)) { + putchar(*c); + } else { + putchar('\\'); + hex(*c); + } + c++; + } + putchar('\n'); +} + +static void byte(MDB_val *v) +{ + unsigned char *c, *end; + + putchar(' '); + c = v->mv_data; + end = c + v->mv_size; + while (c < end) { + hex(*c++); + } + putchar('\n'); +} + +/* Dump in BDB-compatible format */ +static int dumpit(MDB_txn *txn, MDB_dbi dbi, char *name) +{ + MDB_cursor *mc; + MDB_stat ms; + MDB_val key, data; + MDB_envinfo info; + unsigned int flags; + int rc, i; + + rc = mdb_dbi_flags(txn, dbi, &flags); + if (rc) return rc; + + rc = mdb_stat(txn, dbi, &ms); + if (rc) return rc; + + rc = mdb_env_info(mdb_txn_env(txn), &info); + if (rc) return rc; + + printf("VERSION=3\n"); + printf("format=%s\n", mode & PRINT ? "print" : "bytevalue"); + if (name) + printf("database=%s\n", name); + printf("type=btree\n"); + printf("mapsize=%" Z "u\n", info.me_mapsize); + if (info.me_mapaddr) + printf("mapaddr=%p\n", info.me_mapaddr); + printf("maxreaders=%u\n", info.me_maxreaders); + + if (flags & MDB_DUPSORT) + printf("duplicates=1\n"); + + for (i=0; dbflags[i].bit; i++) + if (flags & dbflags[i].bit) + printf("%s=1\n", dbflags[i].name); + + printf("db_pagesize=%d\n", ms.ms_psize); + printf("HEADER=END\n"); + + rc = mdb_cursor_open(txn, dbi, &mc); + if (rc) return rc; + + while ((rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT) == MDB_SUCCESS)) { + if (gotsig) { + rc = EINTR; + break; + } + if (mode & PRINT) { + text(&key); + text(&data); + } else { + byte(&key); + byte(&data); + } + } + printf("DATA=END\n"); + if (rc == MDB_NOTFOUND) + rc = MDB_SUCCESS; + + return rc; +} + +static void usage(char *prog) +{ + fprintf(stderr, "usage: %s dbpath [-V] [-f output] [-l] [-n] [-p] [-a|-s subdb]\n", prog); + exit(EXIT_FAILURE); +} + +int main(int argc, char *argv[]) +{ + int i, rc; + MDB_env *env; + MDB_txn *txn; + MDB_dbi dbi; + char *prog = argv[0]; + char *envname; + char *subname = NULL; + int alldbs = 0, envflags = 0, list = 0; + + if (argc < 2) { + usage(prog); + } + + /* -a: dump main DB and all subDBs + * -s: dump only the named subDB + * -n: use NOSUBDIR flag on env_open + * -p: use printable characters + * -f: write to file instead of stdout + * -V: print version and exit + * (default) dump only the main DB + */ + while ((i = getopt(argc, argv, "af:lnps:V")) != EOF) { + switch(i) { + case 'V': + printf("%s\n", MDB_VERSION_STRING); + exit(0); + break; + case 'l': + list = 1; + /*FALLTHROUGH*/; + case 'a': + if (subname) + usage(prog); + alldbs++; + break; + case 'f': + if (freopen(optarg, "w", stdout) == NULL) { + fprintf(stderr, "%s: %s: reopen: %s\n", + prog, optarg, strerror(errno)); + exit(EXIT_FAILURE); + } + break; + case 'n': + envflags |= MDB_NOSUBDIR; + break; + case 'p': + mode |= PRINT; + break; + case 's': + if (alldbs) + usage(prog); + subname = optarg; + break; + default: + usage(prog); + } + } + + if (optind != argc - 1) + usage(prog); + +#ifdef SIGPIPE + signal(SIGPIPE, dumpsig); +#endif +#ifdef SIGHUP + signal(SIGHUP, dumpsig); +#endif + signal(SIGINT, dumpsig); + signal(SIGTERM, dumpsig); + + envname = argv[optind]; + rc = mdb_env_create(&env); + if (rc) { + fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc)); + return EXIT_FAILURE; + } + + if (alldbs || subname) { + mdb_env_set_maxdbs(env, 2); + } + + rc = mdb_env_open(env, envname, envflags | MDB_RDONLY, 0664); + if (rc) { + fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto env_close; + } + + rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + if (rc) { + fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc)); + goto env_close; + } + + rc = mdb_open(txn, subname, 0, &dbi); + if (rc) { + fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + + if (alldbs) { + MDB_cursor *cursor; + MDB_val key; + int count = 0; + + rc = mdb_cursor_open(txn, dbi, &cursor); + if (rc) { + fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + while ((rc = mdb_cursor_get(cursor, &key, NULL, MDB_NEXT_NODUP)) == 0) { + char *str; + MDB_dbi db2; + if (memchr(key.mv_data, '\0', key.mv_size)) + continue; + count++; + str = malloc(key.mv_size+1); + memcpy(str, key.mv_data, key.mv_size); + str[key.mv_size] = '\0'; + rc = mdb_open(txn, str, 0, &db2); + if (rc == MDB_SUCCESS) { + if (list) { + printf("%s\n", str); + list++; + } else { + rc = dumpit(txn, db2, str); + if (rc) + break; + } + mdb_close(env, db2); + } + free(str); + if (rc) continue; + } + mdb_cursor_close(cursor); + if (!count) { + fprintf(stderr, "%s: %s does not contain multiple databases\n", prog, envname); + rc = MDB_NOTFOUND; + } else if (rc == MDB_NOTFOUND) { + rc = MDB_SUCCESS; + } + } else { + rc = dumpit(txn, dbi, subname); + } + if (rc && rc != MDB_NOTFOUND) + fprintf(stderr, "%s: %s: %s\n", prog, envname, mdb_strerror(rc)); + + mdb_close(env, dbi); +txn_abort: + mdb_txn_abort(txn); +env_close: + mdb_env_close(env); + + return rc ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/external/db_drivers/liblmdb64/mdb_load.1 b/external/db_drivers/liblmdb64/mdb_load.1 new file mode 100644 index 000000000..511ec552b --- /dev/null +++ b/external/db_drivers/liblmdb64/mdb_load.1 @@ -0,0 +1,77 @@ +.TH MDB_LOAD 1 "2014/06/20" "LMDB 0.9.14" +.\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved. +.\" Copying restrictions apply. See COPYRIGHT/LICENSE. +.SH NAME +mdb_load \- LMDB environment import tool +.SH SYNOPSIS +.B mdb_load +.BR \ envpath +[\c +.BR \-V ] +[\c +.BI \-f \ file\fR] +[\c +.BR \-n ] +[\c +.BI \-s \ subdb\fR] +[\c +.BR \-N ] +[\c +.BR \-T ] +.SH DESCRIPTION +The +.B mdb_load +utility reads from the standard input and loads it into the +LMDB environment +.BR envpath . + +The input to +.B mdb_load +must be in the output format specified by the +.BR mdb_dump (1) +utility or as specified by the +.B -T +option below. +.SH OPTIONS +.TP +.BR \-V +Write the library version number to the standard output, and exit. +.TP +.BR \-f \ file +Read from the specified file instead of from the standard input. +.TP +.BR \-n +Load an LMDB database which does not use subdirectories. +.TP +.BR \-s \ subdb +Load a specific subdatabase. If no database is specified, data is loaded into the main database. +.TP +.BR \-N +Don't overwrite existing records when loading into an already existing database; just skip them. +.TP +.BR \-T +Load data from simple text files. The input must be paired lines of text, where the first +line of the pair is the key item, and the second line of the pair is its corresponding +data item. + +A simple escape mechanism, where newline and backslash (\\) characters are special, is +applied to the text input. Newline characters are interpreted as record separators. +Backslash characters in the text will be interpreted in one of two ways: If the backslash +character precedes another backslash character, the pair will be interpreted as a literal +backslash. If the backslash character precedes any other character, the two characters +following the backslash will be interpreted as a hexadecimal specification of a single +character; for example, \\0a is a newline character in the ASCII character set. + +For this reason, any backslash or newline characters that naturally occur in the text +input must be escaped to avoid misinterpretation by +.BR mdb_load . + +.SH DIAGNOSTICS +Exit status is zero if no errors occur. +Errors result in a non-zero exit status and +a diagnostic message being written to standard error. + +.SH "SEE ALSO" +.BR mdb_dump (1) +.SH AUTHOR +Howard Chu of Symas Corporation <http://www.symas.com> diff --git a/external/db_drivers/liblmdb64/mdb_load.c b/external/db_drivers/liblmdb64/mdb_load.c new file mode 100644 index 000000000..1f6ce0b7e --- /dev/null +++ b/external/db_drivers/liblmdb64/mdb_load.c @@ -0,0 +1,454 @@ +/* mdb_load.c - memory-mapped database load tool */ +/* + * Copyright 2011-2015 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ +#include <stdio.h> +#include <stdlib.h> +#include <errno.h> +#include <string.h> +#include <ctype.h> +#include <unistd.h> +#include "lmdb.h" + +#define PRINT 1 +#define NOHDR 2 +static int mode; + +static char *subname = NULL; + +static size_t lineno; +static int version; + +static int flags; + +static char *prog; + +static int Eof; + +static MDB_envinfo info; + +static MDB_val kbuf, dbuf; + +#ifdef _WIN32 +#define Z "I" +#else +#define Z "z" +#endif + +#define STRLENOF(s) (sizeof(s)-1) + +typedef struct flagbit { + int bit; + char *name; + int len; +} flagbit; + +#define S(s) s, STRLENOF(s) + +flagbit dbflags[] = { + { MDB_REVERSEKEY, S("reversekey") }, + { MDB_DUPSORT, S("dupsort") }, + { MDB_INTEGERKEY, S("integerkey") }, + { MDB_DUPFIXED, S("dupfixed") }, + { MDB_INTEGERDUP, S("integerdup") }, + { MDB_REVERSEDUP, S("reversedup") }, + { 0, NULL, 0 } +}; + +static void readhdr(void) +{ + char *ptr; + + while (fgets(dbuf.mv_data, dbuf.mv_size, stdin) != NULL) { + lineno++; + if (!strncmp(dbuf.mv_data, "VERSION=", STRLENOF("VERSION="))) { + version=atoi((char *)dbuf.mv_data+STRLENOF("VERSION=")); + if (version > 3) { + fprintf(stderr, "%s: line %" Z "d: unsupported VERSION %d\n", + prog, lineno, version); + exit(EXIT_FAILURE); + } + } else if (!strncmp(dbuf.mv_data, "HEADER=END", STRLENOF("HEADER=END"))) { + break; + } else if (!strncmp(dbuf.mv_data, "format=", STRLENOF("format="))) { + if (!strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "print", STRLENOF("print"))) + mode |= PRINT; + else if (strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "bytevalue", STRLENOF("bytevalue"))) { + fprintf(stderr, "%s: line %" Z "d: unsupported FORMAT %s\n", + prog, lineno, (char *)dbuf.mv_data+STRLENOF("FORMAT=")); + exit(EXIT_FAILURE); + } + } else if (!strncmp(dbuf.mv_data, "database=", STRLENOF("database="))) { + ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size); + if (ptr) *ptr = '\0'; + if (subname) free(subname); + subname = strdup((char *)dbuf.mv_data+STRLENOF("database=")); + } else if (!strncmp(dbuf.mv_data, "type=", STRLENOF("type="))) { + if (strncmp((char *)dbuf.mv_data+STRLENOF("type="), "btree", STRLENOF("btree"))) { + fprintf(stderr, "%s: line %" Z "d: unsupported type %s\n", + prog, lineno, (char *)dbuf.mv_data+STRLENOF("type=")); + exit(EXIT_FAILURE); + } + } else if (!strncmp(dbuf.mv_data, "mapaddr=", STRLENOF("mapaddr="))) { + int i; + ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size); + if (ptr) *ptr = '\0'; + i = sscanf((char *)dbuf.mv_data+STRLENOF("mapaddr="), "%p", &info.me_mapaddr); + if (i != 1) { + fprintf(stderr, "%s: line %" Z "d: invalid mapaddr %s\n", + prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapaddr=")); + exit(EXIT_FAILURE); + } + } else if (!strncmp(dbuf.mv_data, "mapsize=", STRLENOF("mapsize="))) { + int i; + ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size); + if (ptr) *ptr = '\0'; + i = sscanf((char *)dbuf.mv_data+STRLENOF("mapsize="), "%" Z "u", &info.me_mapsize); + if (i != 1) { + fprintf(stderr, "%s: line %" Z "d: invalid mapsize %s\n", + prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapsize=")); + exit(EXIT_FAILURE); + } + } else if (!strncmp(dbuf.mv_data, "maxreaders=", STRLENOF("maxreaders="))) { + int i; + ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size); + if (ptr) *ptr = '\0'; + i = sscanf((char *)dbuf.mv_data+STRLENOF("maxreaders="), "%u", &info.me_maxreaders); + if (i != 1) { + fprintf(stderr, "%s: line %" Z "d: invalid maxreaders %s\n", + prog, lineno, (char *)dbuf.mv_data+STRLENOF("maxreaders=")); + exit(EXIT_FAILURE); + } + } else { + int i; + for (i=0; dbflags[i].bit; i++) { + if (!strncmp(dbuf.mv_data, dbflags[i].name, dbflags[i].len) && + ((char *)dbuf.mv_data)[dbflags[i].len] == '=') { + flags |= dbflags[i].bit; + break; + } + } + if (!dbflags[i].bit) { + ptr = memchr(dbuf.mv_data, '=', dbuf.mv_size); + if (!ptr) { + fprintf(stderr, "%s: line %" Z "d: unexpected format\n", + prog, lineno); + exit(EXIT_FAILURE); + } else { + *ptr = '\0'; + fprintf(stderr, "%s: line %" Z "d: unrecognized keyword ignored: %s\n", + prog, lineno, (char *)dbuf.mv_data); + } + } + } + } +} + +static void badend(void) +{ + fprintf(stderr, "%s: line %" Z "d: unexpected end of input\n", + prog, lineno); +} + +static int unhex(unsigned char *c2) +{ + int x, c; + x = *c2++ & 0x4f; + if (x & 0x40) + x -= 55; + c = x << 4; + x = *c2 & 0x4f; + if (x & 0x40) + x -= 55; + c |= x; + return c; +} + +static int readline(MDB_val *out, MDB_val *buf) +{ + unsigned char *c1, *c2, *end; + size_t len, l2; + int c; + + if (!(mode & NOHDR)) { + c = fgetc(stdin); + if (c == EOF) { + Eof = 1; + return EOF; + } + if (c != ' ') { + lineno++; + if (fgets(buf->mv_data, buf->mv_size, stdin) == NULL) { +badend: + Eof = 1; + badend(); + return EOF; + } + if (c == 'D' && !strncmp(buf->mv_data, "ATA=END", STRLENOF("ATA=END"))) + return EOF; + goto badend; + } + } + if (fgets(buf->mv_data, buf->mv_size, stdin) == NULL) { + Eof = 1; + return EOF; + } + lineno++; + + c1 = buf->mv_data; + len = strlen((char *)c1); + l2 = len; + + /* Is buffer too short? */ + while (c1[len-1] != '\n') { + buf->mv_data = realloc(buf->mv_data, buf->mv_size*2); + if (!buf->mv_data) { + Eof = 1; + fprintf(stderr, "%s: line %" Z "d: out of memory, line too long\n", + prog, lineno); + return EOF; + } + c1 = buf->mv_data; + c1 += l2; + if (fgets((char *)c1, buf->mv_size+1, stdin) == NULL) { + Eof = 1; + badend(); + return EOF; + } + buf->mv_size *= 2; + len = strlen((char *)c1); + l2 += len; + } + c1 = c2 = buf->mv_data; + len = l2; + c1[--len] = '\0'; + end = c1 + len; + + if (mode & PRINT) { + while (c2 < end) { + if (*c2 == '\\') { + if (c2[1] == '\\') { + c1++; c2 += 2; + } else { + if (c2+3 > end || !isxdigit(c2[1]) || !isxdigit(c2[2])) { + Eof = 1; + badend(); + return EOF; + } + *c1++ = unhex(++c2); + c2 += 2; + } + } else { + c1++; c2++; + } + } + } else { + /* odd length not allowed */ + if (len & 1) { + Eof = 1; + badend(); + return EOF; + } + while (c2 < end) { + if (!isxdigit(*c2) || !isxdigit(c2[1])) { + Eof = 1; + badend(); + return EOF; + } + *c1++ = unhex(c2); + c2 += 2; + } + } + c2 = out->mv_data = buf->mv_data; + out->mv_size = c1 - c2; + + return 0; +} + +static void usage(void) +{ + fprintf(stderr, "usage: %s dbpath [-V] [-f input] [-n] [-s name] [-N] [-T]\n", prog); + exit(EXIT_FAILURE); +} + +int main(int argc, char *argv[]) +{ + int i, rc; + MDB_env *env; + MDB_txn *txn; + MDB_cursor *mc; + MDB_dbi dbi; + char *envname; + int envflags = 0, putflags = 0; + int dohdr = 0; + + prog = argv[0]; + + if (argc < 2) { + usage(); + } + + /* -f: load file instead of stdin + * -n: use NOSUBDIR flag on env_open + * -s: load into named subDB + * -N: use NOOVERWRITE on puts + * -T: read plaintext + * -V: print version and exit + */ + while ((i = getopt(argc, argv, "f:ns:NTV")) != EOF) { + switch(i) { + case 'V': + printf("%s\n", MDB_VERSION_STRING); + exit(0); + break; + case 'f': + if (freopen(optarg, "r", stdin) == NULL) { + fprintf(stderr, "%s: %s: reopen: %s\n", + prog, optarg, strerror(errno)); + exit(EXIT_FAILURE); + } + break; + case 'n': + envflags |= MDB_NOSUBDIR; + break; + case 's': + subname = strdup(optarg); + break; + case 'N': + putflags = MDB_NOOVERWRITE|MDB_NODUPDATA; + break; + case 'T': + mode |= NOHDR; + break; + default: + usage(); + } + } + + if (optind != argc - 1) + usage(); + + dbuf.mv_size = 4096; + dbuf.mv_data = malloc(dbuf.mv_size); + + if (!(mode & NOHDR)) + readhdr(); + + envname = argv[optind]; + rc = mdb_env_create(&env); + if (rc) { + fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc)); + return EXIT_FAILURE; + } + + mdb_env_set_maxdbs(env, 2); + + if (info.me_maxreaders) + mdb_env_set_maxreaders(env, info.me_maxreaders); + + if (info.me_mapsize) + mdb_env_set_mapsize(env, info.me_mapsize); + + if (info.me_mapaddr) + envflags |= MDB_FIXEDMAP; + + rc = mdb_env_open(env, envname, envflags, 0664); + if (rc) { + fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto env_close; + } + + kbuf.mv_size = mdb_env_get_maxkeysize(env) * 2 + 2; + kbuf.mv_data = malloc(kbuf.mv_size); + + while(!Eof) { + MDB_val key, data; + int batch = 0; + flags = 0; + + if (!dohdr) { + dohdr = 1; + } else if (!(mode & NOHDR)) + readhdr(); + + rc = mdb_txn_begin(env, NULL, 0, &txn); + if (rc) { + fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc)); + goto env_close; + } + + rc = mdb_open(txn, subname, flags|MDB_CREATE, &dbi); + if (rc) { + fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + + rc = mdb_cursor_open(txn, dbi, &mc); + if (rc) { + fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + + while(1) { + rc = readline(&key, &kbuf); + if (rc == EOF) + break; + if (rc) + goto txn_abort; + + rc = readline(&data, &dbuf); + if (rc) + goto txn_abort; + + rc = mdb_cursor_put(mc, &key, &data, putflags); + if (rc == MDB_KEYEXIST && putflags) + continue; + if (rc) + goto txn_abort; + batch++; + if (batch == 100) { + rc = mdb_txn_commit(txn); + if (rc) { + fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n", + prog, lineno, mdb_strerror(rc)); + goto env_close; + } + rc = mdb_txn_begin(env, NULL, 0, &txn); + if (rc) { + fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc)); + goto env_close; + } + rc = mdb_cursor_open(txn, dbi, &mc); + if (rc) { + fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + batch = 0; + } + } + rc = mdb_txn_commit(txn); + txn = NULL; + if (rc) { + fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n", + prog, lineno, mdb_strerror(rc)); + goto env_close; + } + mdb_dbi_close(env, dbi); + } + +txn_abort: + mdb_txn_abort(txn); +env_close: + mdb_env_close(env); + + return rc ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/external/db_drivers/liblmdb64/mdb_stat.1 b/external/db_drivers/liblmdb64/mdb_stat.1 new file mode 100644 index 000000000..e6ee5ad58 --- /dev/null +++ b/external/db_drivers/liblmdb64/mdb_stat.1 @@ -0,0 +1,64 @@ +.TH MDB_STAT 1 "2014/06/20" "LMDB 0.9.14" +.\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved. +.\" Copying restrictions apply. See COPYRIGHT/LICENSE. +.SH NAME +mdb_stat \- LMDB environment status tool +.SH SYNOPSIS +.B mdb_stat +.BR \ envpath +[\c +.BR \-V ] +[\c +.BR \-e ] +[\c +.BR \-f [ f [ f ]]] +[\c +.BR \-n ] +[\c +.BR \-r [ r ]] +[\c +.BR \-a \ | +.BI \-s \ subdb\fR] +.SH DESCRIPTION +The +.B mdb_stat +utility displays the status of an LMDB environment. +.SH OPTIONS +.TP +.BR \-V +Write the library version number to the standard output, and exit. +.TP +.BR \-e +Display information about the database environment. +.TP +.BR \-f +Display information about the environment freelist. +If \fB\-ff\fP is given, summarize each freelist entry. +If \fB\-fff\fP is given, display the full list of page IDs in the freelist. +.TP +.BR \-n +Display the status of an LMDB database which does not use subdirectories. +.TP +.BR \-r +Display information about the environment reader table. +Shows the process ID, thread ID, and transaction ID for each active +reader slot. The process ID and transaction ID are in decimal, the +thread ID is in hexadecimal. The transaction ID is displayed as "-" +if the reader does not currently have a read transaction open. +If \fB\-rr\fP is given, check for stale entries in the reader +table and clear them. The reader table will be printed again +after the check is performed. +.TP +.BR \-a +Display the status of all of the subdatabases in the environment. +.TP +.BR \-s \ subdb +Display the status of a specific subdatabase. +.SH DIAGNOSTICS +Exit status is zero if no errors occur. +Errors result in a non-zero exit status and +a diagnostic message being written to standard error. +.SH "SEE ALSO" +.BR mdb_copy (1) +.SH AUTHOR +Howard Chu of Symas Corporation <http://www.symas.com> diff --git a/external/db_drivers/liblmdb64/mdb_stat.c b/external/db_drivers/liblmdb64/mdb_stat.c new file mode 100644 index 000000000..210609b32 --- /dev/null +++ b/external/db_drivers/liblmdb64/mdb_stat.c @@ -0,0 +1,263 @@ +/* mdb_stat.c - memory-mapped database status tool */ +/* + * Copyright 2011-2015 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include "lmdb.h" + +#ifdef _WIN32 +#define Z "I" +#else +#define Z "z" +#endif + +static void prstat(MDB_stat *ms) +{ +#if 0 + printf(" Page size: %u\n", ms->ms_psize); +#endif + printf(" Tree depth: %u\n", ms->ms_depth); + printf(" Branch pages: %"Z"u\n", ms->ms_branch_pages); + printf(" Leaf pages: %"Z"u\n", ms->ms_leaf_pages); + printf(" Overflow pages: %"Z"u\n", ms->ms_overflow_pages); + printf(" Entries: %"Z"u\n", ms->ms_entries); +} + +static void usage(char *prog) +{ + fprintf(stderr, "usage: %s dbpath [-V] [-n] [-e] [-r[r]] [-f[f[f]]] [-a|-s subdb]\n", prog); + exit(EXIT_FAILURE); +} + +int main(int argc, char *argv[]) +{ + int i, rc; + MDB_env *env; + MDB_txn *txn; + MDB_dbi dbi; + MDB_stat mst; + MDB_envinfo mei; + char *prog = argv[0]; + char *envname; + char *subname = NULL; + int alldbs = 0, envinfo = 0, envflags = 0, freinfo = 0, rdrinfo = 0; + + if (argc < 2) { + usage(prog); + } + + /* -a: print stat of main DB and all subDBs + * -s: print stat of only the named subDB + * -e: print env info + * -f: print freelist info + * -r: print reader info + * -n: use NOSUBDIR flag on env_open + * -V: print version and exit + * (default) print stat of only the main DB + */ + while ((i = getopt(argc, argv, "Vaefnrs:")) != EOF) { + switch(i) { + case 'V': + printf("%s\n", MDB_VERSION_STRING); + exit(0); + break; + case 'a': + if (subname) + usage(prog); + alldbs++; + break; + case 'e': + envinfo++; + break; + case 'f': + freinfo++; + break; + case 'n': + envflags |= MDB_NOSUBDIR; + break; + case 'r': + rdrinfo++; + break; + case 's': + if (alldbs) + usage(prog); + subname = optarg; + break; + default: + usage(prog); + } + } + + if (optind != argc - 1) + usage(prog); + + envname = argv[optind]; + rc = mdb_env_create(&env); + if (rc) { + fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc)); + return EXIT_FAILURE; + } + + if (alldbs || subname) { + mdb_env_set_maxdbs(env, 4); + } + + rc = mdb_env_open(env, envname, envflags | MDB_RDONLY, 0664); + if (rc) { + fprintf(stderr, "mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto env_close; + } + + if (envinfo) { + (void)mdb_env_stat(env, &mst); + (void)mdb_env_info(env, &mei); + printf("Environment Info\n"); + printf(" Map address: %p\n", mei.me_mapaddr); + printf(" Map size: %"Z"u\n", mei.me_mapsize); + printf(" Page size: %u\n", mst.ms_psize); + printf(" Max pages: %"Z"u\n", mei.me_mapsize / mst.ms_psize); + printf(" Number of pages used: %"Z"u\n", mei.me_last_pgno+1); + printf(" Last transaction ID: %"Z"u\n", mei.me_last_txnid); + printf(" Max readers: %u\n", mei.me_maxreaders); + printf(" Number of readers used: %u\n", mei.me_numreaders); + } + + if (rdrinfo) { + printf("Reader Table Status\n"); + rc = mdb_reader_list(env, (MDB_msg_func *)fputs, stdout); + if (rdrinfo > 1) { + int dead; + mdb_reader_check(env, &dead); + printf(" %d stale readers cleared.\n", dead); + rc = mdb_reader_list(env, (MDB_msg_func *)fputs, stdout); + } + if (!(subname || alldbs || freinfo)) + goto env_close; + } + + rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + if (rc) { + fprintf(stderr, "mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc)); + goto env_close; + } + + if (freinfo) { + MDB_cursor *cursor; + MDB_val key, data; + size_t pages = 0, *iptr; + + printf("Freelist Status\n"); + dbi = 0; + rc = mdb_cursor_open(txn, dbi, &cursor); + if (rc) { + fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + rc = mdb_stat(txn, dbi, &mst); + if (rc) { + fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + prstat(&mst); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + iptr = data.mv_data; + pages += *iptr; + if (freinfo > 1) { + char *bad = ""; + size_t pg, prev; + ssize_t i, j, span = 0; + j = *iptr++; + for (i = j, prev = 1; --i >= 0; ) { + pg = iptr[i]; + if (pg <= prev) + bad = " [bad sequence]"; + prev = pg; + pg += span; + for (; i >= span && iptr[i-span] == pg; span++, pg++) ; + } + printf(" Transaction %"Z"u, %"Z"d pages, maxspan %"Z"d%s\n", + *(size_t *)key.mv_data, j, span, bad); + if (freinfo > 2) { + for (--j; j >= 0; ) { + pg = iptr[j]; + for (span=1; --j >= 0 && iptr[j] == pg+span; span++) ; + printf(span>1 ? " %9"Z"u[%"Z"d]\n" : " %9"Z"u\n", + pg, span); + } + } + } + } + mdb_cursor_close(cursor); + printf(" Free pages: %"Z"u\n", pages); + } + + rc = mdb_open(txn, subname, 0, &dbi); + if (rc) { + fprintf(stderr, "mdb_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + + rc = mdb_stat(txn, dbi, &mst); + if (rc) { + fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + printf("Status of %s\n", subname ? subname : "Main DB"); + prstat(&mst); + + if (alldbs) { + MDB_cursor *cursor; + MDB_val key; + + rc = mdb_cursor_open(txn, dbi, &cursor); + if (rc) { + fprintf(stderr, "mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + while ((rc = mdb_cursor_get(cursor, &key, NULL, MDB_NEXT_NODUP)) == 0) { + char *str; + MDB_dbi db2; + if (memchr(key.mv_data, '\0', key.mv_size)) + continue; + str = malloc(key.mv_size+1); + memcpy(str, key.mv_data, key.mv_size); + str[key.mv_size] = '\0'; + rc = mdb_open(txn, str, 0, &db2); + if (rc == MDB_SUCCESS) + printf("Status of %s\n", str); + free(str); + if (rc) continue; + rc = mdb_stat(txn, db2, &mst); + if (rc) { + fprintf(stderr, "mdb_stat failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + prstat(&mst); + mdb_close(env, db2); + } + mdb_cursor_close(cursor); + } + + if (rc == MDB_NOTFOUND) + rc = MDB_SUCCESS; + + mdb_close(env, dbi); +txn_abort: + mdb_txn_abort(txn); +env_close: + mdb_env_close(env); + + return rc ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/external/db_drivers/liblmdb64/midl.c b/external/db_drivers/liblmdb64/midl.c new file mode 100644 index 000000000..16782dcaf --- /dev/null +++ b/external/db_drivers/liblmdb64/midl.c @@ -0,0 +1,360 @@ +/** @file midl.c + * @brief ldap bdb back-end ID List functions */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software <http://www.openldap.org/>. + * + * Copyright 2000-2015 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ + +#include <limits.h> +#include <string.h> +#include <stdlib.h> +#include <errno.h> +#include <sys/types.h> +#include "midl.h" + +/** @defgroup internal LMDB Internals + * @{ + */ +/** @defgroup idls ID List Management + * @{ + */ +#define CMP(x,y) ( (x) < (y) ? -1 : (x) > (y) ) + +unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id ) +{ + /* + * binary search of id in ids + * if found, returns position of id + * if not found, returns first position greater than id + */ + unsigned base = 0; + unsigned cursor = 1; + int val = 0; + unsigned n = ids[0]; + + while( 0 < n ) { + unsigned pivot = n >> 1; + cursor = base + pivot + 1; + val = CMP( ids[cursor], id ); + + if( val < 0 ) { + n = pivot; + + } else if ( val > 0 ) { + base = cursor; + n -= pivot + 1; + + } else { + return cursor; + } + } + + if( val > 0 ) { + ++cursor; + } + return cursor; +} + +#if 0 /* superseded by append/sort */ +int mdb_midl_insert( MDB_IDL ids, MDB_ID id ) +{ + unsigned x, i; + + x = mdb_midl_search( ids, id ); + assert( x > 0 ); + + if( x < 1 ) { + /* internal error */ + return -2; + } + + if ( x <= ids[0] && ids[x] == id ) { + /* duplicate */ + assert(0); + return -1; + } + + if ( ++ids[0] >= MDB_IDL_DB_MAX ) { + /* no room */ + --ids[0]; + return -2; + + } else { + /* insert id */ + for (i=ids[0]; i>x; i--) + ids[i] = ids[i-1]; + ids[x] = id; + } + + return 0; +} +#endif + +MDB_IDL mdb_midl_alloc(int num) +{ + MDB_IDL ids = malloc((num+2) * sizeof(MDB_ID)); + if (ids) { + *ids++ = num; + *ids = 0; + } + return ids; +} + +void mdb_midl_free(MDB_IDL ids) +{ + if (ids) + free(ids-1); +} + +int mdb_midl_shrink( MDB_IDL *idp ) +{ + MDB_IDL ids = *idp; + if (*(--ids) > MDB_IDL_UM_MAX && + (ids = realloc(ids, (MDB_IDL_UM_MAX+1) * sizeof(MDB_ID)))) + { + *ids++ = MDB_IDL_UM_MAX; + *idp = ids; + return 1; + } + return 0; +} + +static int mdb_midl_grow( MDB_IDL *idp, int num ) +{ + MDB_IDL idn = *idp-1; + /* grow it */ + idn = realloc(idn, (*idn + num + 2) * sizeof(MDB_ID)); + if (!idn) + return ENOMEM; + *idn++ += num; + *idp = idn; + return 0; +} + +int mdb_midl_need( MDB_IDL *idp, unsigned num ) +{ + MDB_IDL ids = *idp; + num += ids[0]; + if (num > ids[-1]) { + num = (num + num/4 + (256 + 2)) & -256; + if (!(ids = realloc(ids-1, num * sizeof(MDB_ID)))) + return ENOMEM; + *ids++ = num - 2; + *idp = ids; + } + return 0; +} + +int mdb_midl_append( MDB_IDL *idp, MDB_ID id ) +{ + MDB_IDL ids = *idp; + /* Too big? */ + if (ids[0] >= ids[-1]) { + if (mdb_midl_grow(idp, MDB_IDL_UM_MAX)) + return ENOMEM; + ids = *idp; + } + ids[0]++; + ids[ids[0]] = id; + return 0; +} + +int mdb_midl_append_list( MDB_IDL *idp, MDB_IDL app ) +{ + MDB_IDL ids = *idp; + /* Too big? */ + if (ids[0] + app[0] >= ids[-1]) { + if (mdb_midl_grow(idp, app[0])) + return ENOMEM; + ids = *idp; + } + memcpy(&ids[ids[0]+1], &app[1], app[0] * sizeof(MDB_ID)); + ids[0] += app[0]; + return 0; +} + +int mdb_midl_append_range( MDB_IDL *idp, MDB_ID id, unsigned n ) +{ + MDB_ID *ids = *idp, len = ids[0]; + /* Too big? */ + if (len + n > ids[-1]) { + if (mdb_midl_grow(idp, n | MDB_IDL_UM_MAX)) + return ENOMEM; + ids = *idp; + } + ids[0] = len + n; + ids += len; + while (n) + ids[n--] = id++; + return 0; +} + +void mdb_midl_xmerge( MDB_IDL idl, MDB_IDL merge ) +{ + MDB_ID old_id, merge_id, i = merge[0], j = idl[0], k = i+j, total = k; + idl[0] = (MDB_ID)-1; /* delimiter for idl scan below */ + old_id = idl[j]; + while (i) { + merge_id = merge[i--]; + for (; old_id < merge_id; old_id = idl[--j]) + idl[k--] = old_id; + idl[k--] = merge_id; + } + idl[0] = total; +} + +/* Quicksort + Insertion sort for small arrays */ + +#define SMALL 8 +#define MIDL_SWAP(a,b) { itmp=(a); (a)=(b); (b)=itmp; } + +void +mdb_midl_sort( MDB_IDL ids ) +{ + /* Max possible depth of int-indexed tree * 2 items/level */ + int istack[sizeof(int)*CHAR_BIT * 2]; + int i,j,k,l,ir,jstack; + MDB_ID a, itmp; + + ir = (int)ids[0]; + l = 1; + jstack = 0; + for(;;) { + if (ir - l < SMALL) { /* Insertion sort */ + for (j=l+1;j<=ir;j++) { + a = ids[j]; + for (i=j-1;i>=1;i--) { + if (ids[i] >= a) break; + ids[i+1] = ids[i]; + } + ids[i+1] = a; + } + if (jstack == 0) break; + ir = istack[jstack--]; + l = istack[jstack--]; + } else { + k = (l + ir) >> 1; /* Choose median of left, center, right */ + MIDL_SWAP(ids[k], ids[l+1]); + if (ids[l] < ids[ir]) { + MIDL_SWAP(ids[l], ids[ir]); + } + if (ids[l+1] < ids[ir]) { + MIDL_SWAP(ids[l+1], ids[ir]); + } + if (ids[l] < ids[l+1]) { + MIDL_SWAP(ids[l], ids[l+1]); + } + i = l+1; + j = ir; + a = ids[l+1]; + for(;;) { + do i++; while(ids[i] > a); + do j--; while(ids[j] < a); + if (j < i) break; + MIDL_SWAP(ids[i],ids[j]); + } + ids[l+1] = ids[j]; + ids[j] = a; + jstack += 2; + if (ir-i+1 >= j-l) { + istack[jstack] = ir; + istack[jstack-1] = i; + ir = j-1; + } else { + istack[jstack] = j-1; + istack[jstack-1] = l; + l = i; + } + } + } +} + +unsigned mdb_mid2l_search( MDB_ID2L ids, MDB_ID id ) +{ + /* + * binary search of id in ids + * if found, returns position of id + * if not found, returns first position greater than id + */ + unsigned base = 0; + unsigned cursor = 1; + int val = 0; + unsigned n = (unsigned)ids[0].mid; + + while( 0 < n ) { + unsigned pivot = n >> 1; + cursor = base + pivot + 1; + val = CMP( id, ids[cursor].mid ); + + if( val < 0 ) { + n = pivot; + + } else if ( val > 0 ) { + base = cursor; + n -= pivot + 1; + + } else { + return cursor; + } + } + + if( val > 0 ) { + ++cursor; + } + return cursor; +} + +int mdb_mid2l_insert( MDB_ID2L ids, MDB_ID2 *id ) +{ + unsigned x, i; + + x = mdb_mid2l_search( ids, id->mid ); + + if( x < 1 ) { + /* internal error */ + return -2; + } + + if ( x <= ids[0].mid && ids[x].mid == id->mid ) { + /* duplicate */ + return -1; + } + + if ( ids[0].mid >= MDB_IDL_UM_MAX ) { + /* too big */ + return -2; + + } else { + /* insert id */ + ids[0].mid++; + for (i=(unsigned)ids[0].mid; i>x; i--) + ids[i] = ids[i-1]; + ids[x] = *id; + } + + return 0; +} + +int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id ) +{ + /* Too big? */ + if (ids[0].mid >= MDB_IDL_UM_MAX) { + return -2; + } + ids[0].mid++; + ids[ids[0].mid] = *id; + return 0; +} + +/** @} */ +/** @} */ diff --git a/external/db_drivers/liblmdb64/midl.h b/external/db_drivers/liblmdb64/midl.h new file mode 100644 index 000000000..9b041d72d --- /dev/null +++ b/external/db_drivers/liblmdb64/midl.h @@ -0,0 +1,186 @@ +/** @file midl.h + * @brief LMDB ID List header file. + * + * This file was originally part of back-bdb but has been + * modified for use in libmdb. Most of the macros defined + * in this file are unused, just left over from the original. + * + * This file is only used internally in libmdb and its definitions + * are not exposed publicly. + */ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software <http://www.openldap.org/>. + * + * Copyright 2000-2015 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ + +#ifndef _MDB_MIDL_H_ +#define _MDB_MIDL_H_ + +#include <stddef.h> + +#ifdef __cplusplus +extern "C" { +#endif + +/** @defgroup internal LMDB Internals + * @{ + */ + +/** @defgroup idls ID List Management + * @{ + */ + /** A generic unsigned ID number. These were entryIDs in back-bdb. + * Preferably it should have the same size as a pointer. + */ +typedef size_t MDB_ID; + + /** An IDL is an ID List, a sorted array of IDs. The first + * element of the array is a counter for how many actual + * IDs are in the list. In the original back-bdb code, IDLs are + * sorted in ascending order. For libmdb IDLs are sorted in + * descending order. + */ +typedef MDB_ID *MDB_IDL; + +/* IDL sizes - likely should be even bigger + * limiting factors: sizeof(ID), thread stack size + */ +#define MDB_IDL_LOGN 16 /* DB_SIZE is 2^16, UM_SIZE is 2^17 */ +#define MDB_IDL_DB_SIZE (1<<MDB_IDL_LOGN) +#define MDB_IDL_UM_SIZE (1<<(MDB_IDL_LOGN+1)) + +#define MDB_IDL_DB_MAX (MDB_IDL_DB_SIZE-1) +#define MDB_IDL_UM_MAX (MDB_IDL_UM_SIZE-1) + +#define MDB_IDL_SIZEOF(ids) (((ids)[0]+1) * sizeof(MDB_ID)) +#define MDB_IDL_IS_ZERO(ids) ( (ids)[0] == 0 ) +#define MDB_IDL_CPY( dst, src ) (memcpy( dst, src, MDB_IDL_SIZEOF( src ) )) +#define MDB_IDL_FIRST( ids ) ( (ids)[1] ) +#define MDB_IDL_LAST( ids ) ( (ids)[(ids)[0]] ) + + /** Current max length of an #mdb_midl_alloc()ed IDL */ +#define MDB_IDL_ALLOCLEN( ids ) ( (ids)[-1] ) + + /** Append ID to IDL. The IDL must be big enough. */ +#define mdb_midl_xappend(idl, id) do { \ + MDB_ID *xidl = (idl), xlen = ++(xidl[0]); \ + xidl[xlen] = (id); \ + } while (0) + + /** Search for an ID in an IDL. + * @param[in] ids The IDL to search. + * @param[in] id The ID to search for. + * @return The index of the first ID greater than or equal to \b id. + */ +unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id ); + + /** Allocate an IDL. + * Allocates memory for an IDL of the given size. + * @return IDL on success, NULL on failure. + */ +MDB_IDL mdb_midl_alloc(int num); + + /** Free an IDL. + * @param[in] ids The IDL to free. + */ +void mdb_midl_free(MDB_IDL ids); + + /** Shrink an IDL. + * Return the IDL to the default size if it has grown larger. + * @param[in,out] idp Address of the IDL to shrink. + * @return 0 on no change, non-zero if shrunk. + */ +int mdb_midl_shrink(MDB_IDL *idp); + + /** Make room for num additional elements in an IDL. + * @param[in,out] idp Address of the IDL. + * @param[in] num Number of elements to make room for. + * @return 0 on success, ENOMEM on failure. + */ +int mdb_midl_need(MDB_IDL *idp, unsigned num); + + /** Append an ID onto an IDL. + * @param[in,out] idp Address of the IDL to append to. + * @param[in] id The ID to append. + * @return 0 on success, ENOMEM if the IDL is too large. + */ +int mdb_midl_append( MDB_IDL *idp, MDB_ID id ); + + /** Append an IDL onto an IDL. + * @param[in,out] idp Address of the IDL to append to. + * @param[in] app The IDL to append. + * @return 0 on success, ENOMEM if the IDL is too large. + */ +int mdb_midl_append_list( MDB_IDL *idp, MDB_IDL app ); + + /** Append an ID range onto an IDL. + * @param[in,out] idp Address of the IDL to append to. + * @param[in] id The lowest ID to append. + * @param[in] n Number of IDs to append. + * @return 0 on success, ENOMEM if the IDL is too large. + */ +int mdb_midl_append_range( MDB_IDL *idp, MDB_ID id, unsigned n ); + + /** Merge an IDL onto an IDL. The destination IDL must be big enough. + * @param[in] idl The IDL to merge into. + * @param[in] merge The IDL to merge. + */ +void mdb_midl_xmerge( MDB_IDL idl, MDB_IDL merge ); + + /** Sort an IDL. + * @param[in,out] ids The IDL to sort. + */ +void mdb_midl_sort( MDB_IDL ids ); + + /** An ID2 is an ID/pointer pair. + */ +typedef struct MDB_ID2 { + MDB_ID mid; /**< The ID */ + void *mptr; /**< The pointer */ +} MDB_ID2; + + /** An ID2L is an ID2 List, a sorted array of ID2s. + * The first element's \b mid member is a count of how many actual + * elements are in the array. The \b mptr member of the first element is unused. + * The array is sorted in ascending order by \b mid. + */ +typedef MDB_ID2 *MDB_ID2L; + + /** Search for an ID in an ID2L. + * @param[in] ids The ID2L to search. + * @param[in] id The ID to search for. + * @return The index of the first ID2 whose \b mid member is greater than or equal to \b id. + */ +unsigned mdb_mid2l_search( MDB_ID2L ids, MDB_ID id ); + + + /** Insert an ID2 into a ID2L. + * @param[in,out] ids The ID2L to insert into. + * @param[in] id The ID2 to insert. + * @return 0 on success, -1 if the ID was already present in the ID2L. + */ +int mdb_mid2l_insert( MDB_ID2L ids, MDB_ID2 *id ); + + /** Append an ID2 into a ID2L. + * @param[in,out] ids The ID2L to append into. + * @param[in] id The ID2 to append. + * @return 0 on success, -2 if the ID2L is too big. + */ +int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id ); + +/** @} */ +/** @} */ +#ifdef __cplusplus +} +#endif +#endif /* _MDB_MIDL_H_ */ diff --git a/external/db_drivers/liblmdb64/mtest.c b/external/db_drivers/liblmdb64/mtest.c new file mode 100644 index 000000000..66dabc752 --- /dev/null +++ b/external/db_drivers/liblmdb64/mtest.c @@ -0,0 +1,173 @@ +/* mtest.c - memory-mapped database tester/toy */ +/* + * Copyright 2011-2015 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ +#include <stdio.h> +#include <stdlib.h> +#include <time.h> +#include "lmdb.h" + +#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) +#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) +#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \ + "%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort())) + +int main(int argc,char * argv[]) +{ + int i = 0, j = 0, rc; + MDB_env *env; + MDB_dbi dbi; + MDB_val key, data; + MDB_txn *txn; + MDB_stat mst; + MDB_cursor *cursor, *cur2; + MDB_cursor_op op; + int count; + int *values; + char sval[32] = ""; + + srand(time(NULL)); + + count = (rand()%384) + 64; + values = (int *)malloc(count*sizeof(int)); + + for(i = 0;i<count;i++) { + values[i] = rand()%1024; + } + + E(mdb_env_create(&env)); + E(mdb_env_set_mapsize(env, 10485760)); + E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP /*|MDB_NOSYNC*/, 0664)); + E(mdb_txn_begin(env, NULL, 0, &txn)); + E(mdb_open(txn, NULL, 0, &dbi)); + + key.mv_size = sizeof(int); + key.mv_data = sval; + data.mv_size = sizeof(sval); + data.mv_data = sval; + + printf("Adding %d values\n", count); + for (i=0;i<count;i++) { + sprintf(sval, "%03x %d foo bar", values[i], values[i]); + if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) { + j++; + data.mv_size = sizeof(sval); + data.mv_data = sval; + } + } + if (j) printf("%d duplicates skipped\n", j); + E(mdb_txn_commit(txn)); + E(mdb_env_stat(env, &mst)); + + E(mdb_txn_begin(env, NULL, 1, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %p %.*s, data: %p %.*s\n", + key.mv_data, (int) key.mv_size, (char *) key.mv_data, + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + mdb_cursor_close(cursor); + mdb_txn_abort(txn); + + j=0; + key.mv_data = sval; + for (i= count - 1; i > -1; i-= (rand()%5)) { + j++; + txn=NULL; + E(mdb_txn_begin(env, NULL, 0, &txn)); + sprintf(sval, "%03x ", values[i]); + if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) { + j--; + mdb_txn_abort(txn); + } else { + E(mdb_txn_commit(txn)); + } + } + free(values); + printf("Deleted %d values\n", j); + + E(mdb_env_stat(env, &mst)); + E(mdb_txn_begin(env, NULL, 1, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + printf("Cursor next\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + printf("Cursor last\n"); + E(mdb_cursor_get(cursor, &key, &data, MDB_LAST)); + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + printf("Cursor prev\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + printf("Cursor last/prev\n"); + E(mdb_cursor_get(cursor, &key, &data, MDB_LAST)); + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + E(mdb_cursor_get(cursor, &key, &data, MDB_PREV)); + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + + mdb_txn_abort(txn); + + printf("Deleting with cursor\n"); + E(mdb_txn_begin(env, NULL, 0, &txn)); + E(mdb_cursor_open(txn, dbi, &cur2)); + for (i=0; i<50; i++) { + if (RES(MDB_NOTFOUND, mdb_cursor_get(cur2, &key, &data, MDB_NEXT))) + break; + printf("key: %p %.*s, data: %p %.*s\n", + key.mv_data, (int) key.mv_size, (char *) key.mv_data, + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + E(mdb_del(txn, dbi, &key, NULL)); + } + + printf("Restarting cursor in txn\n"); + for (op=MDB_FIRST, i=0; i<=32; op=MDB_NEXT, i++) { + if (RES(MDB_NOTFOUND, mdb_cursor_get(cur2, &key, &data, op))) + break; + printf("key: %p %.*s, data: %p %.*s\n", + key.mv_data, (int) key.mv_size, (char *) key.mv_data, + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + } + mdb_cursor_close(cur2); + E(mdb_txn_commit(txn)); + + printf("Restarting cursor outside txn\n"); + E(mdb_txn_begin(env, NULL, 0, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + for (op=MDB_FIRST, i=0; i<=32; op=MDB_NEXT, i++) { + if (RES(MDB_NOTFOUND, mdb_cursor_get(cursor, &key, &data, op))) + break; + printf("key: %p %.*s, data: %p %.*s\n", + key.mv_data, (int) key.mv_size, (char *) key.mv_data, + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + } + mdb_cursor_close(cursor); + mdb_close(env, dbi); + + mdb_txn_abort(txn); + mdb_env_close(env); + + return 0; +} diff --git a/external/db_drivers/liblmdb64/mtest2.c b/external/db_drivers/liblmdb64/mtest2.c new file mode 100644 index 000000000..33e0e741c --- /dev/null +++ b/external/db_drivers/liblmdb64/mtest2.c @@ -0,0 +1,123 @@ +/* mtest2.c - memory-mapped database tester/toy */ +/* + * Copyright 2011-2015 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ + +/* Just like mtest.c, but using a subDB instead of the main DB */ + +#include <stdio.h> +#include <stdlib.h> +#include <time.h> +#include "lmdb.h" + +#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) +#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) +#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \ + "%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort())) + +int main(int argc,char * argv[]) +{ + int i = 0, j = 0, rc; + MDB_env *env; + MDB_dbi dbi; + MDB_val key, data; + MDB_txn *txn; + MDB_stat mst; + MDB_cursor *cursor; + int count; + int *values; + char sval[32] = ""; + + srand(time(NULL)); + + count = (rand()%384) + 64; + values = (int *)malloc(count*sizeof(int)); + + for(i = 0;i<count;i++) { + values[i] = rand()%1024; + } + + E(mdb_env_create(&env)); + E(mdb_env_set_mapsize(env, 10485760)); + E(mdb_env_set_maxdbs(env, 4)); + E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664)); + E(mdb_txn_begin(env, NULL, 0, &txn)); + E(mdb_open(txn, "id1", MDB_CREATE, &dbi)); + + key.mv_size = sizeof(int); + key.mv_data = sval; + data.mv_size = sizeof(sval); + data.mv_data = sval; + + printf("Adding %d values\n", count); + for (i=0;i<count;i++) { + sprintf(sval, "%03x %d foo bar", values[i], values[i]); + if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) + j++; + } + if (j) printf("%d duplicates skipped\n", j); + E(mdb_txn_commit(txn)); + E(mdb_env_stat(env, &mst)); + + E(mdb_txn_begin(env, NULL, 1, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %p %.*s, data: %p %.*s\n", + key.mv_data, (int) key.mv_size, (char *) key.mv_data, + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + mdb_cursor_close(cursor); + mdb_txn_abort(txn); + + j=0; + key.mv_data = sval; + for (i= count - 1; i > -1; i-= (rand()%5)) { + j++; + txn=NULL; + E(mdb_txn_begin(env, NULL, 0, &txn)); + sprintf(sval, "%03x ", values[i]); + if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, NULL))) { + j--; + mdb_txn_abort(txn); + } else { + E(mdb_txn_commit(txn)); + } + } + free(values); + printf("Deleted %d values\n", j); + + E(mdb_env_stat(env, &mst)); + E(mdb_txn_begin(env, NULL, 1, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + printf("Cursor next\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + printf("Cursor prev\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + mdb_cursor_close(cursor); + mdb_close(env, dbi); + + mdb_txn_abort(txn); + mdb_env_close(env); + + return 0; +} diff --git a/external/db_drivers/liblmdb64/mtest3.c b/external/db_drivers/liblmdb64/mtest3.c new file mode 100644 index 000000000..85872c67e --- /dev/null +++ b/external/db_drivers/liblmdb64/mtest3.c @@ -0,0 +1,133 @@ +/* mtest3.c - memory-mapped database tester/toy */ +/* + * Copyright 2011-2015 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ + +/* Tests for sorted duplicate DBs */ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include "lmdb.h" + +#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) +#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) +#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \ + "%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort())) + +int main(int argc,char * argv[]) +{ + int i = 0, j = 0, rc; + MDB_env *env; + MDB_dbi dbi; + MDB_val key, data; + MDB_txn *txn; + MDB_stat mst; + MDB_cursor *cursor; + int count; + int *values; + char sval[32]; + char kval[sizeof(int)]; + + srand(time(NULL)); + + memset(sval, 0, sizeof(sval)); + + count = (rand()%384) + 64; + values = (int *)malloc(count*sizeof(int)); + + for(i = 0;i<count;i++) { + values[i] = rand()%1024; + } + + E(mdb_env_create(&env)); + E(mdb_env_set_mapsize(env, 10485760)); + E(mdb_env_set_maxdbs(env, 4)); + E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664)); + E(mdb_txn_begin(env, NULL, 0, &txn)); + E(mdb_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi)); + + key.mv_size = sizeof(int); + key.mv_data = kval; + data.mv_size = sizeof(sval); + data.mv_data = sval; + + printf("Adding %d values\n", count); + for (i=0;i<count;i++) { + if (!(i & 0x0f)) + sprintf(kval, "%03x", values[i]); + sprintf(sval, "%03x %d foo bar", values[i], values[i]); + if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA))) + j++; + } + if (j) printf("%d duplicates skipped\n", j); + E(mdb_txn_commit(txn)); + E(mdb_env_stat(env, &mst)); + + E(mdb_txn_begin(env, NULL, 1, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %p %.*s, data: %p %.*s\n", + key.mv_data, (int) key.mv_size, (char *) key.mv_data, + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + mdb_cursor_close(cursor); + mdb_txn_abort(txn); + + j=0; + + for (i= count - 1; i > -1; i-= (rand()%5)) { + j++; + txn=NULL; + E(mdb_txn_begin(env, NULL, 0, &txn)); + sprintf(kval, "%03x", values[i & ~0x0f]); + sprintf(sval, "%03x %d foo bar", values[i], values[i]); + key.mv_size = sizeof(int); + key.mv_data = kval; + data.mv_size = sizeof(sval); + data.mv_data = sval; + if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) { + j--; + mdb_txn_abort(txn); + } else { + E(mdb_txn_commit(txn)); + } + } + free(values); + printf("Deleted %d values\n", j); + + E(mdb_env_stat(env, &mst)); + E(mdb_txn_begin(env, NULL, 1, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + printf("Cursor next\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + printf("Cursor prev\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + mdb_cursor_close(cursor); + mdb_close(env, dbi); + + mdb_txn_abort(txn); + mdb_env_close(env); + + return 0; +} diff --git a/external/db_drivers/liblmdb64/mtest4.c b/external/db_drivers/liblmdb64/mtest4.c new file mode 100644 index 000000000..fdcd46a37 --- /dev/null +++ b/external/db_drivers/liblmdb64/mtest4.c @@ -0,0 +1,168 @@ +/* mtest4.c - memory-mapped database tester/toy */ +/* + * Copyright 2011-2015 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ + +/* Tests for sorted duplicate DBs with fixed-size keys */ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include "lmdb.h" + +#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) +#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) +#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \ + "%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort())) + +int main(int argc,char * argv[]) +{ + int i = 0, j = 0, rc; + MDB_env *env; + MDB_dbi dbi; + MDB_val key, data; + MDB_txn *txn; + MDB_stat mst; + MDB_cursor *cursor; + int count; + int *values; + char sval[8]; + char kval[sizeof(int)]; + + memset(sval, 0, sizeof(sval)); + + count = 510; + values = (int *)malloc(count*sizeof(int)); + + for(i = 0;i<count;i++) { + values[i] = i*5; + } + + E(mdb_env_create(&env)); + E(mdb_env_set_mapsize(env, 10485760)); + E(mdb_env_set_maxdbs(env, 4)); + E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664)); + E(mdb_txn_begin(env, NULL, 0, &txn)); + E(mdb_open(txn, "id4", MDB_CREATE|MDB_DUPSORT|MDB_DUPFIXED, &dbi)); + + key.mv_size = sizeof(int); + key.mv_data = kval; + data.mv_size = sizeof(sval); + data.mv_data = sval; + + printf("Adding %d values\n", count); + strcpy(kval, "001"); + for (i=0;i<count;i++) { + sprintf(sval, "%07x", values[i]); + if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA))) + j++; + } + if (j) printf("%d duplicates skipped\n", j); + E(mdb_txn_commit(txn)); + E(mdb_env_stat(env, &mst)); + + /* there should be one full page of dups now. + */ + E(mdb_txn_begin(env, NULL, 1, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %p %.*s, data: %p %.*s\n", + key.mv_data, (int) key.mv_size, (char *) key.mv_data, + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + mdb_cursor_close(cursor); + mdb_txn_abort(txn); + + /* test all 3 branches of split code: + * 1: new key in lower half + * 2: new key at split point + * 3: new key in upper half + */ + + key.mv_size = sizeof(int); + key.mv_data = kval; + data.mv_size = sizeof(sval); + data.mv_data = sval; + + sprintf(sval, "%07x", values[3]+1); + E(mdb_txn_begin(env, NULL, 0, &txn)); + (void)RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA)); + mdb_txn_abort(txn); + + sprintf(sval, "%07x", values[255]+1); + E(mdb_txn_begin(env, NULL, 0, &txn)); + (void)RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA)); + mdb_txn_abort(txn); + + sprintf(sval, "%07x", values[500]+1); + E(mdb_txn_begin(env, NULL, 0, &txn)); + (void)RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NODUPDATA)); + E(mdb_txn_commit(txn)); + + /* Try MDB_NEXT_MULTIPLE */ + E(mdb_txn_begin(env, NULL, 0, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT_MULTIPLE)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + mdb_cursor_close(cursor); + mdb_txn_abort(txn); + j=0; + + for (i= count - 1; i > -1; i-= (rand()%3)) { + j++; + txn=NULL; + E(mdb_txn_begin(env, NULL, 0, &txn)); + sprintf(sval, "%07x", values[i]); + key.mv_size = sizeof(int); + key.mv_data = kval; + data.mv_size = sizeof(sval); + data.mv_data = sval; + if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) { + j--; + mdb_txn_abort(txn); + } else { + E(mdb_txn_commit(txn)); + } + } + free(values); + printf("Deleted %d values\n", j); + + E(mdb_env_stat(env, &mst)); + E(mdb_txn_begin(env, NULL, 1, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + printf("Cursor next\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + printf("Cursor prev\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + mdb_cursor_close(cursor); + mdb_close(env, dbi); + + mdb_txn_abort(txn); + mdb_env_close(env); + + return 0; +} diff --git a/external/db_drivers/liblmdb64/mtest5.c b/external/db_drivers/liblmdb64/mtest5.c new file mode 100644 index 000000000..5295bce41 --- /dev/null +++ b/external/db_drivers/liblmdb64/mtest5.c @@ -0,0 +1,135 @@ +/* mtest5.c - memory-mapped database tester/toy */ +/* + * Copyright 2011-2015 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ + +/* Tests for sorted duplicate DBs using cursor_put */ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include "lmdb.h" + +#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) +#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) +#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \ + "%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort())) + +int main(int argc,char * argv[]) +{ + int i = 0, j = 0, rc; + MDB_env *env; + MDB_dbi dbi; + MDB_val key, data; + MDB_txn *txn; + MDB_stat mst; + MDB_cursor *cursor; + int count; + int *values; + char sval[32]; + char kval[sizeof(int)]; + + srand(time(NULL)); + + memset(sval, 0, sizeof(sval)); + + count = (rand()%384) + 64; + values = (int *)malloc(count*sizeof(int)); + + for(i = 0;i<count;i++) { + values[i] = rand()%1024; + } + + E(mdb_env_create(&env)); + E(mdb_env_set_mapsize(env, 10485760)); + E(mdb_env_set_maxdbs(env, 4)); + E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664)); + E(mdb_txn_begin(env, NULL, 0, &txn)); + E(mdb_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi)); + E(mdb_cursor_open(txn, dbi, &cursor)); + + key.mv_size = sizeof(int); + key.mv_data = kval; + data.mv_size = sizeof(sval); + data.mv_data = sval; + + printf("Adding %d values\n", count); + for (i=0;i<count;i++) { + if (!(i & 0x0f)) + sprintf(kval, "%03x", values[i]); + sprintf(sval, "%03x %d foo bar", values[i], values[i]); + if (RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NODUPDATA))) + j++; + } + if (j) printf("%d duplicates skipped\n", j); + mdb_cursor_close(cursor); + E(mdb_txn_commit(txn)); + E(mdb_env_stat(env, &mst)); + + E(mdb_txn_begin(env, NULL, 1, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %p %.*s, data: %p %.*s\n", + key.mv_data, (int) key.mv_size, (char *) key.mv_data, + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + mdb_cursor_close(cursor); + mdb_txn_abort(txn); + + j=0; + + for (i= count - 1; i > -1; i-= (rand()%5)) { + j++; + txn=NULL; + E(mdb_txn_begin(env, NULL, 0, &txn)); + sprintf(kval, "%03x", values[i & ~0x0f]); + sprintf(sval, "%03x %d foo bar", values[i], values[i]); + key.mv_size = sizeof(int); + key.mv_data = kval; + data.mv_size = sizeof(sval); + data.mv_data = sval; + if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) { + j--; + mdb_txn_abort(txn); + } else { + E(mdb_txn_commit(txn)); + } + } + free(values); + printf("Deleted %d values\n", j); + + E(mdb_env_stat(env, &mst)); + E(mdb_txn_begin(env, NULL, 1, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + printf("Cursor next\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + printf("Cursor prev\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + mdb_cursor_close(cursor); + mdb_close(env, dbi); + + mdb_txn_abort(txn); + mdb_env_close(env); + + return 0; +} diff --git a/external/db_drivers/liblmdb64/mtest6.c b/external/db_drivers/liblmdb64/mtest6.c new file mode 100644 index 000000000..b351c1a06 --- /dev/null +++ b/external/db_drivers/liblmdb64/mtest6.c @@ -0,0 +1,137 @@ +/* mtest6.c - memory-mapped database tester/toy */ +/* + * Copyright 2011-2015 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ + +/* Tests for DB splits and merges */ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include "lmdb.h" + +#define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr) +#define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0)) +#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \ + "%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort())) + +char dkbuf[1024]; + +int main(int argc,char * argv[]) +{ + int i = 0, j = 0, rc; + MDB_env *env; + MDB_dbi dbi; + MDB_val key, data; + MDB_txn *txn; + MDB_stat mst; + MDB_cursor *cursor; + int count; + int *values; + long kval; + char *sval; + + srand(time(NULL)); + + E(mdb_env_create(&env)); + E(mdb_env_set_mapsize(env, 10485760)); + E(mdb_env_set_maxdbs(env, 4)); + E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664)); + E(mdb_txn_begin(env, NULL, 0, &txn)); + E(mdb_open(txn, "id6", MDB_CREATE|MDB_INTEGERKEY, &dbi)); + E(mdb_cursor_open(txn, dbi, &cursor)); + E(mdb_stat(txn, dbi, &mst)); + + sval = calloc(1, mst.ms_psize / 4); + key.mv_size = sizeof(long); + key.mv_data = &kval; + data.mv_size = mst.ms_psize / 4 - 30; + data.mv_data = sval; + + printf("Adding 12 values, should yield 3 splits\n"); + for (i=0;i<12;i++) { + kval = i*5; + sprintf(sval, "%08x", kval); + (void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE)); + } + printf("Adding 12 more values, should yield 3 splits\n"); + for (i=0;i<12;i++) { + kval = i*5+4; + sprintf(sval, "%08x", kval); + (void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE)); + } + printf("Adding 12 more values, should yield 3 splits\n"); + for (i=0;i<12;i++) { + kval = i*5+1; + sprintf(sval, "%08x", kval); + (void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE)); + } + E(mdb_cursor_get(cursor, &key, &data, MDB_FIRST)); + + do { + printf("key: %p %s, data: %p %.*s\n", + key.mv_data, mdb_dkey(&key, dkbuf), + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + } while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0); + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + mdb_cursor_close(cursor); + mdb_txn_commit(txn); + +#if 0 + j=0; + + for (i= count - 1; i > -1; i-= (rand()%5)) { + j++; + txn=NULL; + E(mdb_txn_begin(env, NULL, 0, &txn)); + sprintf(kval, "%03x", values[i & ~0x0f]); + sprintf(sval, "%03x %d foo bar", values[i], values[i]); + key.mv_size = sizeof(int); + key.mv_data = kval; + data.mv_size = sizeof(sval); + data.mv_data = sval; + if (RES(MDB_NOTFOUND, mdb_del(txn, dbi, &key, &data))) { + j--; + mdb_txn_abort(txn); + } else { + E(mdb_txn_commit(txn)); + } + } + free(values); + printf("Deleted %d values\n", j); + + E(mdb_env_stat(env, &mst)); + E(mdb_txn_begin(env, NULL, 1, &txn)); + E(mdb_cursor_open(txn, dbi, &cursor)); + printf("Cursor next\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + printf("Cursor prev\n"); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_PREV)) == 0) { + printf("key: %.*s, data: %.*s\n", + (int) key.mv_size, (char *) key.mv_data, + (int) data.mv_size, (char *) data.mv_data); + } + CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get"); + mdb_cursor_close(cursor); + mdb_close(env, dbi); + + mdb_txn_abort(txn); +#endif + mdb_env_close(env); + + return 0; +} diff --git a/external/db_drivers/liblmdb64/sample-bdb.txt b/external/db_drivers/liblmdb64/sample-bdb.txt new file mode 100644 index 000000000..563807a2b --- /dev/null +++ b/external/db_drivers/liblmdb64/sample-bdb.txt @@ -0,0 +1,73 @@ +/* sample-bdb.txt - BerkeleyDB toy/sample + * + * Do a line-by-line comparison of this and sample-mdb.txt + */ +/* + * Copyright 2012-2015 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ +#include <stdio.h> +#include <string.h> +#include <db.h> + +int main(int argc,char * argv[]) +{ + int rc; + DB_ENV *env; + DB *dbi; + DBT key, data; + DB_TXN *txn; + DBC *cursor; + char sval[32], kval[32]; + + /* Note: Most error checking omitted for simplicity */ + +#define FLAGS (DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_TXN|DB_INIT_MPOOL|DB_CREATE|DB_THREAD) + rc = db_env_create(&env, 0); + rc = env->open(env, "./testdb", FLAGS, 0664); + rc = db_create(&dbi, env, 0); + rc = env->txn_begin(env, NULL, &txn, 0); + rc = dbi->open(dbi, txn, "test.bdb", NULL, DB_BTREE, DB_CREATE, 0664); + + memset(&key, 0, sizeof(DBT)); + memset(&data, 0, sizeof(DBT)); + key.size = sizeof(int); + key.data = sval; + data.size = sizeof(sval); + data.data = sval; + + sprintf(sval, "%03x %d foo bar", 32, 3141592); + rc = dbi->put(dbi, txn, &key, &data, 0); + rc = txn->commit(txn, 0); + if (rc) { + fprintf(stderr, "txn->commit: (%d) %s\n", rc, db_strerror(rc)); + goto leave; + } + rc = env->txn_begin(env, NULL, &txn, 0); + rc = dbi->cursor(dbi, txn, &cursor, 0); + key.flags = DB_DBT_USERMEM; + key.data = kval; + key.ulen = sizeof(kval); + data.flags = DB_DBT_USERMEM; + data.data = sval; + data.ulen = sizeof(sval); + while ((rc = cursor->c_get(cursor, &key, &data, DB_NEXT)) == 0) { + printf("key: %p %.*s, data: %p %.*s\n", + key.data, (int) key.size, (char *) key.data, + data.data, (int) data.size, (char *) data.data); + } + rc = cursor->c_close(cursor); + rc = txn->abort(txn); +leave: + rc = dbi->close(dbi, 0); + rc = env->close(env, 0); + return rc; +} diff --git a/external/db_drivers/liblmdb64/sample-mdb.txt b/external/db_drivers/liblmdb64/sample-mdb.txt new file mode 100644 index 000000000..d311f8e95 --- /dev/null +++ b/external/db_drivers/liblmdb64/sample-mdb.txt @@ -0,0 +1,62 @@ +/* sample-mdb.txt - MDB toy/sample + * + * Do a line-by-line comparison of this and sample-bdb.txt + */ +/* + * Copyright 2012-2015 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * <http://www.OpenLDAP.org/license.html>. + */ +#include <stdio.h> +#include "lmdb.h" + +int main(int argc,char * argv[]) +{ + int rc; + MDB_env *env; + MDB_dbi dbi; + MDB_val key, data; + MDB_txn *txn; + MDB_cursor *cursor; + char sval[32]; + + /* Note: Most error checking omitted for simplicity */ + + rc = mdb_env_create(&env); + rc = mdb_env_open(env, "./testdb", 0, 0664); + rc = mdb_txn_begin(env, NULL, 0, &txn); + rc = mdb_open(txn, NULL, 0, &dbi); + + key.mv_size = sizeof(int); + key.mv_data = sval; + data.mv_size = sizeof(sval); + data.mv_data = sval; + + sprintf(sval, "%03x %d foo bar", 32, 3141592); + rc = mdb_put(txn, dbi, &key, &data, 0); + rc = mdb_txn_commit(txn); + if (rc) { + fprintf(stderr, "mdb_txn_commit: (%d) %s\n", rc, mdb_strerror(rc)); + goto leave; + } + rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + rc = mdb_cursor_open(txn, dbi, &cursor); + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + printf("key: %p %.*s, data: %p %.*s\n", + key.mv_data, (int) key.mv_size, (char *) key.mv_data, + data.mv_data, (int) data.mv_size, (char *) data.mv_data); + } + mdb_cursor_close(cursor); + mdb_txn_abort(txn); +leave: + mdb_close(env, dbi); + mdb_env_close(env); + return 0; +} diff --git a/external/db_drivers/liblmdb64/tooltag b/external/db_drivers/liblmdb64/tooltag new file mode 100644 index 000000000..229bf16ba --- /dev/null +++ b/external/db_drivers/liblmdb64/tooltag @@ -0,0 +1,22 @@ +<tagfile> + <compound kind="page"> + <name>mdb_copy_1</name> + <title>mdb_copy - environment copy tool</title> + <filename>mdb_copy.1</filename> + </compound> + <compound kind="page"> + <name>mdb_dump_1</name> + <title>mdb_dump - environment export tool</title> + <filename>mdb_dump.1</filename> + </compound> + <compound kind="page"> + <name>mdb_load_1</name> + <title>mdb_load - environment import tool</title> + <filename>mdb_load.1</filename> + </compound> + <compound kind="page"> + <name>mdb_stat_1</name> + <title>mdb_stat - environment status tool</title> + <filename>mdb_stat.1</filename> + </compound> +</tagfile> diff --git a/external/unbound/CMakeLists.txt b/external/unbound/CMakeLists.txt index fdfa0a0fe..72e2addd6 100644 --- a/external/unbound/CMakeLists.txt +++ b/external/unbound/CMakeLists.txt @@ -1,3 +1,31 @@ +# Copyright (c) 2014-2015, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + cmake_minimum_required(VERSION 2.8.7) project(unbound C) @@ -104,7 +132,7 @@ set(common_src set(compat_src) -foreach (symbol IN ITEMS ctime_r gmtime_r inet_aton inet_ntop inet_pton malloc memcmp memmove snprintf strlcat strlcpy strptime explicit_bzero arc4random arc4random_uniform sha512) +foreach (symbol IN ITEMS ctime_r gmtime_r inet_aton inet_ntop inet_pton malloc memcmp memmove snprintf strlcat strlcpy strptime explicit_bzero arc4random arc4random_uniform sha512 reallocarray) string(TOUPPER "${symbol}" upper_sym) if (NOT HAVE_${upper_sym}) list(APPEND compat_src @@ -137,13 +165,13 @@ if (NOT HAVE_GETADDRINFO) endif () set(sldns_src - ldns/keyraw.c - ldns/sbuffer.c - ldns/wire2str.c - ldns/parse.c - ldns/parseutil.c - ldns/rrdef.c - ldns/str2wire.c) + sldns/keyraw.c + sldns/sbuffer.c + sldns/wire2str.c + sldns/parse.c + sldns/parseutil.c + sldns/rrdef.c + sldns/str2wire.c) set(libunbound_src libunbound/context.c diff --git a/external/unbound/Makefile.in b/external/unbound/Makefile.in index 02532a951..8ca74367a 100644 --- a/external/unbound/Makefile.in +++ b/external/unbound/Makefile.in @@ -131,12 +131,12 @@ compat/memcmp.c compat/memmove.c compat/snprintf.c compat/strlcat.c \ compat/strlcpy.c compat/strptime.c compat/getentropy_linux.c \ compat/getentropy_osx.c compat/getentropy_solaris.c compat/getentropy_win.c \ compat/explicit_bzero.c compat/arc4random.c compat/arc4random_uniform.c \ -compat/arc4_lock.c compat/sha512.c +compat/arc4_lock.c compat/sha512.c compat/reallocarray.c COMPAT_OBJ=$(LIBOBJS:.o=.lo) COMPAT_OBJ_WITHOUT_CTIME=$(LIBOBJ_WITHOUT_CTIME:.o=.lo) COMPAT_OBJ_WITHOUT_CTIMEARC4=$(LIBOBJ_WITHOUT_CTIMEARC4:.o=.lo) -SLDNS_SRC=ldns/keyraw.c ldns/sbuffer.c ldns/wire2str.c ldns/parse.c \ -ldns/parseutil.c ldns/rrdef.c ldns/str2wire.c +SLDNS_SRC=sldns/keyraw.c sldns/sbuffer.c sldns/wire2str.c sldns/parse.c \ +sldns/parseutil.c sldns/rrdef.c sldns/str2wire.c SLDNS_OBJ=keyraw.lo sbuffer.lo wire2str.lo parse.lo parseutil.lo rrdef.lo \ str2wire.lo UNITTEST_SRC=testcode/unitanchor.c testcode/unitdname.c \ @@ -459,8 +459,8 @@ strip: $(STRIP) unbound$(EXEEXT) $(STRIP) unbound-checkconf$(EXEEXT) $(STRIP) unbound-control$(EXEEXT) - $(STRIP) unbound-host$(EXEEXT) - $(STRIP) unbound-anchor$(EXEEXT) + $(STRIP) unbound-host$(EXEEXT) || $(STRIP) .libs/unbound-host$(EXEEXT) + $(STRIP) unbound-anchor$(EXEEXT) || $(STRIP) .libs/unbound-anchor$(EXEEXT) pythonmod-install: $(INSTALL) -m 755 -d $(DESTDIR)$(PYTHON_SITE_PKG) @@ -576,6 +576,7 @@ depend: -e 's?$$(srcdir)/util/configlexer.c?util/configlexer.c?g' \ -e 's?$$(srcdir)/util/configparser.c?util/configparser.c?g' \ -e 's?$$(srcdir)/util/configparser.h?util/configparser.h?g' \ + -e 's?$$(srcdir)/dnstap/dnstap_config.h??g' \ -e 's?$$(srcdir)/pythonmod/pythonmod.h?$$(PYTHONMOD_HEADER)?g' \ -e 's!\(.*\)\.o[ :]*!\1.lo \1.o: !g' \ > $(DEPEND_TMP) @@ -596,89 +597,89 @@ dns.lo dns.o: $(srcdir)/services/cache/dns.c config.h $(srcdir)/iterator/iter_de $(srcdir)/validator/val_nsec.h $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/storage/lruhash.h \ $(srcdir)/util/locks.h $(srcdir)/services/cache/dns.h $(srcdir)/util/data/msgreply.h \ $(srcdir)/services/cache/rrset.h $(srcdir)/util/storage/slabhash.h $(srcdir)/util/data/dname.h \ - $(srcdir)/util/module.h $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h \ - $(srcdir)/util/net_help.h $(srcdir)/util/regional.h $(srcdir)/util/config_file.h $(srcdir)/ldns/sbuffer.h -infra.lo infra.o: $(srcdir)/services/cache/infra.c config.h $(srcdir)/ldns/rrdef.h \ + $(srcdir)/util/module.h $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h \ + $(srcdir)/util/net_help.h $(srcdir)/util/regional.h $(srcdir)/util/config_file.h $(srcdir)/sldns/sbuffer.h +infra.lo infra.o: $(srcdir)/services/cache/infra.c config.h $(srcdir)/sldns/rrdef.h \ $(srcdir)/services/cache/infra.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h \ $(srcdir)/util/rtt.h $(srcdir)/util/storage/slabhash.h $(srcdir)/util/storage/lookup3.h \ $(srcdir)/util/data/dname.h $(srcdir)/util/net_help.h $(srcdir)/util/config_file.h $(srcdir)/iterator/iterator.h \ $(srcdir)/services/outbound_list.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/packed_rrset.h \ - $(srcdir)/util/module.h $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h + $(srcdir)/util/module.h $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h rrset.lo rrset.o: $(srcdir)/services/cache/rrset.c config.h $(srcdir)/services/cache/rrset.h \ $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/util/storage/slabhash.h \ - $(srcdir)/util/data/packed_rrset.h $(srcdir)/ldns/rrdef.h $(srcdir)/util/config_file.h \ + $(srcdir)/util/data/packed_rrset.h $(srcdir)/sldns/rrdef.h $(srcdir)/util/config_file.h \ $(srcdir)/util/data/msgreply.h $(srcdir)/util/regional.h $(srcdir)/util/alloc.h dname.lo dname.o: $(srcdir)/util/data/dname.c config.h $(srcdir)/util/data/dname.h \ $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/util/data/msgparse.h \ - $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h $(srcdir)/util/storage/lookup3.h $(srcdir)/ldns/sbuffer.h + $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/util/storage/lookup3.h $(srcdir)/sldns/sbuffer.h msgencode.lo msgencode.o: $(srcdir)/util/data/msgencode.c config.h $(srcdir)/util/data/msgencode.h \ $(srcdir)/util/data/msgreply.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h \ - $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h \ - $(srcdir)/util/data/dname.h $(srcdir)/util/regional.h $(srcdir)/util/net_help.h $(srcdir)/ldns/sbuffer.h + $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h \ + $(srcdir)/util/data/dname.h $(srcdir)/util/regional.h $(srcdir)/util/net_help.h $(srcdir)/sldns/sbuffer.h msgparse.lo msgparse.o: $(srcdir)/util/data/msgparse.c config.h $(srcdir)/util/data/msgparse.h \ - $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/ldns/pkthdr.h \ - $(srcdir)/ldns/rrdef.h $(srcdir)/util/data/dname.h $(srcdir)/util/data/packed_rrset.h \ - $(srcdir)/util/storage/lookup3.h $(srcdir)/util/regional.h $(srcdir)/ldns/sbuffer.h $(srcdir)/ldns/parseutil.h \ - $(srcdir)/ldns/wire2str.h + $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/sldns/pkthdr.h \ + $(srcdir)/sldns/rrdef.h $(srcdir)/util/data/dname.h $(srcdir)/util/data/packed_rrset.h \ + $(srcdir)/util/storage/lookup3.h $(srcdir)/util/regional.h $(srcdir)/sldns/sbuffer.h $(srcdir)/sldns/parseutil.h \ + $(srcdir)/sldns/wire2str.h msgreply.lo msgreply.o: $(srcdir)/util/data/msgreply.c config.h $(srcdir)/util/data/msgreply.h \ $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/util/data/packed_rrset.h \ $(srcdir)/util/storage/lookup3.h $(srcdir)/util/alloc.h $(srcdir)/util/netevent.h $(srcdir)/util/net_help.h \ - $(srcdir)/util/data/dname.h $(srcdir)/util/regional.h $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h \ - $(srcdir)/ldns/rrdef.h $(srcdir)/util/data/msgencode.h $(srcdir)/ldns/sbuffer.h $(srcdir)/ldns/wire2str.h + $(srcdir)/util/data/dname.h $(srcdir)/util/regional.h $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h \ + $(srcdir)/sldns/rrdef.h $(srcdir)/util/data/msgencode.h $(srcdir)/sldns/sbuffer.h $(srcdir)/sldns/wire2str.h packed_rrset.lo packed_rrset.o: $(srcdir)/util/data/packed_rrset.c config.h \ $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h \ $(srcdir)/util/data/dname.h $(srcdir)/util/storage/lookup3.h $(srcdir)/util/alloc.h $(srcdir)/util/regional.h \ - $(srcdir)/util/net_help.h $(srcdir)/ldns/rrdef.h $(srcdir)/ldns/sbuffer.h $(srcdir)/ldns/wire2str.h + $(srcdir)/util/net_help.h $(srcdir)/sldns/rrdef.h $(srcdir)/sldns/sbuffer.h $(srcdir)/sldns/wire2str.h iterator.lo iterator.o: $(srcdir)/iterator/iterator.c config.h $(srcdir)/iterator/iterator.h \ $(srcdir)/services/outbound_list.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/storage/lruhash.h \ $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/module.h \ - $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h $(srcdir)/iterator/iter_utils.h \ + $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/iterator/iter_utils.h \ $(srcdir)/iterator/iter_resptype.h $(srcdir)/iterator/iter_hints.h $(srcdir)/util/storage/dnstree.h \ $(srcdir)/util/rbtree.h $(srcdir)/iterator/iter_fwd.h $(srcdir)/iterator/iter_donotq.h \ $(srcdir)/iterator/iter_delegpt.h $(srcdir)/iterator/iter_scrub.h $(srcdir)/iterator/iter_priv.h \ $(srcdir)/validator/val_neg.h $(srcdir)/services/cache/dns.h $(srcdir)/services/cache/infra.h \ $(srcdir)/util/rtt.h $(srcdir)/util/netevent.h $(srcdir)/util/net_help.h $(srcdir)/util/regional.h \ $(srcdir)/util/data/dname.h $(srcdir)/util/data/msgencode.h $(srcdir)/util/fptr_wlist.h $(srcdir)/util/tube.h \ - $(srcdir)/services/mesh.h $(srcdir)/services/modstack.h $(srcdir)/util/config_file.h $(srcdir)/ldns/wire2str.h \ - $(srcdir)/ldns/parseutil.h $(srcdir)/ldns/sbuffer.h + $(srcdir)/services/mesh.h $(srcdir)/services/modstack.h $(srcdir)/util/config_file.h $(srcdir)/sldns/wire2str.h \ + $(srcdir)/sldns/parseutil.h $(srcdir)/sldns/sbuffer.h iter_delegpt.lo iter_delegpt.o: $(srcdir)/iterator/iter_delegpt.c config.h $(srcdir)/iterator/iter_delegpt.h \ $(srcdir)/util/log.h $(srcdir)/services/cache/dns.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h \ $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/regional.h \ - $(srcdir)/util/data/dname.h $(srcdir)/util/net_help.h $(srcdir)/ldns/rrdef.h $(srcdir)/ldns/sbuffer.h + $(srcdir)/util/data/dname.h $(srcdir)/util/net_help.h $(srcdir)/sldns/rrdef.h $(srcdir)/sldns/sbuffer.h iter_donotq.lo iter_donotq.o: $(srcdir)/iterator/iter_donotq.c config.h $(srcdir)/iterator/iter_donotq.h \ $(srcdir)/util/storage/dnstree.h $(srcdir)/util/rbtree.h $(srcdir)/util/regional.h $(srcdir)/util/log.h \ $(srcdir)/util/config_file.h $(srcdir)/util/net_help.h iter_fwd.lo iter_fwd.o: $(srcdir)/iterator/iter_fwd.c config.h $(srcdir)/iterator/iter_fwd.h \ $(srcdir)/util/rbtree.h $(srcdir)/iterator/iter_delegpt.h $(srcdir)/util/log.h $(srcdir)/util/config_file.h \ $(srcdir)/util/net_help.h $(srcdir)/util/data/dname.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h \ - $(srcdir)/ldns/rrdef.h $(srcdir)/ldns/str2wire.h + $(srcdir)/sldns/rrdef.h $(srcdir)/sldns/str2wire.h iter_hints.lo iter_hints.o: $(srcdir)/iterator/iter_hints.c config.h $(srcdir)/iterator/iter_hints.h \ $(srcdir)/util/storage/dnstree.h $(srcdir)/util/rbtree.h $(srcdir)/iterator/iter_delegpt.h $(srcdir)/util/log.h \ $(srcdir)/util/config_file.h $(srcdir)/util/net_help.h $(srcdir)/util/data/dname.h \ - $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/ldns/rrdef.h $(srcdir)/ldns/str2wire.h \ - $(srcdir)/ldns/wire2str.h + $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/sldns/rrdef.h $(srcdir)/sldns/str2wire.h \ + $(srcdir)/sldns/wire2str.h iter_priv.lo iter_priv.o: $(srcdir)/iterator/iter_priv.c config.h $(srcdir)/iterator/iter_priv.h \ $(srcdir)/util/rbtree.h $(srcdir)/util/regional.h $(srcdir)/util/log.h $(srcdir)/util/config_file.h \ $(srcdir)/util/data/dname.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h \ - $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h $(srcdir)/util/net_help.h \ - $(srcdir)/util/storage/dnstree.h $(srcdir)/ldns/str2wire.h $(srcdir)/ldns/sbuffer.h + $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/util/net_help.h \ + $(srcdir)/util/storage/dnstree.h $(srcdir)/sldns/str2wire.h $(srcdir)/sldns/sbuffer.h iter_resptype.lo iter_resptype.o: $(srcdir)/iterator/iter_resptype.c config.h \ $(srcdir)/iterator/iter_resptype.h $(srcdir)/iterator/iter_delegpt.h $(srcdir)/util/log.h \ $(srcdir)/services/cache/dns.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h \ $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/net_help.h \ - $(srcdir)/util/data/dname.h $(srcdir)/ldns/rrdef.h $(srcdir)/ldns/pkthdr.h + $(srcdir)/util/data/dname.h $(srcdir)/sldns/rrdef.h $(srcdir)/sldns/pkthdr.h iter_scrub.lo iter_scrub.o: $(srcdir)/iterator/iter_scrub.c config.h $(srcdir)/iterator/iter_scrub.h \ $(srcdir)/iterator/iterator.h $(srcdir)/services/outbound_list.h $(srcdir)/util/data/msgreply.h \ $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/util/data/packed_rrset.h \ - $(srcdir)/util/module.h $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h \ + $(srcdir)/util/module.h $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h \ $(srcdir)/iterator/iter_priv.h $(srcdir)/util/rbtree.h $(srcdir)/services/cache/rrset.h \ $(srcdir)/util/storage/slabhash.h $(srcdir)/util/net_help.h $(srcdir)/util/regional.h \ - $(srcdir)/util/config_file.h $(srcdir)/util/data/dname.h $(srcdir)/util/alloc.h $(srcdir)/ldns/sbuffer.h + $(srcdir)/util/config_file.h $(srcdir)/util/data/dname.h $(srcdir)/util/alloc.h $(srcdir)/sldns/sbuffer.h iter_utils.lo iter_utils.o: $(srcdir)/iterator/iter_utils.c config.h $(srcdir)/iterator/iter_utils.h \ $(srcdir)/iterator/iter_resptype.h $(srcdir)/iterator/iterator.h $(srcdir)/services/outbound_list.h \ $(srcdir)/util/data/msgreply.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h \ $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/module.h $(srcdir)/util/data/msgparse.h \ - $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h $(srcdir)/iterator/iter_hints.h $(srcdir)/util/storage/dnstree.h \ + $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/iterator/iter_hints.h $(srcdir)/util/storage/dnstree.h \ $(srcdir)/util/rbtree.h $(srcdir)/iterator/iter_fwd.h $(srcdir)/iterator/iter_donotq.h \ $(srcdir)/iterator/iter_delegpt.h $(srcdir)/iterator/iter_priv.h $(srcdir)/services/cache/infra.h \ $(srcdir)/util/rtt.h $(srcdir)/services/cache/dns.h $(srcdir)/services/cache/rrset.h \ @@ -686,54 +687,56 @@ iter_utils.lo iter_utils.o: $(srcdir)/iterator/iter_utils.c config.h $(srcdir)/i $(srcdir)/util/regional.h $(srcdir)/util/data/dname.h $(srcdir)/util/random.h $(srcdir)/util/fptr_wlist.h \ $(srcdir)/util/netevent.h $(srcdir)/util/tube.h $(srcdir)/services/mesh.h $(srcdir)/services/modstack.h \ $(srcdir)/validator/val_anchor.h $(srcdir)/validator/val_kcache.h $(srcdir)/validator/val_kentry.h \ - $(srcdir)/validator/val_utils.h $(srcdir)/validator/val_sigcrypt.h $(srcdir)/ldns/sbuffer.h + $(srcdir)/validator/val_utils.h $(srcdir)/validator/val_sigcrypt.h $(srcdir)/sldns/sbuffer.h listen_dnsport.lo listen_dnsport.o: $(srcdir)/services/listen_dnsport.c config.h \ $(srcdir)/services/listen_dnsport.h $(srcdir)/util/netevent.h $(srcdir)/services/outside_network.h \ - $(srcdir)/util/rbtree.h $(srcdir)/util/log.h $(srcdir)/util/config_file.h $(srcdir)/util/net_help.h \ - $(srcdir)/ldns/sbuffer.h + $(srcdir)/util/rbtree.h $(srcdir)/util/log.h $(srcdir)/util/config_file.h \ + $(srcdir)/util/net_help.h $(srcdir)/sldns/sbuffer.h localzone.lo localzone.o: $(srcdir)/services/localzone.c config.h $(srcdir)/services/localzone.h \ - $(srcdir)/util/rbtree.h $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/ldns/str2wire.h $(srcdir)/ldns/rrdef.h \ - $(srcdir)/ldns/sbuffer.h $(srcdir)/util/regional.h $(srcdir)/util/config_file.h $(srcdir)/util/data/dname.h \ + $(srcdir)/util/rbtree.h $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/sldns/str2wire.h $(srcdir)/sldns/rrdef.h \ + $(srcdir)/sldns/sbuffer.h $(srcdir)/util/regional.h $(srcdir)/util/config_file.h $(srcdir)/util/data/dname.h \ $(srcdir)/util/storage/lruhash.h $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/data/msgencode.h \ - $(srcdir)/util/net_help.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h + $(srcdir)/util/net_help.h $(srcdir)/util/netevent.h $(srcdir)/util/data/msgreply.h \ + $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h mesh.lo mesh.o: $(srcdir)/services/mesh.c config.h $(srcdir)/services/mesh.h $(srcdir)/util/rbtree.h \ $(srcdir)/util/netevent.h $(srcdir)/util/data/msgparse.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h \ - $(srcdir)/util/log.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h $(srcdir)/util/module.h $(srcdir)/util/data/msgreply.h \ + $(srcdir)/util/log.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/util/module.h $(srcdir)/util/data/msgreply.h \ $(srcdir)/util/data/packed_rrset.h $(srcdir)/services/modstack.h $(srcdir)/services/outbound_list.h \ $(srcdir)/services/cache/dns.h $(srcdir)/util/net_help.h $(srcdir)/util/regional.h \ $(srcdir)/util/data/msgencode.h $(srcdir)/util/timehist.h $(srcdir)/util/fptr_wlist.h $(srcdir)/util/tube.h \ - $(srcdir)/util/alloc.h $(srcdir)/util/config_file.h $(srcdir)/ldns/sbuffer.h + $(srcdir)/util/alloc.h $(srcdir)/util/config_file.h $(srcdir)/sldns/sbuffer.h modstack.lo modstack.o: $(srcdir)/services/modstack.c config.h $(srcdir)/services/modstack.h \ $(srcdir)/util/module.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h \ $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/data/msgparse.h \ - $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h $(srcdir)/util/fptr_wlist.h $(srcdir)/util/netevent.h $(srcdir)/util/tube.h \ + $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/util/fptr_wlist.h $(srcdir)/util/netevent.h $(srcdir)/util/tube.h \ $(srcdir)/services/mesh.h $(srcdir)/util/rbtree.h $(srcdir)/dns64/dns64.h $(srcdir)/iterator/iterator.h \ $(srcdir)/services/outbound_list.h $(srcdir)/validator/validator.h $(srcdir)/validator/val_utils.h outbound_list.lo outbound_list.o: $(srcdir)/services/outbound_list.c config.h \ $(srcdir)/services/outbound_list.h $(srcdir)/services/outside_network.h $(srcdir)/util/rbtree.h \ - $(srcdir)/util/netevent.h + $(srcdir)/util/netevent.h outside_network.lo outside_network.o: $(srcdir)/services/outside_network.c config.h \ $(srcdir)/services/outside_network.h $(srcdir)/util/rbtree.h $(srcdir)/util/netevent.h \ - $(srcdir)/services/listen_dnsport.h $(srcdir)/services/cache/infra.h $(srcdir)/util/storage/lruhash.h \ - $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/util/rtt.h $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h \ - $(srcdir)/ldns/rrdef.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/packed_rrset.h \ - $(srcdir)/util/data/msgencode.h $(srcdir)/util/data/dname.h $(srcdir)/util/net_help.h $(srcdir)/util/random.h \ - $(srcdir)/util/fptr_wlist.h $(srcdir)/util/module.h $(srcdir)/util/tube.h $(srcdir)/services/mesh.h \ - $(srcdir)/services/modstack.h $(srcdir)/ldns/sbuffer.h \ + $(srcdir)/services/listen_dnsport.h $(srcdir)/services/cache/infra.h \ + $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/util/rtt.h \ + $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/util/data/msgreply.h \ + $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/data/msgencode.h $(srcdir)/util/data/dname.h \ + $(srcdir)/util/net_help.h $(srcdir)/util/random.h $(srcdir)/util/fptr_wlist.h $(srcdir)/util/module.h \ + $(srcdir)/util/tube.h $(srcdir)/services/mesh.h $(srcdir)/services/modstack.h $(srcdir)/sldns/sbuffer.h \ + $(srcdir)/dnstap/dnstap.h \ alloc.lo alloc.o: $(srcdir)/util/alloc.c config.h $(srcdir)/util/alloc.h $(srcdir)/util/locks.h $(srcdir)/util/log.h \ $(srcdir)/util/regional.h $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/storage/lruhash.h \ $(srcdir)/util/fptr_wlist.h $(srcdir)/util/netevent.h $(srcdir)/util/module.h $(srcdir)/util/data/msgreply.h \ - $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h $(srcdir)/util/tube.h \ + $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/util/tube.h \ $(srcdir)/services/mesh.h $(srcdir)/util/rbtree.h $(srcdir)/services/modstack.h config_file.lo config_file.o: $(srcdir)/util/config_file.c config.h $(srcdir)/util/log.h \ $(srcdir)/util/configyyrename.h $(srcdir)/util/config_file.h util/configparser.h \ $(srcdir)/util/net_help.h $(srcdir)/util/data/msgparse.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h \ - $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h $(srcdir)/util/module.h $(srcdir)/util/data/msgreply.h \ + $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/util/module.h $(srcdir)/util/data/msgreply.h \ $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/regional.h $(srcdir)/util/fptr_wlist.h \ $(srcdir)/util/netevent.h $(srcdir)/util/tube.h $(srcdir)/services/mesh.h $(srcdir)/util/rbtree.h \ - $(srcdir)/services/modstack.h $(srcdir)/util/data/dname.h $(srcdir)/ldns/wire2str.h $(srcdir)/ldns/parseutil.h \ - $(srcdir)/util/iana_ports.inc + $(srcdir)/services/modstack.h $(srcdir)/util/data/dname.h $(srcdir)/util/rtt.h $(srcdir)/sldns/wire2str.h \ + $(srcdir)/sldns/parseutil.h $(srcdir)/util/iana_ports.inc configlexer.lo configlexer.o: util/configlexer.c config.h $(srcdir)/util/configyyrename.h \ $(srcdir)/util/config_file.h util/configparser.h configparser.lo configparser.o: util/configparser.c config.h $(srcdir)/util/configyyrename.h \ @@ -741,44 +744,46 @@ configparser.lo configparser.o: util/configparser.c config.h $(srcdir)/util/conf fptr_wlist.lo fptr_wlist.o: $(srcdir)/util/fptr_wlist.c config.h $(srcdir)/util/fptr_wlist.h \ $(srcdir)/util/netevent.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h \ $(srcdir)/util/module.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/packed_rrset.h \ - $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h $(srcdir)/util/tube.h \ + $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/util/tube.h \ $(srcdir)/services/mesh.h $(srcdir)/util/rbtree.h $(srcdir)/services/modstack.h $(srcdir)/util/mini_event.h \ - $(srcdir)/util/rbtree.h $(srcdir)/services/outside_network.h $(srcdir)/services/localzone.h \ - $(srcdir)/services/cache/infra.h $(srcdir)/util/rtt.h $(srcdir)/services/cache/rrset.h \ - $(srcdir)/util/storage/slabhash.h $(srcdir)/dns64/dns64.h $(srcdir)/iterator/iterator.h \ - $(srcdir)/services/outbound_list.h $(srcdir)/iterator/iter_fwd.h $(srcdir)/validator/validator.h \ - $(srcdir)/validator/val_utils.h $(srcdir)/validator/val_anchor.h $(srcdir)/validator/val_nsec3.h \ - $(srcdir)/validator/val_sigcrypt.h $(srcdir)/validator/val_kentry.h $(srcdir)/validator/val_neg.h \ - $(srcdir)/validator/autotrust.h $(srcdir)/util/storage/dnstree.h $(srcdir)/libunbound/libworker.h \ - $(srcdir)/libunbound/context.h $(srcdir)/util/alloc.h $(srcdir)/libunbound/unbound.h \ - $(srcdir)/libunbound/worker.h $(srcdir)/ldns/sbuffer.h $(srcdir)/util/config_file.h + $(srcdir)/util/rbtree.h $(srcdir)/services/outside_network.h \ + $(srcdir)/services/localzone.h $(srcdir)/services/cache/infra.h $(srcdir)/util/rtt.h \ + $(srcdir)/services/cache/rrset.h $(srcdir)/util/storage/slabhash.h $(srcdir)/dns64/dns64.h \ + $(srcdir)/iterator/iterator.h $(srcdir)/services/outbound_list.h $(srcdir)/iterator/iter_fwd.h \ + $(srcdir)/validator/validator.h $(srcdir)/validator/val_utils.h $(srcdir)/validator/val_anchor.h \ + $(srcdir)/validator/val_nsec3.h $(srcdir)/validator/val_sigcrypt.h $(srcdir)/validator/val_kentry.h \ + $(srcdir)/validator/val_neg.h $(srcdir)/validator/autotrust.h $(srcdir)/util/storage/dnstree.h \ + $(srcdir)/libunbound/libworker.h $(srcdir)/libunbound/context.h $(srcdir)/util/alloc.h \ + $(srcdir)/libunbound/unbound.h $(srcdir)/libunbound/worker.h $(srcdir)/sldns/sbuffer.h \ + $(srcdir)/util/config_file.h locks.lo locks.o: $(srcdir)/util/locks.c config.h $(srcdir)/util/locks.h $(srcdir)/util/log.h -log.lo log.o: $(srcdir)/util/log.c config.h $(srcdir)/util/log.h $(srcdir)/util/locks.h $(srcdir)/ldns/sbuffer.h +log.lo log.o: $(srcdir)/util/log.c config.h $(srcdir)/util/log.h $(srcdir)/util/locks.h $(srcdir)/sldns/sbuffer.h mini_event.lo mini_event.o: $(srcdir)/util/mini_event.c config.h $(srcdir)/util/mini_event.h $(srcdir)/util/rbtree.h \ $(srcdir)/util/fptr_wlist.h $(srcdir)/util/netevent.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h \ $(srcdir)/util/log.h $(srcdir)/util/module.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/packed_rrset.h \ - $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h $(srcdir)/util/tube.h \ + $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/util/tube.h \ $(srcdir)/services/mesh.h $(srcdir)/util/rbtree.h $(srcdir)/services/modstack.h module.lo module.o: $(srcdir)/util/module.c config.h $(srcdir)/util/module.h $(srcdir)/util/storage/lruhash.h \ $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/packed_rrset.h \ - $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h + $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h netevent.lo netevent.o: $(srcdir)/util/netevent.c config.h $(srcdir)/util/netevent.h $(srcdir)/util/log.h \ $(srcdir)/util/net_help.h $(srcdir)/util/fptr_wlist.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h \ $(srcdir)/util/module.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/packed_rrset.h \ - $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h $(srcdir)/util/tube.h \ - $(srcdir)/services/mesh.h $(srcdir)/util/rbtree.h $(srcdir)/services/modstack.h $(srcdir)/ldns/sbuffer.h \ + $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/util/tube.h \ + $(srcdir)/services/mesh.h $(srcdir)/util/rbtree.h $(srcdir)/services/modstack.h $(srcdir)/sldns/sbuffer.h \ + $(srcdir)/dnstap/dnstap.h \ $(srcdir)/util/mini_event.h $(srcdir)/util/rbtree.h net_help.lo net_help.o: $(srcdir)/util/net_help.c config.h $(srcdir)/util/net_help.h $(srcdir)/util/log.h \ $(srcdir)/util/data/dname.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/module.h \ $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/data/msgparse.h \ - $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h $(srcdir)/util/regional.h $(srcdir)/ldns/parseutil.h \ - $(srcdir)/ldns/wire2str.h \ + $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/util/regional.h $(srcdir)/sldns/parseutil.h \ + $(srcdir)/sldns/wire2str.h \ random.lo random.o: $(srcdir)/util/random.c config.h $(srcdir)/util/random.h $(srcdir)/util/log.h rbtree.lo rbtree.o: $(srcdir)/util/rbtree.c config.h $(srcdir)/util/log.h $(srcdir)/util/fptr_wlist.h \ $(srcdir)/util/netevent.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h \ $(srcdir)/util/module.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/packed_rrset.h \ - $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h $(srcdir)/util/tube.h \ + $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/util/tube.h \ $(srcdir)/services/mesh.h $(srcdir)/util/rbtree.h $(srcdir)/services/modstack.h regional.lo regional.o: $(srcdir)/util/regional.c config.h $(srcdir)/util/log.h $(srcdir)/util/regional.h rtt.lo rtt.o: $(srcdir)/util/rtt.c config.h $(srcdir)/util/rtt.h @@ -789,7 +794,7 @@ lookup3.lo lookup3.o: $(srcdir)/util/storage/lookup3.c config.h $(srcdir)/util/s lruhash.lo lruhash.o: $(srcdir)/util/storage/lruhash.c config.h $(srcdir)/util/storage/lruhash.h \ $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/util/fptr_wlist.h $(srcdir)/util/netevent.h $(srcdir)/util/module.h \ $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/data/msgparse.h \ - $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h $(srcdir)/util/tube.h $(srcdir)/services/mesh.h $(srcdir)/util/rbtree.h \ + $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/util/tube.h $(srcdir)/services/mesh.h $(srcdir)/util/rbtree.h \ $(srcdir)/services/modstack.h slabhash.lo slabhash.o: $(srcdir)/util/storage/slabhash.c config.h $(srcdir)/util/storage/slabhash.h \ $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h @@ -797,43 +802,43 @@ timehist.lo timehist.o: $(srcdir)/util/timehist.c config.h $(srcdir)/util/timehi tube.lo tube.o: $(srcdir)/util/tube.c config.h $(srcdir)/util/tube.h $(srcdir)/util/log.h $(srcdir)/util/net_help.h \ $(srcdir)/util/netevent.h $(srcdir)/util/fptr_wlist.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h \ $(srcdir)/util/module.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/packed_rrset.h \ - $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h $(srcdir)/services/mesh.h \ + $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/services/mesh.h \ $(srcdir)/util/rbtree.h $(srcdir)/services/modstack.h winsock_event.lo winsock_event.o: $(srcdir)/util/winsock_event.c config.h autotrust.lo autotrust.o: $(srcdir)/validator/autotrust.c config.h $(srcdir)/validator/autotrust.h \ $(srcdir)/util/rbtree.h $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/storage/lruhash.h \ $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/validator/val_anchor.h $(srcdir)/validator/val_utils.h \ $(srcdir)/validator/val_sigcrypt.h $(srcdir)/util/data/dname.h $(srcdir)/util/module.h \ - $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h \ + $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h \ $(srcdir)/util/net_help.h $(srcdir)/util/config_file.h $(srcdir)/util/regional.h $(srcdir)/util/random.h \ $(srcdir)/services/mesh.h $(srcdir)/util/netevent.h $(srcdir)/services/modstack.h \ $(srcdir)/services/cache/rrset.h $(srcdir)/util/storage/slabhash.h $(srcdir)/validator/val_kcache.h \ - $(srcdir)/ldns/sbuffer.h $(srcdir)/ldns/wire2str.h $(srcdir)/ldns/str2wire.h $(srcdir)/ldns/keyraw.h \ + $(srcdir)/sldns/sbuffer.h $(srcdir)/sldns/wire2str.h $(srcdir)/sldns/str2wire.h $(srcdir)/sldns/keyraw.h \ val_anchor.lo val_anchor.o: $(srcdir)/validator/val_anchor.c config.h $(srcdir)/validator/val_anchor.h \ $(srcdir)/util/rbtree.h $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/validator/val_sigcrypt.h \ $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/storage/lruhash.h $(srcdir)/validator/autotrust.h \ - $(srcdir)/util/data/dname.h $(srcdir)/util/net_help.h $(srcdir)/util/config_file.h $(srcdir)/ldns/sbuffer.h \ - $(srcdir)/ldns/rrdef.h $(srcdir)/ldns/str2wire.h + $(srcdir)/util/data/dname.h $(srcdir)/util/net_help.h $(srcdir)/util/config_file.h $(srcdir)/sldns/sbuffer.h \ + $(srcdir)/sldns/rrdef.h $(srcdir)/sldns/str2wire.h validator.lo validator.o: $(srcdir)/validator/validator.c config.h $(srcdir)/validator/validator.h \ $(srcdir)/util/module.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h \ $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/data/msgparse.h \ - $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h $(srcdir)/validator/val_utils.h $(srcdir)/validator/val_anchor.h \ + $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/validator/val_utils.h $(srcdir)/validator/val_anchor.h \ $(srcdir)/util/rbtree.h $(srcdir)/validator/val_kcache.h $(srcdir)/util/storage/slabhash.h \ $(srcdir)/validator/val_kentry.h $(srcdir)/validator/val_nsec.h $(srcdir)/validator/val_nsec3.h \ $(srcdir)/validator/val_neg.h $(srcdir)/validator/val_sigcrypt.h $(srcdir)/validator/autotrust.h \ $(srcdir)/services/cache/dns.h $(srcdir)/util/data/dname.h $(srcdir)/util/net_help.h $(srcdir)/util/regional.h \ $(srcdir)/util/config_file.h $(srcdir)/util/fptr_wlist.h $(srcdir)/util/netevent.h $(srcdir)/util/tube.h \ - $(srcdir)/services/mesh.h $(srcdir)/services/modstack.h $(srcdir)/ldns/wire2str.h + $(srcdir)/services/mesh.h $(srcdir)/services/modstack.h $(srcdir)/sldns/wire2str.h val_kcache.lo val_kcache.o: $(srcdir)/validator/val_kcache.c config.h $(srcdir)/validator/val_kcache.h \ $(srcdir)/util/storage/slabhash.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h \ $(srcdir)/validator/val_kentry.h $(srcdir)/util/config_file.h $(srcdir)/util/data/dname.h \ $(srcdir)/util/module.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/packed_rrset.h \ - $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h + $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h val_kentry.lo val_kentry.o: $(srcdir)/validator/val_kentry.c config.h $(srcdir)/validator/val_kentry.h \ $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/util/data/packed_rrset.h \ $(srcdir)/util/data/dname.h $(srcdir)/util/storage/lookup3.h $(srcdir)/util/regional.h $(srcdir)/util/net_help.h \ - $(srcdir)/ldns/rrdef.h $(srcdir)/ldns/keyraw.h \ + $(srcdir)/sldns/rrdef.h $(srcdir)/sldns/keyraw.h \ val_neg.lo val_neg.o: $(srcdir)/validator/val_neg.c config.h \ $(srcdir)/validator/val_neg.h $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/util/rbtree.h \ @@ -841,72 +846,78 @@ val_neg.lo val_neg.o: $(srcdir)/validator/val_neg.c config.h \ $(srcdir)/validator/val_nsec3.h $(srcdir)/validator/val_utils.h $(srcdir)/util/data/dname.h \ $(srcdir)/util/data/msgreply.h $(srcdir)/util/net_help.h $(srcdir)/util/config_file.h \ $(srcdir)/services/cache/rrset.h $(srcdir)/util/storage/slabhash.h $(srcdir)/services/cache/dns.h \ - $(srcdir)/ldns/rrdef.h $(srcdir)/ldns/sbuffer.h + $(srcdir)/sldns/rrdef.h $(srcdir)/sldns/sbuffer.h val_nsec3.lo val_nsec3.o: $(srcdir)/validator/val_nsec3.c config.h \ $(srcdir)/validator/val_nsec3.h $(srcdir)/util/rbtree.h $(srcdir)/util/data/packed_rrset.h \ $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/validator/validator.h \ - $(srcdir)/util/module.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h \ - $(srcdir)/ldns/rrdef.h $(srcdir)/validator/val_utils.h $(srcdir)/validator/val_kentry.h \ + $(srcdir)/util/module.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h \ + $(srcdir)/sldns/rrdef.h $(srcdir)/validator/val_utils.h $(srcdir)/validator/val_kentry.h \ $(srcdir)/services/cache/rrset.h $(srcdir)/util/storage/slabhash.h $(srcdir)/util/regional.h \ - $(srcdir)/util/net_help.h $(srcdir)/util/data/dname.h $(srcdir)/validator/val_nsec.h $(srcdir)/ldns/sbuffer.h + $(srcdir)/util/net_help.h $(srcdir)/util/data/dname.h $(srcdir)/validator/val_nsec.h $(srcdir)/sldns/sbuffer.h val_nsec.lo val_nsec.o: $(srcdir)/validator/val_nsec.c config.h $(srcdir)/validator/val_nsec.h \ $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h \ $(srcdir)/validator/val_utils.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/dname.h \ - $(srcdir)/util/net_help.h $(srcdir)/util/module.h $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h \ - $(srcdir)/ldns/rrdef.h $(srcdir)/services/cache/rrset.h $(srcdir)/util/storage/slabhash.h -val_secalgo.lo val_secalgo.o: $(srcdir)/validator/val_secalgo.c config.h $(srcdir)/validator/val_secalgo.h \ - $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h \ - $(srcdir)/ldns/rrdef.h $(srcdir)/ldns/keyraw.h \ - $(srcdir)/ldns/sbuffer.h \ + $(srcdir)/util/net_help.h $(srcdir)/util/module.h $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h \ + $(srcdir)/sldns/rrdef.h $(srcdir)/services/cache/rrset.h $(srcdir)/util/storage/slabhash.h +val_secalgo.lo val_secalgo.o: $(srcdir)/validator/val_secalgo.c config.h $(srcdir)/util/data/packed_rrset.h \ + $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/validator/val_secalgo.h \ + $(srcdir)/sldns/rrdef.h $(srcdir)/sldns/keyraw.h \ + $(srcdir)/sldns/sbuffer.h \ val_sigcrypt.lo val_sigcrypt.o: $(srcdir)/validator/val_sigcrypt.c config.h \ $(srcdir)/validator/val_sigcrypt.h $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/storage/lruhash.h \ $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/validator/val_secalgo.h $(srcdir)/validator/validator.h \ - $(srcdir)/util/module.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h \ - $(srcdir)/ldns/rrdef.h $(srcdir)/validator/val_utils.h $(srcdir)/util/data/dname.h $(srcdir)/util/rbtree.h \ - $(srcdir)/util/net_help.h $(srcdir)/util/regional.h $(srcdir)/ldns/keyraw.h \ - $(srcdir)/ldns/sbuffer.h $(srcdir)/ldns/parseutil.h $(srcdir)/ldns/wire2str.h \ + $(srcdir)/util/module.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h \ + $(srcdir)/sldns/rrdef.h $(srcdir)/validator/val_utils.h $(srcdir)/util/data/dname.h $(srcdir)/util/rbtree.h \ + $(srcdir)/util/net_help.h $(srcdir)/util/regional.h $(srcdir)/sldns/keyraw.h \ + $(srcdir)/sldns/sbuffer.h $(srcdir)/sldns/parseutil.h $(srcdir)/sldns/wire2str.h \ val_utils.lo val_utils.o: $(srcdir)/validator/val_utils.c config.h $(srcdir)/validator/val_utils.h \ $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h \ $(srcdir)/validator/validator.h $(srcdir)/util/module.h $(srcdir)/util/data/msgreply.h \ - $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h $(srcdir)/validator/val_kentry.h \ + $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/validator/val_kentry.h \ $(srcdir)/validator/val_sigcrypt.h $(srcdir)/validator/val_anchor.h $(srcdir)/util/rbtree.h \ $(srcdir)/validator/val_nsec.h $(srcdir)/validator/val_neg.h $(srcdir)/services/cache/rrset.h \ $(srcdir)/util/storage/slabhash.h $(srcdir)/services/cache/dns.h $(srcdir)/util/data/dname.h \ $(srcdir)/util/net_help.h $(srcdir)/util/regional.h dns64.lo dns64.o: $(srcdir)/dns64/dns64.c config.h $(srcdir)/dns64/dns64.h $(srcdir)/util/module.h \ $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/util/data/msgreply.h \ - $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h \ + $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h \ $(srcdir)/services/cache/dns.h $(srcdir)/services/cache/rrset.h $(srcdir)/util/storage/slabhash.h \ $(srcdir)/util/config_file.h $(srcdir)/util/fptr_wlist.h $(srcdir)/util/netevent.h $(srcdir)/util/tube.h \ $(srcdir)/services/mesh.h $(srcdir)/util/rbtree.h $(srcdir)/services/modstack.h $(srcdir)/util/net_help.h \ $(srcdir)/util/regional.h checklocks.lo checklocks.o: $(srcdir)/testcode/checklocks.c config.h $(srcdir)/util/locks.h $(srcdir)/util/log.h \ $(srcdir)/testcode/checklocks.h +dnstap.lo dnstap.o: $(srcdir)/dnstap/dnstap.c config.h $(srcdir)/sldns/sbuffer.h \ + $(srcdir)/util/config_file.h $(srcdir)/util/net_help.h $(srcdir)/util/log.h $(srcdir)/util/netevent.h \ + $(srcdir)/dnstap/dnstap.h \ + $(srcdir)/dnstap/dnstap.pb-c.h +dnstap.pb-c.lo dnstap.pb-c.o: $(srcdir)/dnstap/dnstap.pb-c.c $(srcdir)/dnstap/dnstap.pb-c.h unitanchor.lo unitanchor.o: $(srcdir)/testcode/unitanchor.c config.h $(srcdir)/util/log.h $(srcdir)/util/data/dname.h \ $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/testcode/unitmain.h \ - $(srcdir)/validator/val_anchor.h $(srcdir)/util/rbtree.h $(srcdir)/ldns/sbuffer.h $(srcdir)/ldns/rrdef.h + $(srcdir)/validator/val_anchor.h $(srcdir)/util/rbtree.h $(srcdir)/sldns/sbuffer.h $(srcdir)/sldns/rrdef.h unitdname.lo unitdname.o: $(srcdir)/testcode/unitdname.c config.h $(srcdir)/util/log.h $(srcdir)/testcode/unitmain.h \ - $(srcdir)/util/data/dname.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/ldns/sbuffer.h \ - $(srcdir)/ldns/str2wire.h $(srcdir)/ldns/rrdef.h + $(srcdir)/util/data/dname.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/sldns/sbuffer.h \ + $(srcdir)/sldns/str2wire.h $(srcdir)/sldns/rrdef.h unitlruhash.lo unitlruhash.o: $(srcdir)/testcode/unitlruhash.c config.h $(srcdir)/testcode/unitmain.h \ $(srcdir)/util/log.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/storage/slabhash.h unitmain.lo unitmain.o: $(srcdir)/testcode/unitmain.c config.h \ - $(srcdir)/ldns/rrdef.h $(srcdir)/ldns/keyraw.h \ - $(srcdir)/util/log.h $(srcdir)/testcode/unitmain.h $(srcdir)/util/alloc.h $(srcdir)/util/locks.h $(srcdir)/util/net_help.h \ + $(srcdir)/sldns/rrdef.h $(srcdir)/sldns/keyraw.h \ + $(srcdir)/util/log.h \ + $(srcdir)/testcode/unitmain.h $(srcdir)/util/alloc.h $(srcdir)/util/locks.h $(srcdir)/util/net_help.h \ $(srcdir)/util/config_file.h $(srcdir)/util/rtt.h $(srcdir)/services/cache/infra.h \ $(srcdir)/util/storage/lruhash.h $(srcdir)/util/random.h unitmsgparse.lo unitmsgparse.o: $(srcdir)/testcode/unitmsgparse.c config.h $(srcdir)/util/log.h \ $(srcdir)/testcode/unitmain.h $(srcdir)/util/data/msgparse.h $(srcdir)/util/storage/lruhash.h \ - $(srcdir)/util/locks.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h $(srcdir)/util/data/msgreply.h \ + $(srcdir)/util/locks.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/util/data/msgreply.h \ $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/data/msgencode.h $(srcdir)/util/data/dname.h \ $(srcdir)/util/alloc.h $(srcdir)/util/regional.h $(srcdir)/util/net_help.h $(srcdir)/testcode/readhex.h \ - $(srcdir)/testcode/testpkts.h $(srcdir)/ldns/sbuffer.h $(srcdir)/ldns/str2wire.h $(srcdir)/ldns/wire2str.h + $(srcdir)/testcode/testpkts.h $(srcdir)/sldns/sbuffer.h $(srcdir)/sldns/str2wire.h $(srcdir)/sldns/wire2str.h unitneg.lo unitneg.o: $(srcdir)/testcode/unitneg.c config.h $(srcdir)/util/log.h $(srcdir)/util/net_help.h \ $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h \ $(srcdir)/util/data/dname.h $(srcdir)/testcode/unitmain.h $(srcdir)/validator/val_neg.h $(srcdir)/util/rbtree.h \ - $(srcdir)/ldns/rrdef.h + $(srcdir)/sldns/rrdef.h unitregional.lo unitregional.o: $(srcdir)/testcode/unitregional.c config.h $(srcdir)/testcode/unitmain.h \ $(srcdir)/util/log.h $(srcdir)/util/regional.h unitslabhash.lo unitslabhash.o: $(srcdir)/testcode/unitslabhash.c config.h $(srcdir)/testcode/unitmain.h \ @@ -916,84 +927,89 @@ unitverify.lo unitverify.o: $(srcdir)/testcode/unitverify.c config.h $(srcdir)/u $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/validator/val_secalgo.h \ $(srcdir)/validator/val_nsec.h $(srcdir)/validator/val_nsec3.h $(srcdir)/util/rbtree.h \ $(srcdir)/validator/validator.h $(srcdir)/util/module.h $(srcdir)/util/data/msgreply.h \ - $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h $(srcdir)/validator/val_utils.h \ + $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/validator/val_utils.h \ $(srcdir)/testcode/testpkts.h $(srcdir)/util/data/dname.h $(srcdir)/util/regional.h $(srcdir)/util/alloc.h \ - $(srcdir)/util/net_help.h $(srcdir)/util/config_file.h $(srcdir)/ldns/sbuffer.h $(srcdir)/ldns/keyraw.h \ - $(srcdir)/ldns/str2wire.h $(srcdir)/ldns/wire2str.h + $(srcdir)/util/net_help.h $(srcdir)/util/config_file.h $(srcdir)/sldns/sbuffer.h $(srcdir)/sldns/keyraw.h \ + $(srcdir)/sldns/str2wire.h $(srcdir)/sldns/wire2str.h readhex.lo readhex.o: $(srcdir)/testcode/readhex.c config.h $(srcdir)/testcode/readhex.h $(srcdir)/util/log.h \ - $(srcdir)/ldns/sbuffer.h $(srcdir)/ldns/parseutil.h + $(srcdir)/sldns/sbuffer.h $(srcdir)/sldns/parseutil.h testpkts.lo testpkts.o: $(srcdir)/testcode/testpkts.c config.h $(srcdir)/testcode/testpkts.h \ - $(srcdir)/util/net_help.h $(srcdir)/util/log.h $(srcdir)/ldns/sbuffer.h $(srcdir)/ldns/rrdef.h $(srcdir)/ldns/pkthdr.h \ - $(srcdir)/ldns/str2wire.h $(srcdir)/ldns/wire2str.h + $(srcdir)/util/net_help.h $(srcdir)/util/log.h $(srcdir)/sldns/sbuffer.h $(srcdir)/sldns/rrdef.h $(srcdir)/sldns/pkthdr.h \ + $(srcdir)/sldns/str2wire.h $(srcdir)/sldns/wire2str.h unitldns.lo unitldns.o: $(srcdir)/testcode/unitldns.c config.h $(srcdir)/util/log.h $(srcdir)/testcode/unitmain.h \ - $(srcdir)/ldns/sbuffer.h $(srcdir)/ldns/str2wire.h $(srcdir)/ldns/rrdef.h $(srcdir)/ldns/wire2str.h + $(srcdir)/sldns/sbuffer.h $(srcdir)/sldns/str2wire.h $(srcdir)/sldns/rrdef.h $(srcdir)/sldns/wire2str.h acl_list.lo acl_list.o: $(srcdir)/daemon/acl_list.c config.h $(srcdir)/daemon/acl_list.h \ $(srcdir)/util/storage/dnstree.h $(srcdir)/util/rbtree.h $(srcdir)/util/regional.h $(srcdir)/util/log.h \ $(srcdir)/util/config_file.h $(srcdir)/util/net_help.h cachedump.lo cachedump.o: $(srcdir)/daemon/cachedump.c config.h \ $(srcdir)/daemon/cachedump.h $(srcdir)/daemon/remote.h $(srcdir)/daemon/worker.h $(srcdir)/libunbound/worker.h \ - $(srcdir)/ldns/sbuffer.h $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/storage/lruhash.h \ + $(srcdir)/sldns/sbuffer.h $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/storage/lruhash.h \ $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/util/netevent.h $(srcdir)/util/alloc.h \ - $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h \ - $(srcdir)/daemon/stats.h $(srcdir)/util/timehist.h $(srcdir)/util/module.h $(srcdir)/services/cache/rrset.h \ - $(srcdir)/util/storage/slabhash.h $(srcdir)/services/cache/dns.h $(srcdir)/services/cache/infra.h \ - $(srcdir)/util/rtt.h $(srcdir)/util/regional.h $(srcdir)/util/net_help.h $(srcdir)/util/data/dname.h \ - $(srcdir)/iterator/iterator.h $(srcdir)/services/outbound_list.h $(srcdir)/iterator/iter_delegpt.h \ - $(srcdir)/iterator/iter_utils.h $(srcdir)/iterator/iter_resptype.h $(srcdir)/iterator/iter_fwd.h \ - $(srcdir)/util/rbtree.h $(srcdir)/iterator/iter_hints.h $(srcdir)/util/storage/dnstree.h \ - $(srcdir)/ldns/wire2str.h $(srcdir)/ldns/str2wire.h + $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h \ + $(srcdir)/daemon/stats.h $(srcdir)/util/timehist.h $(srcdir)/util/module.h $(srcdir)/dnstap/dnstap.h \ + $(srcdir)/services/cache/rrset.h $(srcdir)/util/storage/slabhash.h \ + $(srcdir)/services/cache/dns.h $(srcdir)/services/cache/infra.h $(srcdir)/util/rtt.h $(srcdir)/util/regional.h \ + $(srcdir)/util/net_help.h $(srcdir)/util/data/dname.h $(srcdir)/iterator/iterator.h \ + $(srcdir)/services/outbound_list.h $(srcdir)/iterator/iter_delegpt.h $(srcdir)/iterator/iter_utils.h \ + $(srcdir)/iterator/iter_resptype.h $(srcdir)/iterator/iter_fwd.h $(srcdir)/util/rbtree.h \ + $(srcdir)/iterator/iter_hints.h $(srcdir)/util/storage/dnstree.h $(srcdir)/sldns/wire2str.h \ + $(srcdir)/sldns/str2wire.h daemon.lo daemon.o: $(srcdir)/daemon/daemon.c config.h \ $(srcdir)/daemon/daemon.h $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/util/alloc.h $(srcdir)/services/modstack.h \ - $(srcdir)/daemon/worker.h $(srcdir)/libunbound/worker.h $(srcdir)/ldns/sbuffer.h \ - $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/netevent.h \ - $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h \ - $(srcdir)/daemon/stats.h $(srcdir)/util/timehist.h $(srcdir)/util/module.h $(srcdir)/daemon/remote.h \ + $(srcdir)/daemon/worker.h $(srcdir)/libunbound/worker.h \ + $(srcdir)/sldns/sbuffer.h $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/storage/lruhash.h \ + $(srcdir)/util/netevent.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h \ + $(srcdir)/sldns/rrdef.h $(srcdir)/daemon/stats.h $(srcdir)/util/timehist.h $(srcdir)/util/module.h \ + $(srcdir)/dnstap/dnstap.h $(srcdir)/daemon/remote.h \ $(srcdir)/daemon/acl_list.h $(srcdir)/util/storage/dnstree.h $(srcdir)/util/rbtree.h \ $(srcdir)/util/config_file.h $(srcdir)/util/storage/lookup3.h $(srcdir)/util/storage/slabhash.h \ $(srcdir)/services/listen_dnsport.h $(srcdir)/services/cache/rrset.h $(srcdir)/services/cache/infra.h \ $(srcdir)/util/rtt.h $(srcdir)/services/localzone.h $(srcdir)/util/random.h $(srcdir)/util/tube.h \ - $(srcdir)/util/net_help.h $(srcdir)/ldns/keyraw.h + $(srcdir)/util/net_help.h $(srcdir)/sldns/keyraw.h remote.lo remote.o: $(srcdir)/daemon/remote.c config.h \ $(srcdir)/daemon/remote.h \ - $(srcdir)/daemon/worker.h $(srcdir)/libunbound/worker.h $(srcdir)/ldns/sbuffer.h \ + $(srcdir)/daemon/worker.h $(srcdir)/libunbound/worker.h $(srcdir)/sldns/sbuffer.h \ $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h \ $(srcdir)/util/netevent.h $(srcdir)/util/alloc.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/msgparse.h \ - $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h $(srcdir)/daemon/stats.h $(srcdir)/util/timehist.h $(srcdir)/util/module.h \ - $(srcdir)/daemon/daemon.h $(srcdir)/services/modstack.h $(srcdir)/daemon/cachedump.h \ - $(srcdir)/util/config_file.h $(srcdir)/util/net_help.h $(srcdir)/services/listen_dnsport.h \ - $(srcdir)/services/cache/rrset.h $(srcdir)/util/storage/slabhash.h $(srcdir)/services/cache/infra.h \ - $(srcdir)/util/rtt.h $(srcdir)/services/mesh.h $(srcdir)/util/rbtree.h $(srcdir)/services/localzone.h \ - $(srcdir)/util/fptr_wlist.h $(srcdir)/util/tube.h $(srcdir)/util/data/dname.h $(srcdir)/validator/validator.h \ + $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/daemon/stats.h $(srcdir)/util/timehist.h $(srcdir)/util/module.h \ + $(srcdir)/dnstap/dnstap.h $(srcdir)/daemon/daemon.h \ + $(srcdir)/services/modstack.h $(srcdir)/daemon/cachedump.h $(srcdir)/util/config_file.h \ + $(srcdir)/util/net_help.h $(srcdir)/services/listen_dnsport.h $(srcdir)/services/cache/rrset.h \ + $(srcdir)/util/storage/slabhash.h $(srcdir)/services/cache/infra.h $(srcdir)/util/rtt.h \ + $(srcdir)/services/mesh.h $(srcdir)/util/rbtree.h $(srcdir)/services/localzone.h $(srcdir)/util/fptr_wlist.h \ + $(srcdir)/util/tube.h $(srcdir)/util/data/dname.h $(srcdir)/validator/validator.h \ $(srcdir)/validator/val_utils.h $(srcdir)/validator/val_kcache.h $(srcdir)/validator/val_kentry.h \ $(srcdir)/validator/val_anchor.h $(srcdir)/iterator/iterator.h $(srcdir)/services/outbound_list.h \ $(srcdir)/iterator/iter_fwd.h $(srcdir)/iterator/iter_hints.h $(srcdir)/util/storage/dnstree.h \ - $(srcdir)/iterator/iter_delegpt.h $(srcdir)/services/outside_network.h $(srcdir)/ldns/str2wire.h \ - $(srcdir)/ldns/parseutil.h $(srcdir)/ldns/wire2str.h + $(srcdir)/iterator/iter_delegpt.h $(srcdir)/services/outside_network.h $(srcdir)/sldns/str2wire.h \ + $(srcdir)/sldns/parseutil.h $(srcdir)/sldns/wire2str.h stats.lo stats.o: $(srcdir)/daemon/stats.c config.h $(srcdir)/daemon/stats.h $(srcdir)/util/timehist.h \ - $(srcdir)/daemon/worker.h $(srcdir)/libunbound/worker.h $(srcdir)/ldns/sbuffer.h \ + $(srcdir)/daemon/worker.h $(srcdir)/libunbound/worker.h $(srcdir)/sldns/sbuffer.h \ $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h \ $(srcdir)/util/netevent.h $(srcdir)/util/alloc.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/msgparse.h \ - $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h $(srcdir)/util/module.h $(srcdir)/daemon/daemon.h \ - $(srcdir)/services/modstack.h $(srcdir)/services/mesh.h $(srcdir)/util/rbtree.h \ - $(srcdir)/services/outside_network.h $(srcdir)/util/config_file.h $(srcdir)/util/tube.h \ - $(srcdir)/util/net_help.h $(srcdir)/validator/validator.h $(srcdir)/validator/val_utils.h \ - $(srcdir)/services/cache/rrset.h $(srcdir)/util/storage/slabhash.h $(srcdir)/services/cache/infra.h \ - $(srcdir)/util/rtt.h $(srcdir)/validator/val_kcache.h + $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/util/module.h $(srcdir)/dnstap/dnstap.h \ + $(srcdir)/daemon/daemon.h $(srcdir)/services/modstack.h \ + $(srcdir)/services/mesh.h $(srcdir)/util/rbtree.h $(srcdir)/services/outside_network.h \ + $(srcdir)/util/config_file.h $(srcdir)/util/tube.h $(srcdir)/util/net_help.h $(srcdir)/validator/validator.h \ + $(srcdir)/validator/val_utils.h $(srcdir)/services/cache/rrset.h $(srcdir)/util/storage/slabhash.h \ + $(srcdir)/services/cache/infra.h $(srcdir)/util/rtt.h $(srcdir)/validator/val_kcache.h unbound.lo unbound.o: $(srcdir)/daemon/unbound.c config.h $(srcdir)/util/log.h $(srcdir)/daemon/daemon.h \ - $(srcdir)/util/locks.h $(srcdir)/util/alloc.h $(srcdir)/services/modstack.h $(srcdir)/daemon/remote.h \ + $(srcdir)/util/locks.h $(srcdir)/util/alloc.h $(srcdir)/services/modstack.h \ + $(srcdir)/daemon/remote.h \ $(srcdir)/util/config_file.h $(srcdir)/util/storage/slabhash.h $(srcdir)/util/storage/lruhash.h \ $(srcdir)/services/listen_dnsport.h $(srcdir)/util/netevent.h $(srcdir)/services/cache/rrset.h \ $(srcdir)/util/data/packed_rrset.h $(srcdir)/services/cache/infra.h $(srcdir)/util/rtt.h \ $(srcdir)/util/fptr_wlist.h $(srcdir)/util/module.h $(srcdir)/util/data/msgreply.h \ - $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h $(srcdir)/util/tube.h \ + $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/util/tube.h \ $(srcdir)/services/mesh.h $(srcdir)/util/rbtree.h $(srcdir)/util/net_help.h $(srcdir)/util/mini_event.h \ $(srcdir)/util/rbtree.h worker.lo worker.o: $(srcdir)/daemon/worker.c config.h $(srcdir)/util/log.h $(srcdir)/util/net_help.h \ - $(srcdir)/util/random.h $(srcdir)/daemon/worker.h $(srcdir)/libunbound/worker.h $(srcdir)/ldns/sbuffer.h \ + $(srcdir)/util/random.h $(srcdir)/daemon/worker.h $(srcdir)/libunbound/worker.h $(srcdir)/sldns/sbuffer.h \ $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h \ $(srcdir)/util/netevent.h $(srcdir)/util/alloc.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/msgparse.h \ - $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h $(srcdir)/daemon/stats.h $(srcdir)/util/timehist.h $(srcdir)/util/module.h \ - $(srcdir)/daemon/daemon.h $(srcdir)/services/modstack.h $(srcdir)/daemon/remote.h \ + $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/daemon/stats.h $(srcdir)/util/timehist.h $(srcdir)/util/module.h \ + $(srcdir)/dnstap/dnstap.h $(srcdir)/daemon/daemon.h \ + $(srcdir)/services/modstack.h $(srcdir)/daemon/remote.h \ $(srcdir)/daemon/acl_list.h $(srcdir)/util/storage/dnstree.h $(srcdir)/util/rbtree.h \ $(srcdir)/util/config_file.h $(srcdir)/util/regional.h $(srcdir)/util/storage/slabhash.h \ $(srcdir)/services/listen_dnsport.h $(srcdir)/services/outside_network.h \ @@ -1006,22 +1022,24 @@ worker.lo worker.o: $(srcdir)/daemon/worker.c config.h $(srcdir)/util/log.h $(sr testbound.lo testbound.o: $(srcdir)/testcode/testbound.c config.h $(srcdir)/testcode/testpkts.h \ $(srcdir)/testcode/replay.h $(srcdir)/util/netevent.h $(srcdir)/util/rbtree.h $(srcdir)/testcode/fake_event.h \ $(srcdir)/daemon/remote.h \ - $(srcdir)/util/config_file.h $(srcdir)/ldns/keyraw.h $(srcdir)/daemon/unbound.c $(srcdir)/util/log.h \ + $(srcdir)/util/config_file.h $(srcdir)/sldns/keyraw.h $(srcdir)/daemon/unbound.c $(srcdir)/util/log.h \ $(srcdir)/daemon/daemon.h $(srcdir)/util/locks.h $(srcdir)/util/alloc.h $(srcdir)/services/modstack.h \ - $(srcdir)/util/storage/slabhash.h $(srcdir)/util/storage/lruhash.h $(srcdir)/services/listen_dnsport.h \ - $(srcdir)/services/cache/rrset.h $(srcdir)/util/data/packed_rrset.h $(srcdir)/services/cache/infra.h \ - $(srcdir)/util/rtt.h $(srcdir)/util/fptr_wlist.h $(srcdir)/util/module.h $(srcdir)/util/data/msgreply.h \ - $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h $(srcdir)/util/tube.h \ + $(srcdir)/util/storage/slabhash.h $(srcdir)/util/storage/lruhash.h \ + $(srcdir)/services/listen_dnsport.h $(srcdir)/services/cache/rrset.h \ + $(srcdir)/util/data/packed_rrset.h $(srcdir)/services/cache/infra.h $(srcdir)/util/rtt.h \ + $(srcdir)/util/fptr_wlist.h $(srcdir)/util/module.h $(srcdir)/util/data/msgreply.h \ + $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/util/tube.h \ $(srcdir)/services/mesh.h $(srcdir)/util/net_help.h $(srcdir)/util/mini_event.h $(srcdir)/util/rbtree.h testpkts.lo testpkts.o: $(srcdir)/testcode/testpkts.c config.h $(srcdir)/testcode/testpkts.h \ - $(srcdir)/util/net_help.h $(srcdir)/util/log.h $(srcdir)/ldns/sbuffer.h $(srcdir)/ldns/rrdef.h $(srcdir)/ldns/pkthdr.h \ - $(srcdir)/ldns/str2wire.h $(srcdir)/ldns/wire2str.h + $(srcdir)/util/net_help.h $(srcdir)/util/log.h $(srcdir)/sldns/sbuffer.h $(srcdir)/sldns/rrdef.h $(srcdir)/sldns/pkthdr.h \ + $(srcdir)/sldns/str2wire.h $(srcdir)/sldns/wire2str.h worker.lo worker.o: $(srcdir)/daemon/worker.c config.h $(srcdir)/util/log.h $(srcdir)/util/net_help.h \ - $(srcdir)/util/random.h $(srcdir)/daemon/worker.h $(srcdir)/libunbound/worker.h $(srcdir)/ldns/sbuffer.h \ + $(srcdir)/util/random.h $(srcdir)/daemon/worker.h $(srcdir)/libunbound/worker.h $(srcdir)/sldns/sbuffer.h \ $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h \ $(srcdir)/util/netevent.h $(srcdir)/util/alloc.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/msgparse.h \ - $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h $(srcdir)/daemon/stats.h $(srcdir)/util/timehist.h $(srcdir)/util/module.h \ - $(srcdir)/daemon/daemon.h $(srcdir)/services/modstack.h $(srcdir)/daemon/remote.h \ + $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/daemon/stats.h $(srcdir)/util/timehist.h $(srcdir)/util/module.h \ + $(srcdir)/dnstap/dnstap.h $(srcdir)/daemon/daemon.h \ + $(srcdir)/services/modstack.h $(srcdir)/daemon/remote.h \ $(srcdir)/daemon/acl_list.h $(srcdir)/util/storage/dnstree.h $(srcdir)/util/rbtree.h \ $(srcdir)/util/config_file.h $(srcdir)/util/regional.h $(srcdir)/util/storage/slabhash.h \ $(srcdir)/services/listen_dnsport.h $(srcdir)/services/outside_network.h \ @@ -1036,132 +1054,136 @@ acl_list.lo acl_list.o: $(srcdir)/daemon/acl_list.c config.h $(srcdir)/daemon/ac $(srcdir)/util/config_file.h $(srcdir)/util/net_help.h daemon.lo daemon.o: $(srcdir)/daemon/daemon.c config.h \ $(srcdir)/daemon/daemon.h $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/util/alloc.h $(srcdir)/services/modstack.h \ - $(srcdir)/daemon/worker.h $(srcdir)/libunbound/worker.h $(srcdir)/ldns/sbuffer.h \ - $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/netevent.h \ - $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h \ - $(srcdir)/daemon/stats.h $(srcdir)/util/timehist.h $(srcdir)/util/module.h $(srcdir)/daemon/remote.h \ + $(srcdir)/daemon/worker.h $(srcdir)/libunbound/worker.h \ + $(srcdir)/sldns/sbuffer.h $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/storage/lruhash.h \ + $(srcdir)/util/netevent.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h \ + $(srcdir)/sldns/rrdef.h $(srcdir)/daemon/stats.h $(srcdir)/util/timehist.h $(srcdir)/util/module.h \ + $(srcdir)/dnstap/dnstap.h $(srcdir)/daemon/remote.h \ $(srcdir)/daemon/acl_list.h $(srcdir)/util/storage/dnstree.h $(srcdir)/util/rbtree.h \ $(srcdir)/util/config_file.h $(srcdir)/util/storage/lookup3.h $(srcdir)/util/storage/slabhash.h \ $(srcdir)/services/listen_dnsport.h $(srcdir)/services/cache/rrset.h $(srcdir)/services/cache/infra.h \ $(srcdir)/util/rtt.h $(srcdir)/services/localzone.h $(srcdir)/util/random.h $(srcdir)/util/tube.h \ - $(srcdir)/util/net_help.h $(srcdir)/ldns/keyraw.h + $(srcdir)/util/net_help.h $(srcdir)/sldns/keyraw.h stats.lo stats.o: $(srcdir)/daemon/stats.c config.h $(srcdir)/daemon/stats.h $(srcdir)/util/timehist.h \ - $(srcdir)/daemon/worker.h $(srcdir)/libunbound/worker.h $(srcdir)/ldns/sbuffer.h \ + $(srcdir)/daemon/worker.h $(srcdir)/libunbound/worker.h $(srcdir)/sldns/sbuffer.h \ $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h \ $(srcdir)/util/netevent.h $(srcdir)/util/alloc.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/msgparse.h \ - $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h $(srcdir)/util/module.h $(srcdir)/daemon/daemon.h \ - $(srcdir)/services/modstack.h $(srcdir)/services/mesh.h $(srcdir)/util/rbtree.h \ - $(srcdir)/services/outside_network.h $(srcdir)/util/config_file.h $(srcdir)/util/tube.h \ - $(srcdir)/util/net_help.h $(srcdir)/validator/validator.h $(srcdir)/validator/val_utils.h \ - $(srcdir)/services/cache/rrset.h $(srcdir)/util/storage/slabhash.h $(srcdir)/services/cache/infra.h \ - $(srcdir)/util/rtt.h $(srcdir)/validator/val_kcache.h + $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/util/module.h $(srcdir)/dnstap/dnstap.h \ + $(srcdir)/daemon/daemon.h $(srcdir)/services/modstack.h \ + $(srcdir)/services/mesh.h $(srcdir)/util/rbtree.h $(srcdir)/services/outside_network.h $(srcdir)/services/listen_dnsport.h \ + $(srcdir)/util/config_file.h $(srcdir)/util/tube.h $(srcdir)/util/net_help.h $(srcdir)/validator/validator.h \ + $(srcdir)/validator/val_utils.h $(srcdir)/services/cache/rrset.h $(srcdir)/util/storage/slabhash.h \ + $(srcdir)/services/cache/infra.h $(srcdir)/util/rtt.h $(srcdir)/validator/val_kcache.h replay.lo replay.o: $(srcdir)/testcode/replay.c config.h $(srcdir)/util/log.h $(srcdir)/util/net_help.h \ $(srcdir)/util/config_file.h $(srcdir)/testcode/replay.h $(srcdir)/util/netevent.h $(srcdir)/testcode/testpkts.h \ - $(srcdir)/util/rbtree.h $(srcdir)/testcode/fake_event.h $(srcdir)/ldns/str2wire.h $(srcdir)/ldns/rrdef.h + $(srcdir)/util/rbtree.h $(srcdir)/testcode/fake_event.h $(srcdir)/sldns/str2wire.h $(srcdir)/sldns/rrdef.h fake_event.lo fake_event.o: $(srcdir)/testcode/fake_event.c config.h $(srcdir)/testcode/fake_event.h \ $(srcdir)/util/netevent.h $(srcdir)/util/net_help.h $(srcdir)/util/log.h $(srcdir)/util/data/msgparse.h \ - $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h \ + $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h \ $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/data/msgencode.h \ $(srcdir)/util/data/dname.h $(srcdir)/util/config_file.h $(srcdir)/services/listen_dnsport.h \ - $(srcdir)/services/outside_network.h $(srcdir)/util/rbtree.h $(srcdir)/services/cache/infra.h \ - $(srcdir)/util/rtt.h $(srcdir)/testcode/replay.h $(srcdir)/testcode/testpkts.h $(srcdir)/util/fptr_wlist.h \ - $(srcdir)/util/module.h $(srcdir)/util/tube.h $(srcdir)/services/mesh.h $(srcdir)/services/modstack.h \ - $(srcdir)/ldns/sbuffer.h $(srcdir)/ldns/wire2str.h $(srcdir)/ldns/str2wire.h + $(srcdir)/services/outside_network.h $(srcdir)/util/rbtree.h \ + $(srcdir)/services/cache/infra.h $(srcdir)/util/rtt.h $(srcdir)/testcode/replay.h $(srcdir)/testcode/testpkts.h \ + $(srcdir)/util/fptr_wlist.h $(srcdir)/util/module.h $(srcdir)/util/tube.h $(srcdir)/services/mesh.h \ + $(srcdir)/services/modstack.h $(srcdir)/sldns/sbuffer.h $(srcdir)/sldns/wire2str.h $(srcdir)/sldns/str2wire.h lock_verify.lo lock_verify.o: $(srcdir)/testcode/lock_verify.c config.h $(srcdir)/util/log.h $(srcdir)/util/rbtree.h \ $(srcdir)/util/locks.h $(srcdir)/util/fptr_wlist.h $(srcdir)/util/netevent.h $(srcdir)/util/storage/lruhash.h \ $(srcdir)/util/module.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/packed_rrset.h \ - $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h $(srcdir)/util/tube.h \ + $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/util/tube.h \ $(srcdir)/services/mesh.h $(srcdir)/services/modstack.h pktview.lo pktview.o: $(srcdir)/testcode/pktview.c config.h $(srcdir)/util/log.h $(srcdir)/util/data/dname.h \ - $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h \ - $(srcdir)/ldns/rrdef.h $(srcdir)/testcode/unitmain.h $(srcdir)/testcode/readhex.h $(srcdir)/ldns/sbuffer.h \ - $(srcdir)/ldns/parseutil.h + $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h \ + $(srcdir)/sldns/rrdef.h $(srcdir)/testcode/unitmain.h $(srcdir)/testcode/readhex.h $(srcdir)/sldns/sbuffer.h \ + $(srcdir)/sldns/parseutil.h readhex.lo readhex.o: $(srcdir)/testcode/readhex.c config.h $(srcdir)/testcode/readhex.h $(srcdir)/util/log.h \ - $(srcdir)/ldns/sbuffer.h $(srcdir)/ldns/parseutil.h + $(srcdir)/sldns/sbuffer.h $(srcdir)/sldns/parseutil.h memstats.lo memstats.o: $(srcdir)/testcode/memstats.c config.h $(srcdir)/util/log.h $(srcdir)/util/rbtree.h \ $(srcdir)/util/locks.h $(srcdir)/util/fptr_wlist.h $(srcdir)/util/netevent.h $(srcdir)/util/storage/lruhash.h \ $(srcdir)/util/module.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/packed_rrset.h \ - $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h $(srcdir)/util/tube.h \ + $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/util/tube.h \ $(srcdir)/services/mesh.h $(srcdir)/services/modstack.h unbound-checkconf.lo unbound-checkconf.o: $(srcdir)/smallapp/unbound-checkconf.c config.h $(srcdir)/util/log.h \ $(srcdir)/util/config_file.h $(srcdir)/util/module.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h \ $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/data/msgparse.h \ - $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h $(srcdir)/util/net_help.h $(srcdir)/util/regional.h \ + $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/util/net_help.h $(srcdir)/util/regional.h \ $(srcdir)/iterator/iterator.h $(srcdir)/services/outbound_list.h $(srcdir)/iterator/iter_fwd.h \ $(srcdir)/util/rbtree.h $(srcdir)/iterator/iter_hints.h $(srcdir)/util/storage/dnstree.h \ $(srcdir)/validator/validator.h $(srcdir)/validator/val_utils.h $(srcdir)/services/localzone.h \ - $(srcdir)/ldns/sbuffer.h + $(srcdir)/sldns/sbuffer.h worker_cb.lo worker_cb.o: $(srcdir)/smallapp/worker_cb.c config.h $(srcdir)/libunbound/context.h \ $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/util/alloc.h $(srcdir)/util/rbtree.h $(srcdir)/services/modstack.h \ $(srcdir)/libunbound/unbound.h $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/storage/lruhash.h \ - $(srcdir)/libunbound/worker.h $(srcdir)/ldns/sbuffer.h $(srcdir)/util/fptr_wlist.h $(srcdir)/util/netevent.h \ - $(srcdir)/util/module.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h \ - $(srcdir)/ldns/rrdef.h $(srcdir)/util/tube.h $(srcdir)/services/mesh.h + $(srcdir)/libunbound/worker.h $(srcdir)/sldns/sbuffer.h $(srcdir)/util/fptr_wlist.h $(srcdir)/util/netevent.h \ + $(srcdir)/util/module.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h \ + $(srcdir)/sldns/rrdef.h $(srcdir)/util/tube.h $(srcdir)/services/mesh.h context.lo context.o: $(srcdir)/libunbound/context.c config.h $(srcdir)/libunbound/context.h \ $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/util/alloc.h $(srcdir)/util/rbtree.h $(srcdir)/services/modstack.h \ $(srcdir)/libunbound/unbound.h $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/storage/lruhash.h \ - $(srcdir)/util/module.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h \ - $(srcdir)/ldns/rrdef.h $(srcdir)/util/config_file.h $(srcdir)/util/net_help.h $(srcdir)/services/localzone.h \ + $(srcdir)/util/module.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h \ + $(srcdir)/sldns/rrdef.h $(srcdir)/util/config_file.h $(srcdir)/util/net_help.h $(srcdir)/services/localzone.h \ $(srcdir)/services/cache/rrset.h $(srcdir)/util/storage/slabhash.h $(srcdir)/services/cache/infra.h \ - $(srcdir)/util/rtt.h $(srcdir)/ldns/sbuffer.h + $(srcdir)/util/rtt.h $(srcdir)/sldns/sbuffer.h libunbound.lo libunbound.o: $(srcdir)/libunbound/libunbound.c $(srcdir)/libunbound/unbound.h \ $(srcdir)/libunbound/unbound-event.h config.h $(srcdir)/libunbound/context.h $(srcdir)/util/locks.h \ $(srcdir)/util/log.h $(srcdir)/util/alloc.h $(srcdir)/util/rbtree.h $(srcdir)/services/modstack.h \ $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/storage/lruhash.h $(srcdir)/libunbound/libworker.h \ $(srcdir)/util/config_file.h $(srcdir)/util/module.h $(srcdir)/util/data/msgreply.h \ - $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h $(srcdir)/util/regional.h \ + $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/util/regional.h \ $(srcdir)/util/random.h $(srcdir)/util/net_help.h $(srcdir)/util/tube.h $(srcdir)/services/localzone.h \ $(srcdir)/services/cache/infra.h $(srcdir)/util/rtt.h $(srcdir)/services/cache/rrset.h \ - $(srcdir)/util/storage/slabhash.h $(srcdir)/ldns/sbuffer.h + $(srcdir)/util/storage/slabhash.h $(srcdir)/sldns/sbuffer.h libworker.lo libworker.o: $(srcdir)/libunbound/libworker.c config.h \ $(srcdir)/libunbound/libworker.h $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/storage/lruhash.h \ $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/libunbound/context.h $(srcdir)/util/alloc.h $(srcdir)/util/rbtree.h \ $(srcdir)/services/modstack.h $(srcdir)/libunbound/unbound.h $(srcdir)/libunbound/worker.h \ - $(srcdir)/ldns/sbuffer.h $(srcdir)/libunbound/unbound-event.h $(srcdir)/services/outside_network.h \ - $(srcdir)/util/netevent.h $(srcdir)/services/mesh.h $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h \ - $(srcdir)/ldns/rrdef.h $(srcdir)/util/module.h $(srcdir)/util/data/msgreply.h $(srcdir)/services/localzone.h \ - $(srcdir)/services/cache/rrset.h $(srcdir)/util/storage/slabhash.h $(srcdir)/services/outbound_list.h \ - $(srcdir)/util/fptr_wlist.h $(srcdir)/util/tube.h $(srcdir)/util/regional.h $(srcdir)/util/random.h \ - $(srcdir)/util/config_file.h $(srcdir)/util/storage/lookup3.h $(srcdir)/util/net_help.h \ - $(srcdir)/util/data/dname.h $(srcdir)/util/data/msgencode.h $(srcdir)/iterator/iter_fwd.h \ - $(srcdir)/iterator/iter_hints.h $(srcdir)/util/storage/dnstree.h $(srcdir)/ldns/str2wire.h + $(srcdir)/sldns/sbuffer.h $(srcdir)/libunbound/unbound-event.h $(srcdir)/services/outside_network.h \ + $(srcdir)/util/netevent.h $(srcdir)/services/mesh.h \ + $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/util/module.h \ + $(srcdir)/util/data/msgreply.h $(srcdir)/services/localzone.h $(srcdir)/services/cache/rrset.h \ + $(srcdir)/util/storage/slabhash.h $(srcdir)/services/outbound_list.h $(srcdir)/util/fptr_wlist.h \ + $(srcdir)/util/tube.h $(srcdir)/util/regional.h $(srcdir)/util/random.h $(srcdir)/util/config_file.h \ + $(srcdir)/util/storage/lookup3.h $(srcdir)/util/net_help.h $(srcdir)/util/data/dname.h \ + $(srcdir)/util/data/msgencode.h $(srcdir)/iterator/iter_fwd.h $(srcdir)/iterator/iter_hints.h \ + $(srcdir)/util/storage/dnstree.h $(srcdir)/sldns/str2wire.h unbound-host.lo unbound-host.o: $(srcdir)/smallapp/unbound-host.c config.h $(srcdir)/libunbound/unbound.h \ - $(srcdir)/ldns/rrdef.h $(srcdir)/ldns/wire2str.h + $(srcdir)/sldns/rrdef.h $(srcdir)/sldns/wire2str.h asynclook.lo asynclook.o: $(srcdir)/testcode/asynclook.c config.h $(srcdir)/libunbound/unbound.h \ $(srcdir)/libunbound/context.h $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/util/alloc.h $(srcdir)/util/rbtree.h \ $(srcdir)/services/modstack.h $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/storage/lruhash.h \ - $(srcdir)/ldns/rrdef.h + $(srcdir)/sldns/rrdef.h streamtcp.lo streamtcp.o: $(srcdir)/testcode/streamtcp.c config.h $(srcdir)/util/locks.h $(srcdir)/util/log.h \ $(srcdir)/util/net_help.h $(srcdir)/util/data/msgencode.h $(srcdir)/util/data/msgparse.h \ - $(srcdir)/util/storage/lruhash.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h $(srcdir)/util/data/msgreply.h \ - $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/data/dname.h $(srcdir)/ldns/sbuffer.h \ - $(srcdir)/ldns/str2wire.h $(srcdir)/ldns/wire2str.h \ + $(srcdir)/util/storage/lruhash.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h $(srcdir)/util/data/msgreply.h \ + $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/data/dname.h $(srcdir)/sldns/sbuffer.h \ + $(srcdir)/sldns/str2wire.h $(srcdir)/sldns/wire2str.h \ perf.lo perf.o: $(srcdir)/testcode/perf.c config.h $(srcdir)/util/log.h $(srcdir)/util/locks.h $(srcdir)/util/net_help.h \ $(srcdir)/util/data/msgencode.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/storage/lruhash.h \ - $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h \ - $(srcdir)/ldns/sbuffer.h $(srcdir)/ldns/wire2str.h $(srcdir)/ldns/str2wire.h + $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h \ + $(srcdir)/sldns/sbuffer.h $(srcdir)/sldns/wire2str.h $(srcdir)/sldns/str2wire.h delayer.lo delayer.o: $(srcdir)/testcode/delayer.c config.h $(srcdir)/util/net_help.h $(srcdir)/util/log.h \ - $(srcdir)/util/config_file.h $(srcdir)/ldns/sbuffer.h + $(srcdir)/util/config_file.h $(srcdir)/sldns/sbuffer.h unbound-control.lo unbound-control.o: $(srcdir)/smallapp/unbound-control.c config.h \ - $(srcdir)/util/log.h $(srcdir)/util/config_file.h $(srcdir)/util/locks.h $(srcdir)/util/net_help.h + $(srcdir)/util/log.h \ + $(srcdir)/util/config_file.h $(srcdir)/util/locks.h $(srcdir)/util/net_help.h unbound-anchor.lo unbound-anchor.o: $(srcdir)/smallapp/unbound-anchor.c config.h $(srcdir)/libunbound/unbound.h \ - $(srcdir)/ldns/rrdef.h \ + $(srcdir)/sldns/rrdef.h \ petal.lo petal.o: $(srcdir)/testcode/petal.c config.h \ pythonmod_utils.lo pythonmod_utils.o: $(srcdir)/pythonmod/pythonmod_utils.c config.h $(srcdir)/util/module.h \ $(srcdir)/util/storage/lruhash.h $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/util/data/msgreply.h \ - $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h \ + $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/rrdef.h \ $(srcdir)/util/netevent.h $(srcdir)/util/net_help.h $(srcdir)/services/cache/dns.h \ $(srcdir)/services/cache/rrset.h $(srcdir)/util/storage/slabhash.h $(srcdir)/util/regional.h \ - $(srcdir)/ldns/sbuffer.h + $(srcdir)/iterator/iter_delegpt.h $(srcdir)/sldns/sbuffer.h win_svc.lo win_svc.o: $(srcdir)/winrc/win_svc.c config.h $(srcdir)/winrc/win_svc.h $(srcdir)/winrc/w_inst.h \ $(srcdir)/daemon/daemon.h $(srcdir)/util/locks.h $(srcdir)/util/log.h $(srcdir)/util/alloc.h $(srcdir)/services/modstack.h \ - $(srcdir)/daemon/worker.h $(srcdir)/libunbound/worker.h $(srcdir)/ldns/sbuffer.h \ - $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/storage/lruhash.h $(srcdir)/util/netevent.h \ - $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/msgparse.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/rrdef.h \ - $(srcdir)/daemon/stats.h $(srcdir)/util/timehist.h $(srcdir)/util/module.h $(srcdir)/daemon/remote.h \ + $(srcdir)/daemon/worker.h $(srcdir)/libunbound/worker.h \ + $(srcdir)/sldns/sbuffer.h $(srcdir)/util/data/packed_rrset.h $(srcdir)/util/storage/lruhash.h \ + $(srcdir)/util/netevent.h $(srcdir)/util/data/msgreply.h $(srcdir)/util/data/msgparse.h $(srcdir)/sldns/pkthdr.h \ + $(srcdir)/sldns/rrdef.h $(srcdir)/daemon/stats.h $(srcdir)/util/timehist.h $(srcdir)/util/module.h \ + $(srcdir)/dnstap/dnstap.h $(srcdir)/daemon/remote.h \ $(srcdir)/util/config_file.h $(srcdir)/util/winsock_event.h w_inst.lo w_inst.o: $(srcdir)/winrc/w_inst.c config.h $(srcdir)/winrc/w_inst.h $(srcdir)/winrc/win_svc.h unbound-service-install.lo unbound-service-install.o: $(srcdir)/winrc/unbound-service-install.c config.h \ @@ -1169,20 +1191,20 @@ unbound-service-install.lo unbound-service-install.o: $(srcdir)/winrc/unbound-se unbound-service-remove.lo unbound-service-remove.o: $(srcdir)/winrc/unbound-service-remove.c config.h \ $(srcdir)/winrc/w_inst.h anchor-update.lo anchor-update.o: $(srcdir)/winrc/anchor-update.c config.h $(srcdir)/libunbound/unbound.h \ - $(srcdir)/ldns/rrdef.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/wire2str.h -keyraw.lo keyraw.o: $(srcdir)/ldns/keyraw.c config.h $(srcdir)/ldns/keyraw.h \ - $(srcdir)/ldns/rrdef.h \ + $(srcdir)/sldns/rrdef.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/wire2str.h +keyraw.lo keyraw.o: $(srcdir)/sldns/keyraw.c config.h $(srcdir)/sldns/keyraw.h \ + $(srcdir)/sldns/rrdef.h \ -sbuffer.lo sbuffer.o: $(srcdir)/ldns/sbuffer.c config.h $(srcdir)/ldns/sbuffer.h -wire2str.lo wire2str.o: $(srcdir)/ldns/wire2str.c config.h $(srcdir)/ldns/wire2str.h $(srcdir)/ldns/str2wire.h \ - $(srcdir)/ldns/rrdef.h $(srcdir)/ldns/pkthdr.h $(srcdir)/ldns/parseutil.h $(srcdir)/ldns/sbuffer.h $(srcdir)/ldns/keyraw.h \ +sbuffer.lo sbuffer.o: $(srcdir)/sldns/sbuffer.c config.h $(srcdir)/sldns/sbuffer.h +wire2str.lo wire2str.o: $(srcdir)/sldns/wire2str.c config.h $(srcdir)/sldns/wire2str.h $(srcdir)/sldns/str2wire.h \ + $(srcdir)/sldns/rrdef.h $(srcdir)/sldns/pkthdr.h $(srcdir)/sldns/parseutil.h $(srcdir)/sldns/sbuffer.h $(srcdir)/sldns/keyraw.h \ -parse.lo parse.o: $(srcdir)/ldns/parse.c config.h $(srcdir)/ldns/parse.h $(srcdir)/ldns/parseutil.h \ - $(srcdir)/ldns/sbuffer.h -parseutil.lo parseutil.o: $(srcdir)/ldns/parseutil.c config.h $(srcdir)/ldns/parseutil.h -rrdef.lo rrdef.o: $(srcdir)/ldns/rrdef.c config.h $(srcdir)/ldns/rrdef.h $(srcdir)/ldns/parseutil.h -str2wire.lo str2wire.o: $(srcdir)/ldns/str2wire.c config.h $(srcdir)/ldns/str2wire.h $(srcdir)/ldns/rrdef.h \ - $(srcdir)/ldns/wire2str.h $(srcdir)/ldns/sbuffer.h $(srcdir)/ldns/parse.h $(srcdir)/ldns/parseutil.h +parse.lo parse.o: $(srcdir)/sldns/parse.c config.h $(srcdir)/sldns/parse.h $(srcdir)/sldns/parseutil.h \ + $(srcdir)/sldns/sbuffer.h +parseutil.lo parseutil.o: $(srcdir)/sldns/parseutil.c config.h $(srcdir)/sldns/parseutil.h +rrdef.lo rrdef.o: $(srcdir)/sldns/rrdef.c config.h $(srcdir)/sldns/rrdef.h $(srcdir)/sldns/parseutil.h +str2wire.lo str2wire.o: $(srcdir)/sldns/str2wire.c config.h $(srcdir)/sldns/str2wire.h $(srcdir)/sldns/rrdef.h \ + $(srcdir)/sldns/wire2str.h $(srcdir)/sldns/sbuffer.h $(srcdir)/sldns/parse.h $(srcdir)/sldns/parseutil.h ctime_r.lo ctime_r.o: $(srcdir)/compat/ctime_r.c config.h $(srcdir)/util/locks.h $(srcdir)/util/log.h fake-rfc2553.lo fake-rfc2553.o: $(srcdir)/compat/fake-rfc2553.c $(srcdir)/compat/fake-rfc2553.h config.h gmtime_r.lo gmtime_r.o: $(srcdir)/compat/gmtime_r.c config.h @@ -1195,6 +1217,7 @@ memmove.lo memmove.o: $(srcdir)/compat/memmove.c config.h snprintf.lo snprintf.o: $(srcdir)/compat/snprintf.c config.h strlcat.lo strlcat.o: $(srcdir)/compat/strlcat.c config.h strlcpy.lo strlcpy.o: $(srcdir)/compat/strlcpy.c config.h +reallocarray.lo reallocarray.o: $(srcdir)/compat/reallocarray.c config.h strptime.lo strptime.o: $(srcdir)/compat/strptime.c config.h getentropy_linux.lo getentropy_linux.o: $(srcdir)/compat/getentropy_linux.c config.h \ diff --git a/external/unbound/acx_nlnetlabs.m4 b/external/unbound/acx_nlnetlabs.m4 index e1cf83a70..decf0f586 100644 --- a/external/unbound/acx_nlnetlabs.m4 +++ b/external/unbound/acx_nlnetlabs.m4 @@ -2,7 +2,8 @@ # Copyright 2009, Wouter Wijngaards, NLnet Labs. # BSD licensed. # -# Version 26 +# Version 27 +# 2015-03-17 AHX_CONFIG_REALLOCARRAY added # 2013-09-19 FLTO help text improved. # 2013-07-18 Enable ACX_CHECK_COMPILER_FLAG to test for -Wstrict-prototypes # 2013-06-25 FLTO has --disable-flto option. @@ -1213,6 +1214,16 @@ struct tm *gmtime_r(const time_t *timep, struct tm *result); #endif ]) +dnl provide reallocarray compat prototype. +dnl $1: unique name for compat code +AC_DEFUN([AHX_CONFIG_REALLOCARRAY], +[ +#ifndef HAVE_REALLOCARRAY +#define reallocarray reallocarray$1 +void* reallocarray(void *ptr, size_t nmemb, size_t size); +#endif +]) + dnl provide w32 compat definition for sleep AC_DEFUN([AHX_CONFIG_W32_SLEEP], [ diff --git a/external/unbound/compat/getentropy_linux.c b/external/unbound/compat/getentropy_linux.c index 32d58a7cd..76f0f9df5 100644 --- a/external/unbound/compat/getentropy_linux.c +++ b/external/unbound/compat/getentropy_linux.c @@ -77,6 +77,9 @@ int getentropy(void *buf, size_t len); extern int main(int, char *argv[]); #endif static int gotdata(char *buf, size_t len); +#ifdef SYS_getrandom +static int getentropy_getrandom(void *buf, size_t len); +#endif static int getentropy_urandom(void *buf, size_t len); #ifdef SYS__sysctl static int getentropy_sysctl(void *buf, size_t len); @@ -93,6 +96,17 @@ getentropy(void *buf, size_t len) return -1; } +#ifdef SYS_getrandom + /* + * Try descriptor-less getrandom() + */ + ret = getentropy_getrandom(buf, len); + if (ret != -1) + return (ret); + if (errno != ENOSYS) + return (-1); +#endif + /* * Try to get entropy with /dev/urandom * @@ -178,6 +192,25 @@ gotdata(char *buf, size_t len) return 0; } +#ifdef SYS_getrandom +static int +getentropy_getrandom(void *buf, size_t len) +{ + int pre_errno = errno; + int ret; + if (len > 256) + return (-1); + do { + ret = syscall(SYS_getrandom, buf, len, 0); + } while (ret == -1 && errno == EINTR); + + if (ret != (int)len) + return (-1); + errno = pre_errno; + return (0); +} +#endif + static int getentropy_urandom(void *buf, size_t len) { @@ -251,7 +284,7 @@ getentropy_sysctl(void *buf, size_t len) struct __sysctl_args args = { .name = mib, .nlen = 3, - .oldval = buf + i, + .oldval = (char *)buf + i, .oldlenp = &chunk, }; if (syscall(SYS__sysctl, &args) != 0) @@ -474,22 +507,24 @@ getentropy_fallback(void *buf, size_t len) HD(cnt); } -#ifdef AT_RANDOM +#ifdef HAVE_GETAUXVAL +# ifdef AT_RANDOM /* Not as random as you think but we take what we are given */ p = (char *) getauxval(AT_RANDOM); if (p) HR(p, 16); -#endif -#ifdef AT_SYSINFO_EHDR +# endif +# ifdef AT_SYSINFO_EHDR p = (char *) getauxval(AT_SYSINFO_EHDR); if (p) HR(p, pgs); -#endif -#ifdef AT_BASE +# endif +# ifdef AT_BASE p = (char *) getauxval(AT_BASE); if (p) HD(p); -#endif +# endif +#endif /* HAVE_GETAUXVAL */ SHA512_Final(results, &ctx); memcpy((char*)buf + i, results, min(sizeof(results), len - i)); diff --git a/external/unbound/compat/reallocarray.c b/external/unbound/compat/reallocarray.c new file mode 100644 index 000000000..04d5d71c8 --- /dev/null +++ b/external/unbound/compat/reallocarray.c @@ -0,0 +1,39 @@ +/* $OpenBSD: reallocarray.c,v 1.1 2014/05/08 21:43:49 deraadt Exp $ */ +/* + * Copyright (c) 2008 Otto Moerbeek <otto@drijf.net> + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include "config.h" +#include <sys/types.h> +#include <errno.h> +#include <stdint.h> +#include <stdlib.h> + +/* + * This is sqrt(SIZE_MAX+1), as s1*s2 <= SIZE_MAX + * if both s1 < MUL_NO_OVERFLOW and s2 < MUL_NO_OVERFLOW + */ +#define MUL_NO_OVERFLOW ((size_t)1 << (sizeof(size_t) * 4)) + +void * +reallocarray(void *optr, size_t nmemb, size_t size) +{ + if ((nmemb >= MUL_NO_OVERFLOW || size >= MUL_NO_OVERFLOW) && + nmemb > 0 && SIZE_MAX / nmemb < size) { + errno = ENOMEM; + return NULL; + } + return realloc(optr, size * nmemb); +} diff --git a/external/unbound/config.guess b/external/unbound/config.guess index c0adba94b..b79252d6b 100755 --- a/external/unbound/config.guess +++ b/external/unbound/config.guess @@ -1,14 +1,12 @@ #! /bin/sh # Attempt to guess a canonical system name. -# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, -# 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -# 2011, 2012 Free Software Foundation, Inc. +# Copyright 1992-2013 Free Software Foundation, Inc. -timestamp='2012-06-10' +timestamp='2013-06-10' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or +# the Free Software Foundation; either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, but @@ -22,19 +20,17 @@ timestamp='2012-06-10' # As a special exception to the GNU General Public License, if you # distribute this file as part of a program that contains a # configuration script generated by Autoconf, you may include it under -# the same distribution terms that you use for the rest of that program. - - -# Originally written by Per Bothner. Please send patches (context -# diff format) to <config-patches@gnu.org> and include a ChangeLog -# entry. +# the same distribution terms that you use for the rest of that +# program. This Exception is an additional permission under section 7 +# of the GNU General Public License, version 3 ("GPLv3"). # -# This script attempts to guess a canonical system name similar to -# config.sub. If it succeeds, it prints the system name on stdout, and -# exits with 0. Otherwise, it exits with 1. +# Originally written by Per Bothner. # # You can get the latest version of this script from: # http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD +# +# Please send patches with a ChangeLog entry to config-patches@gnu.org. + me=`echo "$0" | sed -e 's,.*/,,'` @@ -54,9 +50,7 @@ version="\ GNU config.guess ($timestamp) Originally written by Per Bothner. -Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, -2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 -Free Software Foundation, Inc. +Copyright 1992-2013 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -138,6 +132,27 @@ UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown UNAME_SYSTEM=`(uname -s) 2>/dev/null` || UNAME_SYSTEM=unknown UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown +case "${UNAME_SYSTEM}" in +Linux|GNU|GNU/*) + # If the system lacks a compiler, then just pick glibc. + # We could probably try harder. + LIBC=gnu + + eval $set_cc_for_build + cat <<-EOF > $dummy.c + #include <features.h> + #if defined(__UCLIBC__) + LIBC=uclibc + #elif defined(__dietlibc__) + LIBC=dietlibc + #else + LIBC=gnu + #endif + EOF + eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^LIBC'` + ;; +esac + # Note: order is significant - the case branches are not exclusive. case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in @@ -200,6 +215,10 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in # CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used. echo "${machine}-${os}${release}" exit ;; + *:Bitrig:*:*) + UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'` + echo ${UNAME_MACHINE_ARCH}-unknown-bitrig${UNAME_RELEASE} + exit ;; *:OpenBSD:*:*) UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'` echo ${UNAME_MACHINE_ARCH}-unknown-openbsd${UNAME_RELEASE} @@ -302,7 +321,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*) echo arm-acorn-riscix${UNAME_RELEASE} exit ;; - arm:riscos:*:*|arm:RISCOS:*:*) + arm*:riscos:*:*|arm*:RISCOS:*:*) echo arm-unknown-riscos exit ;; SR2?01:HI-UX/MPP:*:* | SR8000:HI-UX/MPP:*:*) @@ -801,6 +820,9 @@ EOF i*:CYGWIN*:*) echo ${UNAME_MACHINE}-pc-cygwin exit ;; + *:MINGW64*:*) + echo ${UNAME_MACHINE}-pc-mingw64 + exit ;; *:MINGW*:*) echo ${UNAME_MACHINE}-pc-mingw32 exit ;; @@ -852,21 +874,21 @@ EOF exit ;; *:GNU:*:*) # the GNU system - echo `echo ${UNAME_MACHINE}|sed -e 's,[-/].*$,,'`-unknown-gnu`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'` + echo `echo ${UNAME_MACHINE}|sed -e 's,[-/].*$,,'`-unknown-${LIBC}`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'` exit ;; *:GNU/*:*:*) # other systems with GNU libc and userland - echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr '[A-Z]' '[a-z]'``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-gnu + echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr '[A-Z]' '[a-z]'``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-${LIBC} exit ;; i*86:Minix:*:*) echo ${UNAME_MACHINE}-pc-minix exit ;; aarch64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; aarch64_be:Linux:*:*) UNAME_MACHINE=aarch64_be - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; alpha:Linux:*:*) case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in @@ -879,59 +901,54 @@ EOF EV68*) UNAME_MACHINE=alphaev68 ;; esac objdump --private-headers /bin/sh | grep -q ld.so.1 - if test "$?" = 0 ; then LIBC="libc1" ; else LIBC="" ; fi - echo ${UNAME_MACHINE}-unknown-linux-gnu${LIBC} + if test "$?" = 0 ; then LIBC="gnulibc1" ; fi + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; + arc:Linux:*:* | arceb:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; arm*:Linux:*:*) eval $set_cc_for_build if echo __ARM_EABI__ | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ARM_EABI__ then - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} else if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ARM_PCS_VFP then - echo ${UNAME_MACHINE}-unknown-linux-gnueabi + echo ${UNAME_MACHINE}-unknown-linux-${LIBC}eabi else - echo ${UNAME_MACHINE}-unknown-linux-gnueabihf + echo ${UNAME_MACHINE}-unknown-linux-${LIBC}eabihf fi fi exit ;; avr32*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; cris:Linux:*:*) - echo ${UNAME_MACHINE}-axis-linux-gnu + echo ${UNAME_MACHINE}-axis-linux-${LIBC} exit ;; crisv32:Linux:*:*) - echo ${UNAME_MACHINE}-axis-linux-gnu + echo ${UNAME_MACHINE}-axis-linux-${LIBC} exit ;; frv:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; hexagon:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; i*86:Linux:*:*) - LIBC=gnu - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c - #ifdef __dietlibc__ - LIBC=dietlibc - #endif -EOF - eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^LIBC'` - echo "${UNAME_MACHINE}-pc-linux-${LIBC}" + echo ${UNAME_MACHINE}-pc-linux-${LIBC} exit ;; ia64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; m32r*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; m68*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; mips:Linux:*:* | mips64:Linux:*:*) eval $set_cc_for_build @@ -950,54 +967,63 @@ EOF #endif EOF eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^CPU'` - test x"${CPU}" != x && { echo "${CPU}-unknown-linux-gnu"; exit; } + test x"${CPU}" != x && { echo "${CPU}-unknown-linux-${LIBC}"; exit; } ;; + or1k:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + exit ;; or32:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; padre:Linux:*:*) - echo sparc-unknown-linux-gnu + echo sparc-unknown-linux-${LIBC} exit ;; parisc64:Linux:*:* | hppa64:Linux:*:*) - echo hppa64-unknown-linux-gnu + echo hppa64-unknown-linux-${LIBC} exit ;; parisc:Linux:*:* | hppa:Linux:*:*) # Look for CPU level case `grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2` in - PA7*) echo hppa1.1-unknown-linux-gnu ;; - PA8*) echo hppa2.0-unknown-linux-gnu ;; - *) echo hppa-unknown-linux-gnu ;; + PA7*) echo hppa1.1-unknown-linux-${LIBC} ;; + PA8*) echo hppa2.0-unknown-linux-${LIBC} ;; + *) echo hppa-unknown-linux-${LIBC} ;; esac exit ;; ppc64:Linux:*:*) - echo powerpc64-unknown-linux-gnu + echo powerpc64-unknown-linux-${LIBC} exit ;; ppc:Linux:*:*) - echo powerpc-unknown-linux-gnu + echo powerpc-unknown-linux-${LIBC} + exit ;; + ppc64le:Linux:*:*) + echo powerpc64le-unknown-linux-${LIBC} + exit ;; + ppcle:Linux:*:*) + echo powerpcle-unknown-linux-${LIBC} exit ;; s390:Linux:*:* | s390x:Linux:*:*) - echo ${UNAME_MACHINE}-ibm-linux + echo ${UNAME_MACHINE}-ibm-linux-${LIBC} exit ;; sh64*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; sh*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; sparc:Linux:*:* | sparc64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; tile*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; vax:Linux:*:*) - echo ${UNAME_MACHINE}-dec-linux-gnu + echo ${UNAME_MACHINE}-dec-linux-${LIBC} exit ;; x86_64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; xtensa*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu + echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; i*86:DYNIX/ptx:4*:*) # ptx 4.0 does uname -s correctly, with DYNIX/ptx in there. @@ -1201,6 +1227,9 @@ EOF BePC:Haiku:*:*) # Haiku running on Intel PC compatible. echo i586-pc-haiku exit ;; + x86_64:Haiku:*:*) + echo x86_64-unknown-haiku + exit ;; SX-4:SUPER-UX:*:*) echo sx4-nec-superux${UNAME_RELEASE} exit ;; @@ -1227,19 +1256,21 @@ EOF exit ;; *:Darwin:*:*) UNAME_PROCESSOR=`uname -p` || UNAME_PROCESSOR=unknown - case $UNAME_PROCESSOR in - i386) - eval $set_cc_for_build - if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then - if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ - (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_64BIT_ARCH >/dev/null - then - UNAME_PROCESSOR="x86_64" - fi - fi ;; - unknown) UNAME_PROCESSOR=powerpc ;; - esac + eval $set_cc_for_build + if test "$UNAME_PROCESSOR" = unknown ; then + UNAME_PROCESSOR=powerpc + fi + if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then + if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ + (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_64BIT_ARCH >/dev/null + then + case $UNAME_PROCESSOR in + i386) UNAME_PROCESSOR=x86_64 ;; + powerpc) UNAME_PROCESSOR=powerpc64 ;; + esac + fi + fi echo ${UNAME_PROCESSOR}-apple-darwin${UNAME_RELEASE} exit ;; *:procnto*:*:* | *:QNX:[0123456789]*:*) @@ -1330,9 +1361,6 @@ EOF exit ;; esac -#echo '(No uname command or uname output not recognized.)' 1>&2 -#echo "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" 1>&2 - eval $set_cc_for_build cat >$dummy.c <<EOF #ifdef _SEQUENT_ diff --git a/external/unbound/config.h.cmake.in b/external/unbound/config.h.cmake.in index 827503b6e..863a65421 100644 --- a/external/unbound/config.h.cmake.in +++ b/external/unbound/config.h.cmake.in @@ -45,7 +45,7 @@ /* Whether the C compiler accepts the "unused" attribute */ #cmakedefine HAVE_ATTR_UNUSED -/* Define to 1 if your system has a working `chown' function. */ +/* Define to 1 if you have the `chown' function. */ #cmakedefine HAVE_CHOWN /* Define to 1 if you have the `chroot' function. */ @@ -73,6 +73,10 @@ if you don't. */ #cmakedefine HAVE_DECL_NID_X9_62_PRIME256V1 +/* Define to 1 if you have the declaration of `reallocarray', and to 0 if you + don't. */ +#cmakedefine HAVE_DECL_REALLOCARRAY + /* Define to 1 if you have the declaration of `sk_SSL_COMP_pop_free', and to 0 if you don't. */ #cmakedefine HAVE_DECL_SK_SSL_COMP_POP_FREE @@ -149,6 +153,9 @@ /* Whether getaddrinfo is available */ #cmakedefine HAVE_GETADDRINFO +/* Define to 1 if you have the `getauxval' function. */ +#cmakedefine HAVE_GETAUXVAL + /* Define to 1 if you have the `getentropy' function. */ #cmakedefine HAVE_GETENTROPY @@ -266,6 +273,9 @@ /* Define to 1 if you have the `random' function. */ #cmakedefine HAVE_RANDOM +/* Define to 1 if you have the `reallocarray' function. */ +#cmakedefine HAVE_REALLOCARRAY + /* Define to 1 if you have the `recvmsg' function. */ #cmakedefine HAVE_RECVMSG @@ -353,6 +363,9 @@ /* Define to 1 if `ipi_spec_dst' is a member of `struct in_pktinfo'. */ #cmakedefine HAVE_STRUCT_IN_PKTINFO_IPI_SPEC_DST +/* Define to 1 if `sun_len' is a member of `struct sockaddr_un'. */ +#cmakedefine HAVE_STRUCT_SOCKADDR_UN_SUN_LEN + /* Define if you have Swig libraries and header files. */ #cmakedefine HAVE_SWIG @@ -383,6 +396,9 @@ /* Define to 1 if you have the <sys/uio.h> header file. */ #cmakedefine HAVE_SYS_UIO_H +/* Define to 1 if you have the <sys/un.h> header file. */ +#cmakedefine HAVE_SYS_UN_H + /* Define to 1 if you have the <sys/wait.h> header file. */ #cmakedefine HAVE_SYS_WAIT_H @@ -805,6 +821,10 @@ #endif +#ifndef AF_LOCAL +#define AF_LOCAL AF_UNIX +#endif + #ifdef HAVE_ATTR_FORMAT # define ATTR_FORMAT(archetype, string_index, first_to_check) \ @@ -887,6 +907,12 @@ struct tm *gmtime_r(const time_t *timep, struct tm *result); #endif +#ifndef HAVE_REALLOCARRAY +#define reallocarray reallocarrayunbound +void* reallocarray(void *ptr, size_t nmemb, size_t size); +#endif + + #if !defined(HAVE_SLEEP) || defined(HAVE_WINDOWS_H) #define sleep(x) Sleep((x)*1000) /* on win32 */ #endif /* HAVE_SLEEP */ @@ -952,6 +978,9 @@ uint32_t arc4random(void); # if !HAVE_DECL_ARC4RANDOM_UNIFORM && defined(HAVE_ARC4RANDOM_UNIFORM) uint32_t arc4random_uniform(uint32_t upper_bound); # endif +# if !HAVE_DECL_REALLOCARRAY +void *reallocarray(void *ptr, size_t nmemb, size_t size); +# endif #endif /* HAVE_LIBRESSL */ #ifndef HAVE_ARC4RANDOM void explicit_bzero(void* buf, size_t len); diff --git a/external/unbound/config.h.in b/external/unbound/config.h.in index 2b7770b5c..723b3ad02 100644 --- a/external/unbound/config.h.in +++ b/external/unbound/config.h.in @@ -42,7 +42,7 @@ /* Whether the C compiler accepts the "unused" attribute */ #undef HAVE_ATTR_UNUSED -/* Define to 1 if your system has a working `chown' function. */ +/* Define to 1 if you have the `chown' function. */ #undef HAVE_CHOWN /* Define to 1 if you have the `chroot' function. */ @@ -70,6 +70,10 @@ if you don't. */ #undef HAVE_DECL_NID_X9_62_PRIME256V1 +/* Define to 1 if you have the declaration of `reallocarray', and to 0 if you + don't. */ +#undef HAVE_DECL_REALLOCARRAY + /* Define to 1 if you have the declaration of `sk_SSL_COMP_pop_free', and to 0 if you don't. */ #undef HAVE_DECL_SK_SSL_COMP_POP_FREE @@ -146,6 +150,9 @@ /* Whether getaddrinfo is available */ #undef HAVE_GETADDRINFO +/* Define to 1 if you have the `getauxval' function. */ +#undef HAVE_GETAUXVAL + /* Define to 1 if you have the `getentropy' function. */ #undef HAVE_GETENTROPY @@ -263,6 +270,9 @@ /* Define to 1 if you have the `random' function. */ #undef HAVE_RANDOM +/* Define to 1 if you have the `reallocarray' function. */ +#undef HAVE_REALLOCARRAY + /* Define to 1 if you have the `recvmsg' function. */ #undef HAVE_RECVMSG @@ -350,6 +360,9 @@ /* Define to 1 if `ipi_spec_dst' is a member of `struct in_pktinfo'. */ #undef HAVE_STRUCT_IN_PKTINFO_IPI_SPEC_DST +/* Define to 1 if `sun_len' is a member of `struct sockaddr_un'. */ +#undef HAVE_STRUCT_SOCKADDR_UN_SUN_LEN + /* Define if you have Swig libraries and header files. */ #undef HAVE_SWIG @@ -380,6 +393,9 @@ /* Define to 1 if you have the <sys/uio.h> header file. */ #undef HAVE_SYS_UIO_H +/* Define to 1 if you have the <sys/un.h> header file. */ +#undef HAVE_SYS_UN_H + /* Define to 1 if you have the <sys/wait.h> header file. */ #undef HAVE_SYS_WAIT_H @@ -793,6 +809,10 @@ #define ARG_LL "%I64" #endif +#ifndef AF_LOCAL +#define AF_LOCAL AF_UNIX +#endif + #ifdef HAVE_ATTR_FORMAT @@ -876,6 +896,12 @@ struct tm *gmtime_r(const time_t *timep, struct tm *result); #endif +#ifndef HAVE_REALLOCARRAY +#define reallocarray reallocarrayunbound +void* reallocarray(void *ptr, size_t nmemb, size_t size); +#endif + + #if !defined(HAVE_SLEEP) || defined(HAVE_WINDOWS_H) #define sleep(x) Sleep((x)*1000) /* on win32 */ #endif /* HAVE_SLEEP */ @@ -941,6 +967,9 @@ uint32_t arc4random(void); # if !HAVE_DECL_ARC4RANDOM_UNIFORM && defined(HAVE_ARC4RANDOM_UNIFORM) uint32_t arc4random_uniform(uint32_t upper_bound); # endif +# if !HAVE_DECL_REALLOCARRAY +void *reallocarray(void *ptr, size_t nmemb, size_t size); +# endif #endif /* HAVE_LIBRESSL */ #ifndef HAVE_ARC4RANDOM void explicit_bzero(void* buf, size_t len); diff --git a/external/unbound/config.sub b/external/unbound/config.sub index 6205f8423..d2a961303 100755 --- a/external/unbound/config.sub +++ b/external/unbound/config.sub @@ -1,24 +1,18 @@ #! /bin/sh # Configuration validation subroutine script. -# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, -# 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -# 2011, 2012 Free Software Foundation, Inc. +# Copyright 1992-2013 Free Software Foundation, Inc. -timestamp='2012-04-18' +timestamp='2013-08-10' -# This file is (in principle) common to ALL GNU software. -# The presence of a machine in this file suggests that SOME GNU software -# can handle that machine. It does not imply ALL GNU software can. -# -# This file is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or # (at your option) any later version. # -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, see <http://www.gnu.org/licenses/>. @@ -26,11 +20,12 @@ timestamp='2012-04-18' # As a special exception to the GNU General Public License, if you # distribute this file as part of a program that contains a # configuration script generated by Autoconf, you may include it under -# the same distribution terms that you use for the rest of that program. +# the same distribution terms that you use for the rest of that +# program. This Exception is an additional permission under section 7 +# of the GNU General Public License, version 3 ("GPLv3"). -# Please send patches to <config-patches@gnu.org>. Submit a context -# diff and a properly formatted GNU ChangeLog entry. +# Please send patches with a ChangeLog entry to config-patches@gnu.org. # # Configuration subroutine to validate and canonicalize a configuration type. # Supply the specified configuration type as an argument. @@ -73,9 +68,7 @@ Report bugs and patches to <config-patches@gnu.org>." version="\ GNU config.sub ($timestamp) -Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, -2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 -Free Software Foundation, Inc. +Copyright 1992-2013 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -123,7 +116,7 @@ esac maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'` case $maybe_os in nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc | linux-newlib* | \ - linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \ + linux-musl* | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \ knetbsd*-gnu* | netbsd*-gnu* | \ kopensolaris*-gnu* | \ storm-chaos* | os2-emx* | rtmk-nova*) @@ -156,7 +149,7 @@ case $os in -convergent* | -ncr* | -news | -32* | -3600* | -3100* | -hitachi* |\ -c[123]* | -convex* | -sun | -crds | -omron* | -dg | -ultra | -tti* | \ -harris | -dolphin | -highlevel | -gould | -cbm | -ns | -masscomp | \ - -apple | -axis | -knuth | -cray | -microblaze) + -apple | -axis | -knuth | -cray | -microblaze*) os= basic_machine=$1 ;; @@ -259,10 +252,12 @@ case $basic_machine in | alpha | alphaev[4-8] | alphaev56 | alphaev6[78] | alphapca5[67] \ | alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] | alpha64pca5[67] \ | am33_2.0 \ - | arc | arm | arm[bl]e | arme[lb] | armv[2345] | armv[345][lb] | avr | avr32 \ - | be32 | be64 \ + | arc | arceb \ + | arm | arm[bl]e | arme[lb] | armv[2-8] | armv[3-8][lb] | armv7[arm] \ + | avr | avr32 \ + | be32 | be64 \ | bfin \ - | c4x | clipper \ + | c4x | c8051 | clipper \ | d10v | d30v | dlx | dsp16xx \ | epiphany \ | fido | fr30 | frv \ @@ -273,7 +268,7 @@ case $basic_machine in | le32 | le64 \ | lm32 \ | m32c | m32r | m32rle | m68000 | m68k | m88k \ - | maxq | mb | microblaze | mcore | mep | metag \ + | maxq | mb | microblaze | microblazeel | mcore | mep | metag \ | mips | mipsbe | mipseb | mipsel | mipsle \ | mips16 \ | mips64 | mips64el \ @@ -291,16 +286,17 @@ case $basic_machine in | mipsisa64r2 | mipsisa64r2el \ | mipsisa64sb1 | mipsisa64sb1el \ | mipsisa64sr71k | mipsisa64sr71kel \ + | mipsr5900 | mipsr5900el \ | mipstx39 | mipstx39el \ | mn10200 | mn10300 \ | moxie \ | mt \ | msp430 \ | nds32 | nds32le | nds32be \ - | nios | nios2 \ + | nios | nios2 | nios2eb | nios2el \ | ns16k | ns32k \ | open8 \ - | or32 \ + | or1k | or32 \ | pdp10 | pdp11 | pj | pjl \ | powerpc | powerpc64 | powerpc64le | powerpcle \ | pyramid \ @@ -370,13 +366,13 @@ case $basic_machine in | aarch64-* | aarch64_be-* \ | alpha-* | alphaev[4-8]-* | alphaev56-* | alphaev6[78]-* \ | alpha64-* | alpha64ev[4-8]-* | alpha64ev56-* | alpha64ev6[78]-* \ - | alphapca5[67]-* | alpha64pca5[67]-* | arc-* \ + | alphapca5[67]-* | alpha64pca5[67]-* | arc-* | arceb-* \ | arm-* | armbe-* | armle-* | armeb-* | armv*-* \ | avr-* | avr32-* \ | be32-* | be64-* \ | bfin-* | bs2000-* \ | c[123]* | c30-* | [cjt]90-* | c4x-* \ - | clipper-* | craynv-* | cydra-* \ + | c8051-* | clipper-* | craynv-* | cydra-* \ | d10v-* | d30v-* | dlx-* \ | elxsi-* \ | f30[01]-* | f700-* | fido-* | fr30-* | frv-* | fx80-* \ @@ -389,7 +385,8 @@ case $basic_machine in | lm32-* \ | m32c-* | m32r-* | m32rle-* \ | m68000-* | m680[012346]0-* | m68360-* | m683?2-* | m68k-* \ - | m88110-* | m88k-* | maxq-* | mcore-* | metag-* | microblaze-* \ + | m88110-* | m88k-* | maxq-* | mcore-* | metag-* \ + | microblaze-* | microblazeel-* \ | mips-* | mipsbe-* | mipseb-* | mipsel-* | mipsle-* \ | mips16-* \ | mips64-* | mips64el-* \ @@ -407,12 +404,13 @@ case $basic_machine in | mipsisa64r2-* | mipsisa64r2el-* \ | mipsisa64sb1-* | mipsisa64sb1el-* \ | mipsisa64sr71k-* | mipsisa64sr71kel-* \ + | mipsr5900-* | mipsr5900el-* \ | mipstx39-* | mipstx39el-* \ | mmix-* \ | mt-* \ | msp430-* \ | nds32-* | nds32le-* | nds32be-* \ - | nios-* | nios2-* \ + | nios-* | nios2-* | nios2eb-* | nios2el-* \ | none-* | np1-* | ns16k-* | ns32k-* \ | open8-* \ | orion-* \ @@ -788,11 +786,15 @@ case $basic_machine in basic_machine=ns32k-utek os=-sysv ;; - microblaze) + microblaze*) basic_machine=microblaze-xilinx ;; + mingw64) + basic_machine=x86_64-pc + os=-mingw64 + ;; mingw32) - basic_machine=i386-pc + basic_machine=i686-pc os=-mingw32 ;; mingw32ce) @@ -828,7 +830,7 @@ case $basic_machine in basic_machine=`echo $basic_machine | sed -e 's/ms1-/mt-/'` ;; msys) - basic_machine=i386-pc + basic_machine=i686-pc os=-msys ;; mvs) @@ -1004,7 +1006,7 @@ case $basic_machine in ;; ppc64) basic_machine=powerpc64-unknown ;; - ppc64-*) basic_machine=powerpc64-`echo $basic_machine | sed 's/^[^-]*-//'` + ppc64-* | ppc64p7-*) basic_machine=powerpc64-`echo $basic_machine | sed 's/^[^-]*-//'` ;; ppc64le | powerpc64little | ppc64-le | powerpc64-little) basic_machine=powerpc64le-unknown @@ -1019,7 +1021,11 @@ case $basic_machine in basic_machine=i586-unknown os=-pw32 ;; - rdos) + rdos | rdos64) + basic_machine=x86_64-pc + os=-rdos + ;; + rdos32) basic_machine=i386-pc os=-rdos ;; @@ -1346,21 +1352,21 @@ case $os in -gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \ | -*vms* | -sco* | -esix* | -isc* | -aix* | -cnk* | -sunos | -sunos[34]*\ | -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \ - | -sym* | -kopensolaris* \ + | -sym* | -kopensolaris* | -plan9* \ | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \ | -aos* | -aros* \ | -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \ | -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \ | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* \ - | -openbsd* | -solidbsd* \ + | -bitrig* | -openbsd* | -solidbsd* \ | -ekkobsd* | -kfreebsd* | -freebsd* | -riscix* | -lynxos* \ | -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \ | -ptx* | -coff* | -ecoff* | -winnt* | -domain* | -vsta* \ | -udi* | -eabi* | -lites* | -ieee* | -go32* | -aux* \ | -chorusos* | -chorusrdb* | -cegcc* \ | -cygwin* | -msys* | -pe* | -psos* | -moss* | -proelf* | -rtems* \ - | -mingw32* | -linux-gnu* | -linux-android* \ - | -linux-newlib* | -linux-uclibc* \ + | -mingw32* | -mingw64* | -linux-gnu* | -linux-android* \ + | -linux-newlib* | -linux-musl* | -linux-uclibc* \ | -uxpv* | -beos* | -mpeix* | -udk* \ | -interix* | -uwin* | -mks* | -rhapsody* | -darwin* | -opened* \ | -openstep* | -oskit* | -conix* | -pw32* | -nonstopux* \ @@ -1492,9 +1498,6 @@ case $os in -aros*) os=-aros ;; - -kaos*) - os=-kaos - ;; -zvmoe) os=-zvmoe ;; @@ -1543,6 +1546,9 @@ case $basic_machine in c4x-* | tic4x-*) os=-coff ;; + c8051-*) + os=-elf + ;; hexagon-*) os=-elf ;; @@ -1586,6 +1592,9 @@ case $basic_machine in mips*-*) os=-elf ;; + or1k-*) + os=-elf + ;; or32-*) os=-coff ;; diff --git a/external/unbound/configure b/external/unbound/configure index bdfc14f22..425466ebd 100755 --- a/external/unbound/configure +++ b/external/unbound/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for unbound 1.5.1. +# Generated by GNU Autoconf 2.69 for unbound 1.5.4. # # Report bugs to <unbound-bugs@nlnetlabs.nl>. # @@ -590,8 +590,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='unbound' PACKAGE_TARNAME='unbound' -PACKAGE_VERSION='1.5.1' -PACKAGE_STRING='unbound 1.5.1' +PACKAGE_VERSION='1.5.4' +PACKAGE_STRING='unbound 1.5.4' PACKAGE_BUGREPORT='unbound-bugs@nlnetlabs.nl' PACKAGE_URL='' @@ -733,6 +733,7 @@ UNBOUND_PIDFILE UNBOUND_SHARE_DIR UNBOUND_CHROOT_DIR UNBOUND_RUN_DIR +ub_conf_dir ub_conf_file EGREP GREP @@ -1387,7 +1388,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures unbound 1.5.1 to adapt to many kinds of systems. +\`configure' configures unbound 1.5.4 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1452,7 +1453,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of unbound 1.5.1:";; + short | recursive ) echo "Configuration of unbound 1.5.4:";; esac cat <<\_ACEOF @@ -1627,7 +1628,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -unbound configure 1.5.1 +unbound configure 1.5.4 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2336,7 +2337,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by unbound $as_me 1.5.1, which was +It was created by unbound $as_me 1.5.4, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -2688,11 +2689,11 @@ UNBOUND_VERSION_MAJOR=1 UNBOUND_VERSION_MINOR=5 -UNBOUND_VERSION_MICRO=1 +UNBOUND_VERSION_MICRO=4 LIBUNBOUND_CURRENT=5 -LIBUNBOUND_REVISION=3 +LIBUNBOUND_REVISION=7 LIBUNBOUND_AGE=3 # 1.0.0 had 0:12:0 # 1.0.1 had 0:13:0 @@ -2732,7 +2733,10 @@ LIBUNBOUND_AGE=3 # 1.4.21 had 4:1:2 # 1.4.22 had 4:1:2 # 1.5.0 had 5:3:3 # adds ub_ctx_add_ta_autr -# 1.5.1 had 5:4:3 +# 1.5.1 had 5:3:3 +# 1.5.2 had 5:5:3 +# 1.5.3 had 5:6:3 +# 1.5.4 had 5:7:3 # Current -- the number of the binary API that we're implementing # Revision -- which iteration of the implementation of the binary @@ -4047,6 +4051,30 @@ cat >>confdefs.h <<_ACEOF #define CONFIGFILE "$hdr_config" _ACEOF +ub_conf_dir=`$as_dirname -- "$ub_conf_file" || +$as_expr X"$ub_conf_file" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$ub_conf_file" : 'X\(//\)[^/]' \| \ + X"$ub_conf_file" : 'X\(//\)$' \| \ + X"$ub_conf_file" : 'X\(/\)' \| . 2>/dev/null || +$as_echo X"$ub_conf_file" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ + s//\1/ + q + } + /^X\(\/\/\)[^/].*/{ + s//\1/ + q + } + /^X\(\/\/\)$/{ + s//\1/ + q + } + /^X\(\/\).*/{ + s//\1/ + q + } + s/.*/./; q'` + # Determine run, chroot directory and pidfile locations @@ -13713,7 +13741,7 @@ CC="$lt_save_CC" # Checks for header files. -for ac_header in stdarg.h stdbool.h netinet/in.h sys/param.h sys/socket.h sys/uio.h sys/resource.h arpa/inet.h syslog.h netdb.h sys/wait.h pwd.h glob.h grp.h login_cap.h winsock2.h ws2tcpip.h endian.h +for ac_header in stdarg.h stdbool.h netinet/in.h sys/param.h sys/socket.h sys/un.h sys/uio.h sys/resource.h arpa/inet.h syslog.h netdb.h sys/wait.h pwd.h glob.h grp.h login_cap.h winsock2.h ws2tcpip.h endian.h do : as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh` ac_fn_c_check_header_compile "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default @@ -16701,6 +16729,16 @@ fi cat >>confdefs.h <<_ACEOF #define HAVE_DECL_ARC4RANDOM_UNIFORM $ac_have_decl _ACEOF +ac_fn_c_check_decl "$LINENO" "reallocarray" "ac_cv_have_decl_reallocarray" "$ac_includes_default" +if test "x$ac_cv_have_decl_reallocarray" = xyes; then : + ac_have_decl=1 +else + ac_have_decl=0 +fi + +cat >>confdefs.h <<_ACEOF +#define HAVE_DECL_REALLOCARRAY $ac_have_decl +_ACEOF else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 @@ -17820,6 +17858,22 @@ fi fi +ac_fn_c_check_member "$LINENO" "struct sockaddr_un" "sun_len" "ac_cv_member_struct_sockaddr_un_sun_len" " +$ac_includes_default +#ifdef HAVE_SYS_UN_H +#include <sys/un.h> +#endif + +" +if test "x$ac_cv_member_struct_sockaddr_un_sun_len" = xyes; then : + +cat >>confdefs.h <<_ACEOF +#define HAVE_STRUCT_SOCKADDR_UN_SUN_LEN 1 +_ACEOF + + +fi + ac_fn_c_check_member "$LINENO" "struct in_pktinfo" "ipi_spec_dst" "ac_cv_member_struct_in_pktinfo_ipi_spec_dst" " $ac_includes_default #if HAVE_SYS_PARAM_H @@ -17916,7 +17970,7 @@ if test "$ac_res" != no; then : fi -for ac_func in tzset sigprocmask fcntl getpwnam getrlimit setrlimit setsid sbrk chroot kill sleep usleep random srandom recvmsg sendmsg writev socketpair glob initgroups strftime localtime_r setusercontext _beginthreadex endservent endprotoent +for ac_func in tzset sigprocmask fcntl getpwnam getrlimit setrlimit setsid sbrk chroot kill chown sleep usleep random srandom recvmsg sendmsg writev socketpair glob initgroups strftime localtime_r setusercontext _beginthreadex endservent endprotoent do : as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh` ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var" @@ -18093,6 +18147,20 @@ esac fi +ac_fn_c_check_func "$LINENO" "reallocarray" "ac_cv_func_reallocarray" +if test "x$ac_cv_func_reallocarray" = xyes; then : + $as_echo "#define HAVE_REALLOCARRAY 1" >>confdefs.h + +else + case " $LIBOBJS " in + *" reallocarray.$ac_objext "* ) ;; + *) LIBOBJS="$LIBOBJS reallocarray.$ac_objext" + ;; +esac + +fi + + LIBOBJ_WITHOUT_CTIMEARC4="$LIBOBJS" if test "$USE_NSS" = "no"; then @@ -18211,6 +18279,62 @@ done # this lib needed for sha2 on solaris LIBS="$LIBS -lmd" fi + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing clock_gettime" >&5 +$as_echo_n "checking for library containing clock_gettime... " >&6; } +if ${ac_cv_search_clock_gettime+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_func_search_save_LIBS=$LIBS +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char clock_gettime (); +int +main () +{ +return clock_gettime (); + ; + return 0; +} +_ACEOF +for ac_lib in '' rt; do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO"; then : + ac_cv_search_clock_gettime=$ac_res +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext + if ${ac_cv_search_clock_gettime+:} false; then : + break +fi +done +if ${ac_cv_search_clock_gettime+:} false; then : + +else + ac_cv_search_clock_gettime=no +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_clock_gettime" >&5 +$as_echo "$ac_cv_search_clock_gettime" >&6; } +ac_res=$ac_cv_search_clock_gettime +if test "$ac_res" != no; then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + +fi + ;; Linux|*) case " $LIBOBJS " in @@ -18255,6 +18379,17 @@ fi done + for ac_func in getauxval +do : + ac_fn_c_check_func "$LINENO" "getauxval" "ac_cv_func_getauxval" +if test "x$ac_cv_func_getauxval" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_GETAUXVAL 1 +_ACEOF + +fi +done + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing clock_gettime" >&5 $as_echo_n "checking for library containing clock_gettime... " >&6; } if ${ac_cv_search_clock_gettime+:} false; then : @@ -18752,7 +18887,7 @@ _ACEOF -version=1.5.1 +version=1.5.4 date=`date +'%b %e, %Y'` @@ -19267,7 +19402,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by unbound $as_me 1.5.1, which was +This file was extended by unbound $as_me 1.5.4, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -19333,7 +19468,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -unbound config.status 1.5.1 +unbound config.status 1.5.4 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/external/unbound/configure.ac b/external/unbound/configure.ac index 7e5da1a9f..602813834 100644 --- a/external/unbound/configure.ac +++ b/external/unbound/configure.ac @@ -10,14 +10,14 @@ sinclude(dnstap/dnstap.m4) # must be numbers. ac_defun because of later processing m4_define([VERSION_MAJOR],[1]) m4_define([VERSION_MINOR],[5]) -m4_define([VERSION_MICRO],[1]) +m4_define([VERSION_MICRO],[4]) AC_INIT(unbound, m4_defn([VERSION_MAJOR]).m4_defn([VERSION_MINOR]).m4_defn([VERSION_MICRO]), unbound-bugs@nlnetlabs.nl, unbound) AC_SUBST(UNBOUND_VERSION_MAJOR, [VERSION_MAJOR]) AC_SUBST(UNBOUND_VERSION_MINOR, [VERSION_MINOR]) AC_SUBST(UNBOUND_VERSION_MICRO, [VERSION_MICRO]) LIBUNBOUND_CURRENT=5 -LIBUNBOUND_REVISION=3 +LIBUNBOUND_REVISION=7 LIBUNBOUND_AGE=3 # 1.0.0 had 0:12:0 # 1.0.1 had 0:13:0 @@ -57,7 +57,10 @@ LIBUNBOUND_AGE=3 # 1.4.21 had 4:1:2 # 1.4.22 had 4:1:2 # 1.5.0 had 5:3:3 # adds ub_ctx_add_ta_autr -# 1.5.1 had 5:4:3 +# 1.5.1 had 5:3:3 +# 1.5.2 had 5:5:3 +# 1.5.3 had 5:6:3 +# 1.5.4 had 5:7:3 # Current -- the number of the binary API that we're implementing # Revision -- which iteration of the implementation of the binary @@ -118,6 +121,8 @@ AC_ARG_WITH([conf_file], AC_SUBST(ub_conf_file) ACX_ESCAPE_BACKSLASH($ub_conf_file, hdr_config) AC_DEFINE_UNQUOTED(CONFIGFILE, ["$hdr_config"], [Pathname to the Unbound configuration file]) +ub_conf_dir=`AS_DIRNAME(["$ub_conf_file"])` +AC_SUBST(ub_conf_dir) # Determine run, chroot directory and pidfile locations AC_ARG_WITH(run-dir, @@ -266,7 +271,7 @@ AC_CHECK_TOOL(STRIP, strip) ACX_LIBTOOL_C_ONLY # Checks for header files. -AC_CHECK_HEADERS([stdarg.h stdbool.h netinet/in.h sys/param.h sys/socket.h sys/uio.h sys/resource.h arpa/inet.h syslog.h netdb.h sys/wait.h pwd.h glob.h grp.h login_cap.h winsock2.h ws2tcpip.h endian.h],,, [AC_INCLUDES_DEFAULT]) +AC_CHECK_HEADERS([stdarg.h stdbool.h netinet/in.h sys/param.h sys/socket.h sys/un.h sys/uio.h sys/resource.h arpa/inet.h syslog.h netdb.h sys/wait.h pwd.h glob.h grp.h login_cap.h winsock2.h ws2tcpip.h endian.h],,, [AC_INCLUDES_DEFAULT]) # check for types. # Using own tests for int64* because autoconf builtin only give 32bit. @@ -564,7 +569,7 @@ if grep OPENSSL_VERSION_TEXT $ssldir/include/openssl/opensslv.h | grep "LibreSSL AC_DEFINE([HAVE_LIBRESSL], [1], [Define if we have LibreSSL]) # libressl provides these compat functions, but they may also be # declared by the OS in libc. See if they have been declared. - AC_CHECK_DECLS([strlcpy,strlcat,arc4random,arc4random_uniform]) + AC_CHECK_DECLS([strlcpy,strlcat,arc4random,arc4random_uniform,reallocarray]) else AC_MSG_RESULT([no]) fi @@ -938,6 +943,12 @@ if test $ac_cv_func_daemon = yes; then ]) fi +AC_CHECK_MEMBERS([struct sockaddr_un.sun_len],,,[ +AC_INCLUDES_DEFAULT +#ifdef HAVE_SYS_UN_H +#include <sys/un.h> +#endif +]) AC_CHECK_MEMBERS([struct in_pktinfo.ipi_spec_dst],,,[ AC_INCLUDES_DEFAULT #if HAVE_SYS_PARAM_H @@ -969,7 +980,7 @@ AC_INCLUDES_DEFAULT #endif ]) AC_SEARCH_LIBS([setusercontext], [util]) -AC_CHECK_FUNCS([tzset sigprocmask fcntl getpwnam getrlimit setrlimit setsid sbrk chroot kill sleep usleep random srandom recvmsg sendmsg writev socketpair glob initgroups strftime localtime_r setusercontext _beginthreadex endservent endprotoent]) +AC_CHECK_FUNCS([tzset sigprocmask fcntl getpwnam getrlimit setrlimit setsid sbrk chroot kill chown sleep usleep random srandom recvmsg sendmsg writev socketpair glob initgroups strftime localtime_r setusercontext _beginthreadex endservent endprotoent]) AC_CHECK_FUNCS([setresuid],,[AC_CHECK_FUNCS([setreuid])]) AC_CHECK_FUNCS([setresgid],,[AC_CHECK_FUNCS([setregid])]) @@ -985,6 +996,7 @@ AC_REPLACE_FUNCS(strlcat) AC_REPLACE_FUNCS(strlcpy) AC_REPLACE_FUNCS(memmove) AC_REPLACE_FUNCS(gmtime_r) +AC_REPLACE_FUNCS(reallocarray) LIBOBJ_WITHOUT_CTIMEARC4="$LIBOBJS" AC_SUBST(LIBOBJ_WITHOUT_CTIMEARC4) if test "$USE_NSS" = "no"; then @@ -1012,6 +1024,7 @@ if test "$USE_NSS" = "no"; then # this lib needed for sha2 on solaris LIBS="$LIBS -lmd" fi + AC_SEARCH_LIBS([clock_gettime], [rt]) ;; Linux|*) AC_LIBOBJ(getentropy_linux) @@ -1020,6 +1033,7 @@ if test "$USE_NSS" = "no"; then AC_LIBOBJ(sha512) ]) AC_CHECK_HEADERS([sys/sysctl.h],,, [AC_INCLUDES_DEFAULT]) + AC_CHECK_FUNCS([getauxval]) AC_SEARCH_LIBS([clock_gettime], [rt]) ;; esac @@ -1205,6 +1219,10 @@ dnl includes #else #define ARG_LL "%I64" #endif + +#ifndef AF_LOCAL +#define AF_LOCAL AF_UNIX +#endif ] AHX_CONFIG_FORMAT_ATTRIBUTE @@ -1219,6 +1237,7 @@ AHX_CONFIG_MEMMOVE(unbound) AHX_CONFIG_STRLCAT(unbound) AHX_CONFIG_STRLCPY(unbound) AHX_CONFIG_GMTIME_R(unbound) +AHX_CONFIG_REALLOCARRAY(unbound) AHX_CONFIG_W32_SLEEP AHX_CONFIG_W32_USLEEP AHX_CONFIG_W32_RANDOM @@ -1252,6 +1271,9 @@ uint32_t arc4random(void); # if !HAVE_DECL_ARC4RANDOM_UNIFORM && defined(HAVE_ARC4RANDOM_UNIFORM) uint32_t arc4random_uniform(uint32_t upper_bound); # endif +# if !HAVE_DECL_REALLOCARRAY +void *reallocarray(void *ptr, size_t nmemb, size_t size); +# endif #endif /* HAVE_LIBRESSL */ #ifndef HAVE_ARC4RANDOM void explicit_bzero(void* buf, size_t len); diff --git a/external/unbound/configure_checks.cmake b/external/unbound/configure_checks.cmake index ead51e98f..6b812858e 100644 --- a/external/unbound/configure_checks.cmake +++ b/external/unbound/configure_checks.cmake @@ -33,6 +33,7 @@ check_include_file(sys/stat.h HAVE_SYS_STAT_H) check_include_file(sys/sysctl.h HAVE_SYS_SYSCTL_H) check_include_file(sys/types.h HAVE_SYS_TYPES_H) check_include_file(sys/uio.h HAVE_SYS_UIO_H) +check_include_file(sys/un.h HAVE_SYS_UN_H) check_include_file(sys/wait.h HAVE_SYS_WAIT_H) check_include_file(syslog.h HAVE_SYSLOG_H) check_include_file(time.h HAVE_TIME_H) @@ -59,6 +60,7 @@ check_function_exists(endservent HAVE_ENDSERVENT) check_function_exists(fnctl HAVE_FNCTL) check_function_exists(fork HAVE_FORK) check_function_exists(fseeko HAVE_FSEEKO) +check_function_exists(getauxval HAVE_GETAUXVAL) check_function_exists(getentropy HAVE_GETENTROPY) check_function_exists(getpwnam HAVE_GETPWNAM) check_function_exists(getrlimit HAVE_GETRLIMIT) @@ -74,6 +76,7 @@ check_function_exists(localtime_r HAVE_LOCALTIME_R) check_function_exists(malloc HAVE_MALLOC) check_function_exists(memmove HAVE_MEMMOVE) check_function_exists(random HAVE_RANDOM) +check_function_exists(reallocarray HAVE_DECL_REALLOCARRAY) check_function_exists(recvmsg HAVE_RECVMSG) check_function_exists(sbrk HAVE_SBRK) check_function_exists(sendmsg HAVE_SENDMSG) diff --git a/external/unbound/contrib/README b/external/unbound/contrib/README index 49dee02e5..34c8cc46a 100644 --- a/external/unbound/contrib/README +++ b/external/unbound/contrib/README @@ -29,4 +29,5 @@ distribution but may be helpful. works like the BIND feature (removes AAAA records unless AAAA-only domain). Useful for certain 'broken IPv6 default route' scenarios. Patch from Stephane Lapie for ASAHI Net. - +* unbound_smf22.tar.gz: Solaris SMF installation/removal scripts. + Contributed by Yuri Voinov. diff --git a/external/unbound/contrib/unbound_cache.cmd b/external/unbound/contrib/unbound_cache.cmd index 0f0069388..532162b16 100644 --- a/external/unbound/contrib/unbound_cache.cmd +++ b/external/unbound/contrib/unbound_cache.cmd @@ -2,7 +2,7 @@ rem --------------------------------------------------------------
rem -- DNS cache save/load script
rem --
-rem -- Version 1.0
+rem -- Version 1.2
rem -- By Yuri Voinov (c) 2014
rem --------------------------------------------------------------
@@ -19,31 +19,42 @@ exit 1 :start
-set arg=%1
+rem arg1 - command (optional)
+rem arg2 - file name (optional)
+set arg1=%1
+set arg2=%2
-if /I "%arg%" == "-h" goto help
+if /I "%arg1%" == "-h" goto help
-if "%arg%" == "" (
+if "%arg1%" == "" (
echo Loading cache from %program_path%\%fname%
+dir /a %program_path%\%fname%
type %program_path%\%fname%|%uc% load_cache
goto end
)
-if /I "%arg%" == "-s" (
+if defined %arg2% (goto Not_Defined) else (goto Defined)
+
+rem If file not specified; use default dump file
+:Not_defined
+if /I "%arg1%" == "-s" (
echo Saving cache to %program_path%\%fname%
%uc% dump_cache>%program_path%\%fname%
+dir /a %program_path%\%fname%
echo ok
goto end
)
-if /I "%arg%" == "-l" (
+if /I "%arg1%" == "-l" (
echo Loading cache from %program_path%\%fname%
+dir /a %program_path%\%fname%
type %program_path%\%fname%|%uc% load_cache
goto end
)
-if /I "%arg%" == "-r" (
+if /I "%arg1%" == "-r" (
echo Saving cache to %program_path%\%fname%
+dir /a %program_path%\%fname%
%uc% dump_cache>%program_path%\%fname%
echo ok
echo Loading cache from %program_path%\%fname%
@@ -51,15 +62,44 @@ type %program_path%\%fname%|%uc% load_cache goto end
)
+rem If file name specified; use this filename
+:Defined
+if /I "%arg1%" == "-s" (
+echo Saving cache to %arg2%
+%uc% dump_cache>%arg2%
+dir /a %arg2%
+echo ok
+goto end
+)
+
+if /I "%arg1%" == "-l" (
+echo Loading cache from %arg2%
+dir /a %arg2%
+type %arg2%|%uc% load_cache
+goto end
+)
+
+if /I "%arg1%" == "-r" (
+echo Saving cache to %arg2%
+dir /a %arg2%
+%uc% dump_cache>%arg2%
+echo ok
+echo Loading cache from %arg2%
+type %arg2%|%uc% load_cache
+goto end
+)
+
:help
-echo Usage: unbound_cache.cmd [-s] or [-l] or [-r] or [-h]
+echo Usage: unbound_cache.cmd [-s] or [-l] or [-r] or [-h] [filename]
echo.
echo l - Load - default mode. Warming up Unbound DNS cache from saved file. cache-ttl must be high value.
echo s - Save - save Unbound DNS cache contents to plain file with domain names.
echo r - Reload - reloadind new cache entries and refresh existing cache
echo h - this screen.
+echo filename - file to save/load dumped cache. If not specified, %program_path%\%fname% will be used instead.
echo Note: Run without any arguments will be in default mode.
echo Also, unbound-control must be configured.
exit 1
:end
+exit 0
diff --git a/external/unbound/contrib/unbound_cache.sh b/external/unbound/contrib/unbound_cache.sh index c3dd9c3a2..b3e876ba9 100644 --- a/external/unbound/contrib/unbound_cache.sh +++ b/external/unbound/contrib/unbound_cache.sh @@ -1,13 +1,13 @@ #!/sbin/sh -# + # -------------------------------------------------------------- # -- DNS cache save/load script # -- -# -- Version 1.0 +# -- Version 1.2 # -- By Yuri Voinov (c) 2006, 2014 # -------------------------------------------------------------- # -# ident "@(#)unbound_cache.sh 1.1 14/04/26 YV" +# ident "@(#)unbound_cache.sh 1.2 14/10/30 YV" # ############# @@ -27,9 +27,10 @@ BASENAME=`which basename` CAT=`which cat` CUT=`which cut` ECHO=`which echo` +EXPR=`which expr` GETOPT=`which getopt` ID=`which id` -PRINTF=`which printf` +LS=`which ls` ############### # Subroutines # @@ -38,12 +39,13 @@ PRINTF=`which printf` usage_note () { # Script usage note - $ECHO "Usage: `$BASENAME $0` [-s] or [-l] or [-r] or [-h]" - $ECHO + $ECHO "Usage: `$BASENAME $0` [-s] or [-l] or [-r] or [-h] [filename]" + $ECHO . $ECHO "l - Load - default mode. Warming up Unbound DNS cache from saved file. cache-ttl must be high value." $ECHO "s - Save - save Unbound DNS cache contents to plain file with domain names." $ECHO "r - Reload - reloadind new cache entries and refresh existing cache" $ECHO "h - this screen." + $ECHO "filename - file to save/load dumped cache. If not specified, $CONF/$FNAME will be used instead." $ECHO "Note: Run without any arguments will be in default mode." $ECHO " Also, unbound-control must be configured." exit 0 @@ -68,7 +70,12 @@ check_uc () check_saved_file () { - if [ ! -f "$CONF/$FNAME" ]; then + filename=$1 + if [ ! -z "$filename" -a ! -f "$filename" ]; then + $ECHO . + $ECHO "ERROR: File $filename does not exists. Save it first." + exit 1 + elif [ ! -f "$CONF/$FNAME" ]; then $ECHO . $ECHO "ERROR: File $CONF/$FNAME does not exists. Save it first." exit 1 @@ -78,24 +85,42 @@ check_saved_file () save_cache () { # Save unbound cache - $PRINTF "Saving cache in $CONF/$FNAME..." - $UC dump_cache>$CONF/$FNAME + filename=$1 + if [ -z "$filename" ]; then + $ECHO "Saving cache in $CONF/$FNAME..." + $UC dump_cache>$CONF/$FNAME + $LS -lh $CONF/$FNAME + else + $ECHO "Saving cache in $filename..." + $UC dump_cache>$filename + $LS -lh $filename + fi $ECHO "ok" } load_cache () { - # Load saved cache contents and warmup DNS cache - $PRINTF "Loading cache from saved $CONF/$FNAME..." - check_saved_file - $CAT $CONF/$FNAME|$UC load_cache + # Load saved cache contents and warmup cache + filename=$1 + if [ -z "$filename" ]; then + $ECHO "Loading cache from saved $CONF/$FNAME..." + $LS -lh $CONF/$FNAME + check_saved_file $filename + $CAT $CONF/$FNAME|$UC load_cache + else + $ECHO "Loading cache from saved $filename..." + $LS -lh $filename + check_saved_file $filename + $CAT $filename|$UC load_cache + fi } reload_cache () { # Reloading and refresh existing cache and saved dump - save_cache - load_cache + filename=$1 + save_cache $filename + load_cache $filename } ############## @@ -109,27 +134,41 @@ root_check check_uc # Check command-line arguments -if [ "x$1" = "x" ]; then -# If arguments list empty, load cache by default +if [ "x$*" = "x" ]; then + # If arguments list empty,load cache by default load_cache else - arg_list=$1 + arg_list=$* # Parse command line set -- `$GETOPT sSlLrRhH: $arg_list` || { usage_note 1>&2 } - # Read arguments + # Read arguments for i in $arg_list do case $i in - -s | -S) save_cache;; - -l | -L) load_cache;; - -r | -R) reload_cache;; + -s | -S) save="1";; + -l | -L) save="0";; + -r | -R) save="2";; -h | -H | \?) usage_note;; + *) shift + file=$1 + break;; esac - break + shift done + + # Remove trailing -- + shift `$EXPR $OPTIND - 1` +fi + +if [ "$save" = "1" ]; then + save_cache $file +elif [ "$save" = "0" ]; then + load_cache $file +elif [ "$save" = "2" ]; then + reload_cache $file fi exit 0
\ No newline at end of file diff --git a/external/unbound/contrib/unbound_munin_ b/external/unbound/contrib/unbound_munin_ index 1f9f39a3e..69e9f3116 100755 --- a/external/unbound/contrib/unbound_munin_ +++ b/external/unbound/contrib/unbound_munin_ @@ -210,6 +210,7 @@ exist_config ( ) { if grep '^'$1'=' $state >/dev/null 2>&1; then echo "$mn.label $2" echo "$mn.min 0" + echo "$mn.type ABSOLUTE" fi } @@ -218,6 +219,7 @@ p_config ( ) { mn=`echo $1 | sed $ABBREV | tr . _` echo $mn.label "$2" echo $mn.min 0 + echo $mn.type $3 } if test "$1" = "config" ; then @@ -228,20 +230,21 @@ if test "$1" = "config" ; then hits) echo "graph_title Unbound DNS traffic and cache hits" echo "graph_args --base 1000 -l 0" - echo "graph_vlabel queries / second" + echo "graph_vlabel queries / \${graph_period}" + echo "graph_scale no" echo "graph_category DNS" for x in `grep "^thread[0-9][0-9]*\.num\.queries=" $state | sed -e 's/=.*//'`; do exist_config $x "queries handled by `basename $x .num.queries`" done - p_config "total.num.queries" "total queries from clients" - p_config "total.num.cachehits" "cache hits" - p_config "total.num.prefetch" "cache prefetch" - p_config "num.query.tcp" "TCP queries" - p_config "num.query.tcpout" "TCP out queries" - p_config "num.query.ipv6" "IPv6 queries" - p_config "unwanted.queries" "queries that failed acl" - p_config "unwanted.replies" "unwanted or unsolicited replies" + p_config "total.num.queries" "total queries from clients" "ABSOLUTE" + p_config "total.num.cachehits" "cache hits" "ABSOLUTE" + p_config "total.num.prefetch" "cache prefetch" "ABSOLUTE" + p_config "num.query.tcp" "TCP queries" "ABSOLUTE" + p_config "num.query.tcpout" "TCP out queries" "ABSOLUTE" + p_config "num.query.ipv6" "IPv6 queries" "ABSOLUTE" + p_config "unwanted.queries" "queries that failed acl" "ABSOLUTE" + p_config "unwanted.replies" "unwanted or unsolicited replies" "ABSOLUTE" echo "u_replies.warning $warn" echo "u_replies.critical $crit" echo "graph_info DNS queries to the recursive resolver. The unwanted replies could be innocent duplicate packets, late replies, or spoof threats." @@ -250,11 +253,12 @@ if test "$1" = "config" ; then echo "graph_title Unbound requestlist size" echo "graph_args --base 1000 -l 0" echo "graph_vlabel number of queries" + echo "graph_scale no" echo "graph_category DNS" - p_config "total.requestlist.avg" "Average size of queue on insert" - p_config "total.requestlist.max" "Max size of queue (in 5 min)" - p_config "total.requestlist.overwritten" "Number of queries replaced by new ones" - p_config "total.requestlist.exceeded" "Number of queries dropped due to lack of space" + p_config "total.requestlist.avg" "Average size of queue on insert" "GAUGE" + p_config "total.requestlist.max" "Max size of queue (in 5 min)" "GAUGE" + p_config "total.requestlist.overwritten" "Number of queries replaced by new ones" "GAUGE" + p_config "total.requestlist.exceeded" "Number of queries dropped due to lack of space" "GAUGE" echo "graph_info The queries that did not hit the cache and need recursion service take up space in the requestlist. If there are too many queries, first queries get overwritten, and at last resort dropped." ;; memory) @@ -262,128 +266,144 @@ if test "$1" = "config" ; then echo "graph_args --base 1024 -l 0" echo "graph_vlabel memory used in bytes" echo "graph_category DNS" - p_config "mem.total.sbrk" "Total memory" - p_config "mem.cache.rrset" "RRset cache memory" - p_config "mem.cache.message" "Message cache memory" - p_config "mem.mod.iterator" "Iterator module memory" - p_config "mem.mod.validator" "Validator module and key cache memory" - p_config "msg.cache.count" "msg cache count" - p_config "rrset.cache.count" "rrset cache count" - p_config "infra.cache.count" "infra cache count" - p_config "key.cache.count" "key cache count" + p_config "mem.total.sbrk" "Total memory" "GAUGE" + p_config "mem.cache.rrset" "RRset cache memory" "GAUGE" + p_config "mem.cache.message" "Message cache memory" "GAUGE" + p_config "mem.mod.iterator" "Iterator module memory" "GAUGE" + p_config "mem.mod.validator" "Validator module and key cache memory" "GAUGE" + p_config "msg.cache.count" "msg cache count" "GAUGE" + p_config "rrset.cache.count" "rrset cache count" "GAUGE" + p_config "infra.cache.count" "infra cache count" "GAUGE" + p_config "key.cache.count" "key cache count" "GAUGE" echo "graph_info The memory used by unbound." ;; by_type) echo "graph_title Unbound DNS queries by type" echo "graph_args --base 1000 -l 0" - echo "graph_vlabel queries / second" + echo "graph_vlabel queries / \${graph_period}" + echo "graph_scale no" echo "graph_category DNS" for x in `grep "^num.query.type" $state`; do nm=`echo $x | sed -e 's/=.*$//'` tp=`echo $nm | sed -e s/num.query.type.//` - p_config "$nm" "$tp" + p_config "$nm" "$tp" "ABSOLUTE" done echo "graph_info queries by DNS RR type queried for" ;; by_class) echo "graph_title Unbound DNS queries by class" echo "graph_args --base 1000 -l 0" - echo "graph_vlabel queries / second" + echo "graph_vlabel queries / \${graph_period}" + echo "graph_scale no" echo "graph_category DNS" for x in `grep "^num.query.class" $state`; do nm=`echo $x | sed -e 's/=.*$//'` tp=`echo $nm | sed -e s/num.query.class.//` - p_config "$nm" "$tp" + p_config "$nm" "$tp" "ABSOLUTE" done echo "graph_info queries by DNS RR class queried for." ;; by_opcode) echo "graph_title Unbound DNS queries by opcode" echo "graph_args --base 1000 -l 0" - echo "graph_vlabel queries / second" + echo "graph_vlabel queries / \${graph_period}" + echo "graph_scale no" echo "graph_category DNS" for x in `grep "^num.query.opcode" $state`; do nm=`echo $x | sed -e 's/=.*$//'` tp=`echo $nm | sed -e s/num.query.opcode.//` - p_config "$nm" "$tp" + p_config "$nm" "$tp" "ABSOLUTE" done echo "graph_info queries by opcode in the query packet." ;; by_rcode) echo "graph_title Unbound DNS answers by return code" echo "graph_args --base 1000 -l 0" - echo "graph_vlabel answer packets / second" + echo "graph_vlabel answer packets / \${graph_period}" + echo "graph_scale no" echo "graph_category DNS" for x in `grep "^num.answer.rcode" $state`; do nm=`echo $x | sed -e 's/=.*$//'` tp=`echo $nm | sed -e s/num.answer.rcode.//` - p_config "$nm" "$tp" + p_config "$nm" "$tp" "ABSOLUTE" done - p_config "num.answer.secure" "answer secure" - p_config "num.answer.bogus" "answer bogus" - p_config "num.rrset.bogus" "num rrsets marked bogus" - echo "graph_info answers sorted by return value. rrsets bogus is the number of rrsets marked bogus per second by the validator" + p_config "num.answer.secure" "answer secure" "ABSOLUTE" + p_config "num.answer.bogus" "answer bogus" "ABSOLUTE" + p_config "num.rrset.bogus" "num rrsets marked bogus" "ABSOLUTE" + echo "graph_info answers sorted by return value. rrsets bogus is the number of rrsets marked bogus per \${graph_period} by the validator" ;; by_flags) echo "graph_title Unbound DNS incoming queries by flags" echo "graph_args --base 1000 -l 0" - echo "graph_vlabel queries / second" + echo "graph_vlabel queries / \${graph_period}" + echo "graph_scale no" echo "graph_category DNS" - p_config "num.query.flags.QR" "QR (query reply) flag" - p_config "num.query.flags.AA" "AA (auth answer) flag" - p_config "num.query.flags.TC" "TC (truncated) flag" - p_config "num.query.flags.RD" "RD (recursion desired) flag" - p_config "num.query.flags.RA" "RA (rec avail) flag" - p_config "num.query.flags.Z" "Z (zero) flag" - p_config "num.query.flags.AD" "AD (auth data) flag" - p_config "num.query.flags.CD" "CD (check disabled) flag" - p_config "num.query.edns.present" "EDNS OPT present" - p_config "num.query.edns.DO" "DO (DNSSEC OK) flag" + p_config "num.query.flags.QR" "QR (query reply) flag" "ABSOLUTE" + p_config "num.query.flags.AA" "AA (auth answer) flag" "ABSOLUTE" + p_config "num.query.flags.TC" "TC (truncated) flag" "ABSOLUTE" + p_config "num.query.flags.RD" "RD (recursion desired) flag" "ABSOLUTE" + p_config "num.query.flags.RA" "RA (rec avail) flag" "ABSOLUTE" + p_config "num.query.flags.Z" "Z (zero) flag" "ABSOLUTE" + p_config "num.query.flags.AD" "AD (auth data) flag" "ABSOLUTE" + p_config "num.query.flags.CD" "CD (check disabled) flag" "ABSOLUTE" + p_config "num.query.edns.present" "EDNS OPT present" "ABSOLUTE" + p_config "num.query.edns.DO" "DO (DNSSEC OK) flag" "ABSOLUTE" echo "graph_info This graphs plots the flags inside incoming queries. For example, if QR, AA, TC, RA, Z flags are set, the query can be rejected. RD, AD, CD and DO are legitimately set by some software." ;; histogram) echo "graph_title Unbound DNS histogram of reply time" echo "graph_args --base 1000 -l 0" - echo "graph_vlabel queries / second" + echo "graph_vlabel queries / \${graph_period}" + echo "graph_scale no" echo "graph_category DNS" echo hcache.label "cache hits" echo hcache.min 0 + echo hcache.type ABSOLUTE echo hcache.draw AREA echo hcache.colour 999999 echo h64ms.label "0 msec - 66 msec" echo h64ms.min 0 + echo h64ms.type ABSOLUTE echo h64ms.draw STACK echo h64ms.colour 0000FF echo h128ms.label "66 msec - 131 msec" echo h128ms.min 0 + echo h128ms.type ABSOLUTE echo h128ms.colour 1F00DF echo h128ms.draw STACK echo h256ms.label "131 msec - 262 msec" echo h256ms.min 0 + echo h256ms.type ABSOLUTE echo h256ms.draw STACK echo h256ms.colour 3F00BF echo h512ms.label "262 msec - 524 msec" echo h512ms.min 0 + echo h512ms.type ABSOLUTE echo h512ms.draw STACK echo h512ms.colour 5F009F echo h1s.label "524 msec - 1 sec" echo h1s.min 0 + echo h1s.type ABSOLUTE echo h1s.draw STACK echo h1s.colour 7F007F echo h2s.label "1 sec - 2 sec" echo h2s.min 0 + echo h2s.type ABSOLUTE echo h2s.draw STACK echo h2s.colour 9F005F echo h4s.label "2 sec - 4 sec" echo h4s.min 0 + echo h4s.type ABSOLUTE echo h4s.draw STACK echo h4s.colour BF003F echo h8s.label "4 sec - 8 sec" echo h8s.min 0 + echo h8s.type ABSOLUTE echo h8s.draw STACK echo h8s.colour DF001F echo h16s.label "8 sec - ..." echo h16s.min 0 + echo h16s.type ABSOLUTE echo h16s.draw STACK echo h16s.colour FF0000 echo "graph_info Histogram of the reply times for queries." @@ -404,27 +424,21 @@ if test $value = 0 || test $value = "0.000000"; then fi elapsed="$value" -# print value for $1 / elapsed -print_qps ( ) { +# print value for $1 +print_value ( ) { mn=`echo $1 | sed $ABBREV | tr . _` get_value $1 - echo "$mn.value" `echo scale=6';' $value / $elapsed | bc ` + echo "$mn.value" $value } -# print qps if line already found in $2 -print_qps_line ( ) { +# print value if line already found in $2 +print_value_line ( ) { mn=`echo $1 | sed $ABBREV | tr . _` value="`echo $2 | sed -e 's/^.*=//'`" - echo "$mn.value" `echo scale=6';' $value / $elapsed | bc ` -} - -# print value for $1 -print_value ( ) { - mn=`echo $1 | sed $ABBREV | tr . _` - get_value $1 echo "$mn.value" $value } + case $id in hits) for x in `grep "^thread[0-9][0-9]*\.num\.queries=" $state | @@ -433,7 +447,7 @@ hits) num.query.tcpout num.query.ipv6 unwanted.queries \ unwanted.replies; do if grep "^"$x"=" $state >/dev/null 2>&1; then - print_qps $x + print_value $x fi done ;; @@ -467,38 +481,38 @@ memory) by_type) for x in `grep "^num.query.type" $state`; do nm=`echo $x | sed -e 's/=.*$//'` - print_qps_line $nm $x + print_value_line $nm $x done ;; by_class) for x in `grep "^num.query.class" $state`; do nm=`echo $x | sed -e 's/=.*$//'` - print_qps_line $nm $x + print_value_line $nm $x done ;; by_opcode) for x in `grep "^num.query.opcode" $state`; do nm=`echo $x | sed -e 's/=.*$//'` - print_qps_line $nm $x + print_value_line $nm $x done ;; by_rcode) for x in `grep "^num.answer.rcode" $state`; do nm=`echo $x | sed -e 's/=.*$//'` - print_qps_line $nm $x + print_value_line $nm $x done - print_qps "num.answer.secure" - print_qps "num.answer.bogus" - print_qps "num.rrset.bogus" + print_value "num.answer.secure" + print_value "num.answer.bogus" + print_value "num.rrset.bogus" ;; by_flags) for x in num.query.flags.QR num.query.flags.AA num.query.flags.TC num.query.flags.RD num.query.flags.RA num.query.flags.Z num.query.flags.AD num.query.flags.CD num.query.edns.present num.query.edns.DO; do - print_qps $x + print_value $x done ;; histogram) get_value total.num.cachehits - echo hcache.value `echo scale=6';' $value / $elapsed | bc ` + echo hcache.value $value r=0 for x in histogram.000000.000000.to.000000.000001 \ histogram.000000.000001.to.000000.000002 \ @@ -520,21 +534,21 @@ histogram) get_value $x r=`expr $r + $value` done - echo h64ms.value `echo scale=6';' $r / $elapsed | bc ` + echo h64ms.value $r get_value histogram.000000.065536.to.000000.131072 - echo h128ms.value `echo scale=6';' $value / $elapsed | bc ` + echo h128ms.value $value get_value histogram.000000.131072.to.000000.262144 - echo h256ms.value `echo scale=6';' $value / $elapsed | bc ` + echo h256ms.value $value get_value histogram.000000.262144.to.000000.524288 - echo h512ms.value `echo scale=6';' $value / $elapsed | bc ` + echo h512ms.value $value get_value histogram.000000.524288.to.000001.000000 - echo h1s.value `echo scale=6';' $value / $elapsed | bc ` + echo h1s.value $value get_value histogram.000001.000000.to.000002.000000 - echo h2s.value `echo scale=6';' $value / $elapsed | bc ` + echo h2s.value $value get_value histogram.000002.000000.to.000004.000000 - echo h4s.value `echo scale=6';' $value / $elapsed | bc ` + echo h4s.value $value get_value histogram.000004.000000.to.000008.000000 - echo h8s.value `echo scale=6';' $value / $elapsed | bc ` + echo h8s.value $value r=0 for x in histogram.000008.000000.to.000016.000000 \ histogram.000016.000000.to.000032.000000 \ @@ -555,6 +569,6 @@ histogram) get_value $x r=`expr $r + $value` done - echo h16s.value `echo scale=6';' $r / $elapsed | bc ` + echo h16s.value $r ;; esac diff --git a/external/unbound/contrib/unbound_smf22.tar.gz b/external/unbound/contrib/unbound_smf22.tar.gz Binary files differnew file mode 100644 index 000000000..e4c51c3dc --- /dev/null +++ b/external/unbound/contrib/unbound_smf22.tar.gz diff --git a/external/unbound/contrib/warmup.cmd b/external/unbound/contrib/warmup.cmd index d7df01827..b3895a86f 100644 --- a/external/unbound/contrib/warmup.cmd +++ b/external/unbound/contrib/warmup.cmd @@ -1,68 +1,153 @@ @echo off
rem --------------------------------------------------------------
-rem -- Warm up DNS cache script by your own MRU domains
+rem -- Warm up DNS cache script by your own MRU domains or from
+rem -- file when it specified as script argument.
rem --
-rem -- Version 1.0
+rem -- Version 1.1
rem -- By Yuri Voinov (c) 2014
rem --------------------------------------------------------------
+rem DNS host address
+set address="127.0.0.1"
+
rem Check dig installed
for /f "delims=" %%a in ('where dig') do @set dig=%%a
if /I "%dig%"=="" echo Dig not found. If installed, add path to PATH environment variable. & exit 1
echo Dig found: %dig%
-echo Warming up cache by MRU domains...
-rem dig -f my_domains 1>nul 2>nul
-rem echo Done.
+set arg=%1%
+
+if defined %arg% (goto builtin) else (goto from_file)
+:builtin
+echo Warming up cache by MRU domains...
for %%a in (
-mail.ru
-my.mail.ru
-mra.mail.ru
+2gis.ru
+admir.kz
+adobe.com
agent.mail.ru
-news.mail.ru
-icq.com
-lenta.ru
-gazeta.ru
-peerbet.ru
-www.opennet.ru
-snob.ru
+aimp.ru
+akamai.com
+akamai.net
+almaty.tele2.kz
+aol.com
+apple.com
+arin.com
artlebedev.ru
-mail.google.com
-translate.google.com
-drive.google.com
-google.com
-google.kz
-drive.google.com
+auto.mail.ru
+beeline.kz
+bing.com
blogspot.com
+comodo.com
+dnscrypt.org
+drive.google.com
+drive.mail.ru
+facebook.com
farmanager.com
+fb.com
+firefox.com
forum.farmanager.com
+gazeta.ru
+getsharex.com
+gismeteo.ru
+google.com
+google.kz
+google.ru
+googlevideo.com
+goto.kz
+iana.org
+icq.com
+imap.mail.ru
+instagram.com
+intel.com
+irr.kz
+java.com
+kaspersky.com
+kaspersky.ru
+kcell.kz
+krisha.kz
+lady.mail.ru
+lenta.ru
+libreoffice.org
+linkedin.com
+livejournal.com
+mail.google.com
+mail.ru
+microsoft.com
+mozilla.org
+mra.mail.ru
+munin-monitoring.org
+my.mail.ru
+news.bbcimg.co.uk
+news.mail.ru
+newsimg.bbc.net.uk
+nvidia.com
+odnoklassniki.ru
+ok.ru
+opencsw.org
+opendns.com
+opendns.org
+opennet.ru
+opera.com
+oracle.com
+peerbet.ru
+piriform.com
plugring.farmanager.com
+privoxy.org
+qip.ru
+raidcall.com
+rambler.ru
+reddit.com
+ru.wikipedia.org
+shallalist.de
+skype.com
+snob.ru
+squid-cache.org
+squidclamav.darold.net
+squidguard.org
+ssl.comodo.com
+ssl.verisign.com
symantec.com
symantecliveupdate.com
-shalla.de
-torstatus.blutmagie.de
+tele2.kz
+tengrinews.kz
+thunderbird.com
torproject.org
-dnscrypt.org
+torstatus.blutmagie.de
+translate.google.com
unbound.net
-getsharex.com
-skype.com
+verisign.com
+vk.com
+vk.me
+vk.ru
+vkontakte.com
+vkontakte.ru
vlc.org
-aimp.ru
-mozilla.org
-libreoffice.org
-piriform.com
-raidcall.com
-nvidia.com
-intel.com
-microsoft.com
+watsapp.net
+weather.mail.ru
windowsupdate.com
-ru.wikipedia.org
+www.baidu.com
www.bbc.co.uk
-tengrinews.kz
-) do "%dig%" %%a 1>nul 2>nul
+www.internic.net
+www.opennet.ru
+www.topgear.com
+ya.ru
+yahoo.com
+yandex.com
+yandex.ru
+youtube.com
+ytimg.com
+) do "%dig%" %%a @%address% 1>nul 2>nul
+goto end
+:from_file
+echo Warming up cache from %1% file...
+%dig% -f %arg% @%address% 1>nul 2>nul
+
+:end
echo Saving cache...
-unbound_cache.cmd -s
+if exist unbound_cache.cmd unbound_cache.cmd -s
echo Done.
+
+exit 0
\ No newline at end of file diff --git a/external/unbound/contrib/warmup.sh b/external/unbound/contrib/warmup.sh index 820f019d7..b4d9135a6 100644 --- a/external/unbound/contrib/warmup.sh +++ b/external/unbound/contrib/warmup.sh @@ -1,65 +1,150 @@ #!/bin/sh # -------------------------------------------------------------- -# -- Warm up DNS cache script by your own MRU domains +# -- Warm up DNS cache script by your own MRU domains or from +# -- file when it specified as script argument. # -- -# -- Version 1.0 +# -- Version 1.1 # -- By Yuri Voinov (c) 2014 # -------------------------------------------------------------- +# Default DNS host address +address="127.0.0.1" + +cat=`which cat` dig=`which dig` +if [ -z "$1" ]; then echo "Warming up cache by MRU domains..." -$dig -f - >/dev/null 2>&1 <<EOT -mail.ru -my.mail.ru -mra.mail.ru +$dig -f - @$address >/dev/null 2>&1 <<EOT +2gis.ru +admir.kz +adobe.com agent.mail.ru -news.mail.ru -icq.com -lenta.ru -gazeta.ru -peerbet.ru -www.opennet.ru -snob.ru +aimp.ru +akamai.com +akamai.net +almaty.tele2.kz +aol.com +apple.com +arin.com artlebedev.ru -mail.google.com -translate.google.com -drive.google.com -google.com -google.kz -drive.google.com +auto.mail.ru +beeline.kz +bing.com blogspot.com +clamav.net +comodo.com +dnscrypt.org +drive.google.com +drive.mail.ru +facebook.com farmanager.com +fb.com +firefox.com forum.farmanager.com +gazeta.ru +getsharex.com +gismeteo.ru +google.com +google.kz +google.ru +googlevideo.com +goto.kz +iana.org +icq.com +imap.mail.ru +instagram.com +instagram.com +intel.com +irr.kz +java.com +kaspersky.com +kaspersky.ru +kcell.kz +krisha.kz +lady.mail.ru +lenta.ru +libreoffice.org +linkedin.com +livejournal.com +mail.google.com +mail.ru +microsoft.com +mozilla.org +mra.mail.ru +munin-monitoring.org +my.mail.ru +news.bbcimg.co.uk +news.mail.ru +newsimg.bbc.net.uk +nvidia.com +odnoklassniki.ru +ok.ru +opencsw.org +opendns.com +opendns.org +opennet.ru +opera.com +oracle.com +peerbet.ru +piriform.com plugring.farmanager.com +privoxy.org +qip.ru +raidcall.com +rambler.ru +reddit.com +ru.wikipedia.org +shallalist.de +skype.com +snob.ru +squid-cache.org +squidclamav.darold.net +squidguard.org +ssl.comodo.com +ssl.verisign.com symantec.com symantecliveupdate.com -shalla.de -torstatus.blutmagie.de +tele2.kz +tengrinews.kz +thunderbird.com torproject.org -dnscrypt.org +torstatus.blutmagie.de +translate.google.com unbound.net -getsharex.com -skype.com +verisign.com +vk.com +vk.me +vk.ru +vkontakte.com +vkontakte.ru vlc.org -aimp.ru -mozilla.org -libreoffice.org -piriform.com -raidcall.com -nvidia.com -intel.com -microsoft.com +watsapp.net +weather.mail.ru windowsupdate.com -ru.wikipedia.org +www.baidu.com www.bbc.co.uk -tengrinews.kz +www.internic.net +www.opennet.ru +www.topgear.com +ya.ru +yahoo.com +yandex.com +yandex.ru +youtube.com +ytimg.com EOT +else + echo "Warming up cache from $1 file..." + $cat $1 | $dig -f - @$address >/dev/null 2>&1 +fi + echo "Done." echo "Saving cache..." -/usr/local/bin/unbound_cache.sh -s +script=`which unbound_cache.sh` +[ -f "$script" ] && $script -s echo "Done." exit 0 diff --git a/external/unbound/daemon/cachedump.c b/external/unbound/daemon/cachedump.c index 20a46ae4d..4b0a583a6 100644 --- a/external/unbound/daemon/cachedump.c +++ b/external/unbound/daemon/cachedump.c @@ -56,9 +56,9 @@ #include "iterator/iter_utils.h" #include "iterator/iter_fwd.h" #include "iterator/iter_hints.h" -#include "ldns/sbuffer.h" -#include "ldns/wire2str.h" -#include "ldns/str2wire.h" +#include "sldns/sbuffer.h" +#include "sldns/wire2str.h" +#include "sldns/str2wire.h" /** dump one rrset zonefile line */ static int @@ -223,6 +223,8 @@ copy_msg(struct regional* region, struct lruhash_entry* e, struct query_info** k, struct reply_info** d) { struct reply_info* rep = (struct reply_info*)e->data; + if(rep->rrset_count > RR_COUNT_MAX) + return 0; /* to protect against integer overflow */ *d = (struct reply_info*)regional_alloc_init(region, e->data, sizeof(struct reply_info) + sizeof(struct rrset_ref) * (rep->rrset_count-1) + @@ -470,6 +472,10 @@ load_rrset(SSL* ssl, sldns_buffer* buf, struct worker* worker) log_warn("bad rrset without contents"); return 0; } + if(rr_count > RR_COUNT_MAX || rrsig_count > RR_COUNT_MAX) { + log_warn("bad rrset with too many rrs"); + return 0; + } d->count = (size_t)rr_count; d->rrsig_count = (size_t)rrsig_count; d->security = (enum sec_status)security; @@ -646,6 +652,10 @@ load_msg(SSL* ssl, sldns_buffer* buf, struct worker* worker) rep.ttl = (time_t)ttl; rep.prefetch_ttl = PREFETCH_TTL_CALC(rep.ttl); rep.security = (enum sec_status)security; + if(an > RR_COUNT_MAX || ns > RR_COUNT_MAX || ar > RR_COUNT_MAX) { + log_warn("error too many rrsets"); + return 0; /* protect against integer overflow in alloc */ + } rep.an_numrrsets = (size_t)an; rep.ns_numrrsets = (size_t)ns; rep.ar_numrrsets = (size_t)ar; diff --git a/external/unbound/daemon/daemon.c b/external/unbound/daemon/daemon.c index f693a0285..0cd37ae82 100644 --- a/external/unbound/daemon/daemon.c +++ b/external/unbound/daemon/daemon.c @@ -84,7 +84,7 @@ #include "util/random.h" #include "util/tube.h" #include "util/net_help.h" -#include "ldns/keyraw.h" +#include "sldns/keyraw.h" #include <signal.h> /** How many quit requests happened. */ diff --git a/external/unbound/daemon/remote.c b/external/unbound/daemon/remote.c index ff3d769d4..24008bf17 100644 --- a/external/unbound/daemon/remote.c +++ b/external/unbound/daemon/remote.c @@ -46,6 +46,10 @@ #ifdef HAVE_OPENSSL_ERR_H #include <openssl/err.h> #endif +#ifndef HEADER_DH_H +#include <openssl/dh.h> +#endif + #include <ctype.h> #include "daemon/remote.h" #include "daemon/worker.h" @@ -74,14 +78,17 @@ #include "iterator/iter_delegpt.h" #include "services/outbound_list.h" #include "services/outside_network.h" -#include "ldns/str2wire.h" -#include "ldns/parseutil.h" -#include "ldns/wire2str.h" -#include "ldns/sbuffer.h" +#include "sldns/str2wire.h" +#include "sldns/parseutil.h" +#include "sldns/wire2str.h" +#include "sldns/sbuffer.h" #ifdef HAVE_SYS_TYPES_H # include <sys/types.h> #endif +#ifdef HAVE_SYS_STAT_H +#include <sys/stat.h> +#endif #ifdef HAVE_NETDB_H #include <netdb.h> #endif @@ -131,6 +138,41 @@ timeval_divide(struct timeval* avg, const struct timeval* sum, size_t d) #endif } +/* + * The following function was generated using the openssl utility, using + * the command : "openssl dhparam -dsaparam -C 512" + */ +#ifndef S_SPLINT_S +DH *get_dh512() +{ + static unsigned char dh512_p[]={ + 0xC9,0xD7,0x05,0xDA,0x5F,0xAB,0x14,0xE8,0x11,0x56,0x77,0x85, + 0xB1,0x24,0x2C,0x95,0x60,0xEA,0xE2,0x10,0x6F,0x0F,0x84,0xEC, + 0xF4,0x45,0xE8,0x90,0x7A,0xA7,0x03,0xFF,0x5B,0x88,0x53,0xDE, + 0xC4,0xDE,0xBC,0x42,0x78,0x71,0x23,0x7E,0x24,0xA5,0x5E,0x4E, + 0xEF,0x6F,0xFF,0x5F,0xAF,0xBE,0x8A,0x77,0x62,0xB4,0x65,0x82, + 0x7E,0xC9,0xED,0x2F, + }; + static unsigned char dh512_g[]={ + 0x8D,0x3A,0x52,0xBC,0x8A,0x71,0x94,0x33,0x2F,0xE1,0xE8,0x4C, + 0x73,0x47,0x03,0x4E,0x7D,0x40,0xE5,0x84,0xA0,0xB5,0x6D,0x10, + 0x6F,0x90,0x43,0x05,0x1A,0xF9,0x0B,0x6A,0xD1,0x2A,0x9C,0x25, + 0x0A,0xB9,0xD1,0x14,0xDC,0x35,0x1C,0x48,0x7C,0xC6,0x0C,0x6D, + 0x32,0x1D,0xD3,0xC8,0x10,0xA8,0x82,0x14,0xA2,0x1C,0xF4,0x53, + 0x23,0x3B,0x1C,0xB9, + }; + DH *dh; + + if ((dh=DH_new()) == NULL) return(NULL); + dh->p=BN_bin2bn(dh512_p,sizeof(dh512_p),NULL); + dh->g=BN_bin2bn(dh512_g,sizeof(dh512_g),NULL); + if ((dh->p == NULL) || (dh->g == NULL)) + { DH_free(dh); return(NULL); } + dh->length = 160; + return(dh); +} +#endif /* SPLINT */ + struct daemon_remote* daemon_remote_create(struct config_file* cfg) { @@ -165,6 +207,24 @@ daemon_remote_create(struct config_file* cfg) daemon_remote_delete(rc); return NULL; } + + if (cfg->remote_control_use_cert == 0) { + /* No certificates are requested */ + if(!SSL_CTX_set_cipher_list(rc->ctx, "aNULL")) { + log_crypto_err("Failed to set aNULL cipher list"); + return NULL; + } + + /* Since we have no certificates and hence no source of + * DH params, let's generate and set them + */ + if(!SSL_CTX_set_tmp_dh(rc->ctx,get_dh512())) { + log_crypto_err("Wanted to set DH param, but failed"); + return NULL; + } + return rc; + } + rc->use_cert = 1; s_cert = fname_after_chroot(cfg->server_cert_file, cfg, 1); s_key = fname_after_chroot(cfg->server_key_file, cfg, 1); if(!s_cert || !s_key) { @@ -241,10 +301,12 @@ void daemon_remote_delete(struct daemon_remote* rc) * @param nr: port nr * @param list: list head * @param noproto_is_err: if lack of protocol support is an error. + * @param cfg: config with username for chown of unix-sockets. * @return false on failure. */ static int -add_open(const char* ip, int nr, struct listen_port** list, int noproto_is_err) +add_open(const char* ip, int nr, struct listen_port** list, int noproto_is_err, + struct config_file* cfg) { struct addrinfo hints; struct addrinfo* res; @@ -255,29 +317,52 @@ add_open(const char* ip, int nr, struct listen_port** list, int noproto_is_err) snprintf(port, sizeof(port), "%d", nr); port[sizeof(port)-1]=0; memset(&hints, 0, sizeof(hints)); - hints.ai_socktype = SOCK_STREAM; - hints.ai_flags = AI_PASSIVE | AI_NUMERICHOST; - if((r = getaddrinfo(ip, port, &hints, &res)) != 0 || !res) { -#ifdef USE_WINSOCK - if(!noproto_is_err && r == EAI_NONAME) { - /* tried to lookup the address as name */ - return 1; /* return success, but do nothing */ + + if(ip[0] == '/') { + /* This looks like a local socket */ + fd = create_local_accept_sock(ip, &noproto); + /* + * Change socket ownership and permissions so users other + * than root can access it provided they are in the same + * group as the user we run as. + */ + if(fd != -1) { +#ifdef HAVE_CHOWN + if (cfg->username && cfg->username[0] && + cfg_uid != (uid_t)-1) + chown(ip, cfg_uid, cfg_gid); + chmod(ip, (mode_t)(S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP)); +#else + (void)cfg; +#endif } + } else { + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_PASSIVE | AI_NUMERICHOST; + if((r = getaddrinfo(ip, port, &hints, &res)) != 0 || !res) { +#ifdef USE_WINSOCK + if(!noproto_is_err && r == EAI_NONAME) { + /* tried to lookup the address as name */ + return 1; /* return success, but do nothing */ + } #endif /* USE_WINSOCK */ - log_err("control interface %s:%s getaddrinfo: %s %s", - ip?ip:"default", port, gai_strerror(r), + log_err("control interface %s:%s getaddrinfo: %s %s", + ip?ip:"default", port, gai_strerror(r), #ifdef EAI_SYSTEM - r==EAI_SYSTEM?(char*)strerror(errno):"" + r==EAI_SYSTEM?(char*)strerror(errno):"" #else - "" + "" #endif ); - return 0; + return 0; + } + + /* open fd */ + fd = create_tcp_accept_sock(res, 1, &noproto, 0, + cfg->ip_transparent); + freeaddrinfo(res); } - /* open fd */ - fd = create_tcp_accept_sock(res, 1, &noproto, 0); - freeaddrinfo(res); if(fd == -1 && noproto) { if(!noproto_is_err) return 1; /* return success, but do nothing */ @@ -314,7 +399,7 @@ struct listen_port* daemon_remote_open_ports(struct config_file* cfg) if(cfg->control_ifs) { struct config_strlist* p; for(p = cfg->control_ifs; p; p = p->next) { - if(!add_open(p->str, cfg->control_port, &l, 1)) { + if(!add_open(p->str, cfg->control_port, &l, 1, cfg)) { listening_ports_free(l); return NULL; } @@ -322,12 +407,12 @@ struct listen_port* daemon_remote_open_ports(struct config_file* cfg) } else { /* defaults */ if(cfg->do_ip6 && - !add_open("::1", cfg->control_port, &l, 0)) { + !add_open("::1", cfg->control_port, &l, 0, cfg)) { listening_ports_free(l); return NULL; } if(cfg->do_ip4 && - !add_open("127.0.0.1", cfg->control_port, &l, 1)) { + !add_open("127.0.0.1", cfg->control_port, &l, 1, cfg)) { listening_ports_free(l); return NULL; } @@ -641,6 +726,8 @@ print_stats(SSL* ssl, const char* nm, struct stats_info* s) (long long)avg.tv_sec, (int)avg.tv_usec)) return 0; if(!ssl_printf(ssl, "%s.recursion.time.median"SQ"%g\n", nm, s->mesh_time_median)) return 0; + if(!ssl_printf(ssl, "%s.tcpusage"SQ"%lu\n", nm, + (unsigned long)s->svr.tcp_accept_usage)) return 0; return 1; } @@ -1990,7 +2077,7 @@ dump_infra_host(struct lruhash_entry* e, void* arg) d->rtt.srtt, d->rtt.rttvar, rtt_notimeout(&d->rtt), d->rtt.rto, d->timeout_A, d->timeout_AAAA, d->timeout_other, (int)d->edns_lame_known, (int)d->edns_version, - (int)(a->now<d->probedelay?d->probedelay-a->now:0), + (int)(a->now<d->probedelay?(d->probedelay - a->now):0), (int)d->isdnsseclame, (int)d->rec_lame, (int)d->lame_type_A, (int)d->lame_other)) { a->ssl_failed = 1; @@ -2434,7 +2521,9 @@ int remote_control_callback(struct comm_point* c, void* arg, int err, s->shake_state = rc_none; /* once handshake has completed, check authentication */ - if(SSL_get_verify_result(s->ssl) == X509_V_OK) { + if (!rc->use_cert) { + verbose(VERB_ALGO, "unauthenticated remote control connection"); + } else if(SSL_get_verify_result(s->ssl) == X509_V_OK) { X509* x = SSL_get_peer_certificate(s->ssl); if(!x) { verbose(VERB_DETAIL, "remote control connection " diff --git a/external/unbound/daemon/remote.h b/external/unbound/daemon/remote.h index cc670b701..b25bfb1af 100644 --- a/external/unbound/daemon/remote.h +++ b/external/unbound/daemon/remote.h @@ -89,6 +89,8 @@ struct daemon_remote { struct worker* worker; /** commpoints for accepting remote control connections */ struct listen_list* accept_list; + /* if certificates are used */ + int use_cert; /** number of active commpoints that are handling remote control */ int active; /** max active commpoints */ diff --git a/external/unbound/daemon/stats.c b/external/unbound/daemon/stats.c index d3f41de03..838cf05ae 100644 --- a/external/unbound/daemon/stats.c +++ b/external/unbound/daemon/stats.c @@ -50,12 +50,13 @@ #include "daemon/daemon.h" #include "services/mesh.h" #include "services/outside_network.h" +#include "services/listen_dnsport.h" #include "util/config_file.h" #include "util/tube.h" #include "util/timehist.h" #include "util/net_help.h" #include "validator/validator.h" -#include "ldns/sbuffer.h" +#include "sldns/sbuffer.h" #include "services/cache/rrset.h" #include "services/cache/infra.h" #include "validator/val_kcache.h" @@ -140,6 +141,7 @@ void server_stats_compile(struct worker* worker, struct stats_info* s, int reset) { int i; + struct listen_list* lp; s->svr = worker->stats; s->mesh_num_states = worker->env.mesh->all.count; @@ -174,6 +176,13 @@ server_stats_compile(struct worker* worker, struct stats_info* s, int reset) s->svr.key_cache_count = count_slabhash_entries(worker->env.key_cache->slab); else s->svr.key_cache_count = 0; + /* get tcp accept usage */ + s->svr.tcp_accept_usage = 0; + for(lp = worker->front->cps; lp; lp = lp->next) { + if(lp->com->type == comm_tcp_accept) + s->svr.tcp_accept_usage += lp->com->cur_tcp_count; + } + if(reset && !worker->env.cfg->stat_cumulative) { worker_stats_clear(worker); } @@ -247,6 +256,7 @@ void server_stats_add(struct stats_info* total, struct stats_info* a) total->svr.rrset_bogus += a->svr.rrset_bogus; total->svr.unwanted_replies += a->svr.unwanted_replies; total->svr.unwanted_queries += a->svr.unwanted_queries; + total->svr.tcp_accept_usage += a->svr.tcp_accept_usage; for(i=0; i<STATS_QTYPE_NUM; i++) total->svr.qtype[i] += a->svr.qtype[i]; for(i=0; i<STATS_QCLASS_NUM; i++) diff --git a/external/unbound/daemon/stats.h b/external/unbound/daemon/stats.h index 5ea00a0da..6985446ce 100644 --- a/external/unbound/daemon/stats.h +++ b/external/unbound/daemon/stats.h @@ -129,6 +129,8 @@ struct server_stats { size_t unwanted_replies; /** unwanted traffic received on client-facing ports */ size_t unwanted_queries; + /** usage of tcp accept list */ + size_t tcp_accept_usage; /** histogram data exported to array * if the array is the same size, no data is lost, and diff --git a/external/unbound/daemon/unbound.c b/external/unbound/daemon/unbound.c index 5ded5a964..8e07c3895 100644 --- a/external/unbound/daemon/unbound.c +++ b/external/unbound/daemon/unbound.c @@ -443,18 +443,10 @@ perform_setup(struct daemon* daemon, struct config_file* cfg, int debug_mode, { #ifdef HAVE_GETPWNAM struct passwd *pwd = NULL; - uid_t uid; - gid_t gid; - /* initialize, but not to 0 (root) */ - memset(&uid, 112, sizeof(uid)); - memset(&gid, 112, sizeof(gid)); - log_assert(cfg); if(cfg->username && cfg->username[0]) { if((pwd = getpwnam(cfg->username)) == NULL) fatal_exit("user '%s' does not exist.", cfg->username); - uid = pwd->pw_uid; - gid = pwd->pw_gid; /* endpwent below, in case we need pwd for setusercontext */ } #endif @@ -511,33 +503,28 @@ perform_setup(struct daemon* daemon, struct config_file* cfg, int debug_mode, #ifdef HAVE_KILL if(cfg->pidfile && cfg->pidfile[0]) { writepid(daemon->pidfile, getpid()); - if(!(cfg->chrootdir && cfg->chrootdir[0]) || - (cfg->chrootdir && cfg->chrootdir[0] && - strncmp(daemon->pidfile, cfg->chrootdir, - strlen(cfg->chrootdir))==0)) { - /* delete of pidfile could potentially work, - * chown to get permissions */ - if(cfg->username && cfg->username[0]) { - if(chown(daemon->pidfile, uid, gid) == -1) { + if(cfg->username && cfg->username[0] && cfg_uid != (uid_t)-1) { +# ifdef HAVE_CHOWN + if(chown(daemon->pidfile, cfg_uid, cfg_gid) == -1) { log_err("cannot chown %u.%u %s: %s", - (unsigned)uid, (unsigned)gid, + (unsigned)cfg_uid, (unsigned)cfg_gid, daemon->pidfile, strerror(errno)); - } } +# endif /* HAVE_CHOWN */ } } #else (void)daemon; -#endif +#endif /* HAVE_KILL */ /* Set user context */ #ifdef HAVE_GETPWNAM - if(cfg->username && cfg->username[0]) { + if(cfg->username && cfg->username[0] && cfg_uid != (uid_t)-1) { #ifdef HAVE_SETUSERCONTEXT /* setusercontext does initgroups, setuid, setgid, and * also resource limits from login config, but we * still call setresuid, setresgid to be sure to set all uid*/ - if(setusercontext(NULL, pwd, uid, (unsigned) + if(setusercontext(NULL, pwd, cfg_uid, (unsigned) LOGIN_SETALL & ~LOGIN_SETUSER & ~LOGIN_SETGROUP) != 0) log_warn("unable to setusercontext %s: %s", cfg->username, strerror(errno)); @@ -599,29 +586,29 @@ perform_setup(struct daemon* daemon, struct config_file* cfg, int debug_mode, /* drop permissions after chroot, getpwnam, pidfile, syslog done*/ #ifdef HAVE_GETPWNAM - if(cfg->username && cfg->username[0]) { + if(cfg->username && cfg->username[0] && cfg_uid != (uid_t)-1) { # ifdef HAVE_INITGROUPS - if(initgroups(cfg->username, gid) != 0) + if(initgroups(cfg->username, cfg_gid) != 0) log_warn("unable to initgroups %s: %s", cfg->username, strerror(errno)); # endif /* HAVE_INITGROUPS */ endpwent(); #ifdef HAVE_SETRESGID - if(setresgid(gid,gid,gid) != 0) + if(setresgid(cfg_gid,cfg_gid,cfg_gid) != 0) #elif defined(HAVE_SETREGID) && !defined(DARWIN_BROKEN_SETREUID) - if(setregid(gid,gid) != 0) + if(setregid(cfg_gid,cfg_gid) != 0) #else /* use setgid */ - if(setgid(gid) != 0) + if(setgid(cfg_gid) != 0) #endif /* HAVE_SETRESGID */ fatal_exit("unable to set group id of %s: %s", cfg->username, strerror(errno)); #ifdef HAVE_SETRESUID - if(setresuid(uid,uid,uid) != 0) + if(setresuid(cfg_uid,cfg_uid,cfg_uid) != 0) #elif defined(HAVE_SETREUID) && !defined(DARWIN_BROKEN_SETREUID) - if(setreuid(uid,uid) != 0) + if(setreuid(cfg_uid,cfg_uid) != 0) #else /* use setuid */ - if(setuid(uid) != 0) + if(setuid(cfg_uid) != 0) #endif /* HAVE_SETRESUID */ fatal_exit("unable to set user id of %s: %s", cfg->username, strerror(errno)); @@ -666,6 +653,8 @@ run_daemon(const char* cfgfile, int cmdline_verbose, int debug_mode) log_warn("Continuing with default config settings"); } apply_settings(daemon, cfg, cmdline_verbose, debug_mode); + if(!done_setup) + config_lookup_uid(cfg); /* prepare */ if(!daemon_open_shared_ports(daemon)) diff --git a/external/unbound/daemon/worker.c b/external/unbound/daemon/worker.c index 59ae9dfce..93481354f 100644 --- a/external/unbound/daemon/worker.c +++ b/external/unbound/daemon/worker.c @@ -71,7 +71,7 @@ #include "validator/val_anchor.h" #include "libunbound/context.h" #include "libunbound/libworker.h" -#include "ldns/sbuffer.h" +#include "sldns/sbuffer.h" #ifdef HAVE_SYS_TYPES_H # include <sys/types.h> @@ -900,7 +900,7 @@ worker_handle_request(struct comm_point* c, void* arg, int error, goto send_reply; } if(local_zones_answer(worker->daemon->local_zones, &qinfo, &edns, - c->buffer, worker->scratchpad)) { + c->buffer, worker->scratchpad, repinfo)) { regional_free_all(worker->scratchpad); if(sldns_buffer_limit(c->buffer) == 0) { comm_point_drop_reply(repinfo); diff --git a/external/unbound/dns64/dns64.c b/external/unbound/dns64/dns64.c index eaaa26f7c..63cc8084e 100644 --- a/external/unbound/dns64/dns64.c +++ b/external/unbound/dns64/dns64.c @@ -590,6 +590,10 @@ dns64_synth_aaaa_data(const struct ub_packed_rrset_key* fk, * for the RRs themselves. Each RR has a length, TTL, pointer to wireformat * data, 2 bytes of data length, and 16 bytes of IPv6 address. */ + if(fd->count > RR_COUNT_MAX) { + *dd_out = NULL; + return; /* integer overflow protection in alloc */ + } if (!(dd = *dd_out = regional_alloc(region, sizeof(struct packed_rrset_data) + fd->count * (sizeof(size_t) + sizeof(time_t) + @@ -713,6 +717,8 @@ dns64_adjust_a(int id, struct module_qstate* super, struct module_qstate* qstate if(i<rep->an_numrrsets && fk->rk.type == htons(LDNS_RR_TYPE_A)) { /* also sets dk->entry.hash */ dns64_synth_aaaa_data(fk, fd, dk, &dd, super->region, dns64_env); + if(!dd) + return; /* Delete negative AAAA record from cache stored by * the iterator module */ rrset_cache_remove(super->env->rrset_cache, dk->rk.dname, diff --git a/external/unbound/dnstap/dnstap.c b/external/unbound/dnstap/dnstap.c index b2dc053bd..b62dc5b8c 100644 --- a/external/unbound/dnstap/dnstap.c +++ b/external/unbound/dnstap/dnstap.c @@ -39,7 +39,7 @@ #include "config.h" #include <string.h> #include <sys/time.h> -#include "ldns/sbuffer.h" +#include "sldns/sbuffer.h" #include "util/config_file.h" #include "util/net_help.h" #include "util/netevent.h" diff --git a/external/unbound/doc/Changelog b/external/unbound/doc/Changelog index c82ae8ade..31f84c445 100644 --- a/external/unbound/doc/Changelog +++ b/external/unbound/doc/Changelog @@ -1,3 +1,173 @@ +26 March 2015: Wouter + - remote.c probedelay line is easier to read. + - rename ldns subdirectory to sldns to avoid name collision. + +25 March 2015: Wouter + - Fix #657: libunbound(3) recommends deprecated + CRYPTO_set_id_callback. + - If unknown trust anchor algorithm, and libressl is used, error + message encourages upgrade of the libressl package. + +23 March 2015: Wouter + - Fix segfault on user not found at startup (from Maciej Soltysiak). + +20 March 2015: Wouter + - Fixed to add integer overflow checks on allocation (defense in depth). + +19 March 2015: Wouter + - Add ip-transparent config option for bind to non-local addresses. + +17 March 2015: Wouter + - Use reallocarray for integer overflow protection, patch submitted + by Loganaden Velvindron. + +16 March 2015: Wouter + - Fixup compile on cygwin, more portable openssl thread id. + +12 March 2015: Wouter + - Updated default keylength in unbound-control-setup to 3k. + +10 March 2015: Wouter + - Fix lintian warning in unbound-checkconf man page (from Andreas + Schulze). + - print svnroot when building windows dist. + - iana portlist update. + - Fix warning on sign compare in getentropy_linux. + +9 March 2015: Wouter + - Fix #644: harden-algo-downgrade option, if turned off, fixes the + reported excessive validation failure when multiple algorithms + are present. It allows the weakest algorithm to validate the zone. + - iana portlist update. + +5 March 2015: Wouter + - contrib/unbound_smf22.tar.gz: Solaris SMF installation/removal + scripts. Contributed by Yuri Voinov. + - Document that incoming-num-tcp increase is good for large servers. + - stats reports tcp usage, of incoming-num-tcp buffers. + +4 March 2015: Wouter + - Patch from Brad Smith that syncs compat/getentropy_linux with + OpenBSD's version (2015-03-04). + - 0x20 fallback improved: servfail responses do not count as missing + comparisons (except if all responses are errors), + inability to find nameservers does not fail equality comparisons, + many nameservers does not try to compare more than max-sent-count, + parse failures start 0x20 fallback procedure. + - store caps_response with best response in case downgrade response + happens to be the last one. + - Document windows 8 tests. + +3 March 2015: Wouter + - tag 1.5.3rc1 + [ This became 1.5.3 on 10 March, trunk is 1.5.4 in development ] + +2 March 2015: Wouter + - iana portlist update. + +20 February 2015: Wouter + - Use the getrandom syscall introduced in Linux 3.17 (from Heiner + Kallweit). + - Fix #645 Portability to Solaris 10, use AF_LOCAL. + - Fix #646 Portability to Solaris, -lrt for getentropy_solaris. + - Fix #647 crash in 1.5.2 because pwd.db no longer accessible after + reload. + +19 February 2015: Wouter + - 1.5.2 release tag. + - svn trunk contains 1.5.3 under development. + +13 February 2015: Wouter + - Fix #643: doc/example.conf.in: unnecessary whitespace. + +12 February 2015: Wouter + - tag 1.5.2rc1 + +11 February 2015: Wouter + - iana portlist update. + +10 February 2015: Wouter + - Fix scrubber with harden-glue turned off to reject NS (and other + not-address) records. + +9 February 2015: Wouter + - Fix validation failure in case upstream forwarder (ISC BIND) does + not have the same trust anchors and decides to insert unsigned NS + record in authority section. + +2 February 2015: Wouter + - infra-cache-min-rtt patch from Florian Riehm, for expected long + uplink roundtrip times. + +30 January 2015: Wouter + - Fix 0x20 capsforid fallback to omit gratuitous NS and additional + section changes. + - Portability fix for Solaris ('sun' is not usable for a variable). + +29 January 2015: Wouter + - Fix pyunbound byte string representation for python3. + +26 January 2015: Wouter + - Fix unintended use of gcc extension for incomplete enum types, + compile with pedantic c99 compliance (from Daniel Dickman). + +23 January 2015: Wouter + - windows port fixes, no AF_LOCAL, no chown, no chmod(grp). + +16 January 2015: Wouter + - unit test for local unix connection. Documentation and log_addr + does not inspect port for AF_LOCAL. + - unbound-checkconf -f prints chroot with pidfile path. + +13 January 2015: Wouter + - iana portlist update. + +12 January 2015: Wouter + - Cast sun_len sizeof to socklen_t. + - Fix pyunbound ord call, portable for python 2 and 3. + +7 January 2015: Wouter + - Fix warnings in pythonmod changes. + +6 January 2015: Wouter + - iana portlist update. + - patch for remote control over local sockets, from Dag-Erling + Smorgrav, Ilya Bakulin. Use control-interface: /path/sock and + control-use-cert: no. + - Fixup that patch and uid lookup (only for daemon). + - coded the default of control-use-cert, to yes. + +5 January 2015: Wouter + - getauxval test for ppc64 linux compatibility. + - make strip works for unbound-host and unbound-anchor. + - patch from Stephane Lapie that adds to the python API, that + exposes struct delegpt, and adds the find_delegation function. + - print query name when max target count is exceeded. + - patch from Stuart Henderson that fixes DESTDIR in + unbound-control-setup for installs where config is not in + the prefix location. + - Fix #634: fix fail to start on Linux LTS 3.14.X, ignores missing + IP_MTU_DISCOVER OMIT option (fix from Remi Gacogne). + - Updated contrib warmup.cmd/sh to support two modes - load + from pre-defined list of domains or (with filename as argument) + load from user-specified list of domains, and updated contrib + unbound_cache.sh/cmd to support loading/save/reload cache to/from + default path or (with secondary argument) arbitrary path/filename, + from Yuri Voinov. + - Patch from Philip Paeps to contrib/unbound_munin_ that uses + type ABSOLUTE. Allows munin.conf: [idleserver.example.net] + unbound_munin_hits.graph_period minute + +9 December 2014: Wouter + - svn trunk has 1.5.2 in development. + - config.guess and config.sub update from libtoolize. + - local-zone: example.com inform makes unbound log a message with + client IP for queries in that zone. Eg. for finding infected hosts. + +8 December 2014: Wouter + - Fix CVE-2014-8602: denial of service by making resolver chase + endless series of delegations. + 1 December 2014: Wouter - Fix bug#632: unbound fails to build on AArch64, protects getentropy compat code from calling sysctl if it is has been removed. diff --git a/external/unbound/doc/example.conf.in b/external/unbound/doc/example.conf.in index 03f6184a4..69b3cf39e 100644 --- a/external/unbound/doc/example.conf.in +++ b/external/unbound/doc/example.conf.in @@ -87,6 +87,10 @@ server: # use SO_REUSEPORT to distribute queries over threads. # so-reuseport: no + + # use IP_TRANSPARENT so the interface: addresses can be non-local + # and you can config non-existing IPs that are going to work later on + # ip-transparent: no # EDNS reassembly buffer to advertise to UDP peers (the actual buffer # is set with msg-buffer-size). 1480 can solve fragmentation (timeouts). @@ -138,6 +142,9 @@ server: # the time to live (TTL) value for cached roundtrip times, lameness and # EDNS version information for hosts. In seconds. # infra-host-ttl: 900 + + # minimum wait time for responses, increase if uplink is long. In msec. + # infra-cache-min-rtt: 50 # the number of slabs to use for the Infrastructure cache. # the number of slabs must be a power of 2. @@ -281,6 +288,11 @@ server: # implementation of draft-wijngaards-dnsext-resolver-side-mitigation. # harden-referral-path: no + # Harden against algorithm downgrade when multiple algorithms are + # advertised in the DS record. If no, allows the weakest algorithm + # to validate the zone. + # harden-algo-downgrade: yes + # Use 0x20-encoded random bits in the query to foil spoof attempts. # This feature is an experimental implementation of draft dns-0x20. # use-caps-for-id: no @@ -437,7 +449,7 @@ server: # the amount of memory to use for the negative cache (used for DLV). # plain value in bytes or you can append k, m or G. default is "1Mb". # neg-cache-size: 1m - + # By default, for a number of zones a small default 'nothing here' # reply is built-in. Query traffic is thus blocked. If you # wish to serve such zone you can unblock them by uncommenting one @@ -497,6 +509,7 @@ server: # o redirect serves the zone data for any subdomain in the zone. # o nodefault can be used to normally resolve AS112 zones. # o typetransparent resolves normally for other types and other names + # o inform resolves normally, but logs client IP address # # defaults are localhost address, reverse for 127.0.0.1 and ::1 # and nxdomain for AS112 zones. If you configure one of these zones @@ -552,6 +565,10 @@ remote-control: # set up the keys and certificates with unbound-control-setup. # control-enable: no + # Set to no and use an absolute path as control-interface to use + # a unix local named pipe for unbound-control. + # control-use-cert: yes + # what interfaces are listened to for remote control. # give 0.0.0.0 and ::0 to listen to all interfaces. # control-interface: 127.0.0.1 diff --git a/external/unbound/doc/libunbound.3.in b/external/unbound/doc/libunbound.3.in index 7f693e950..1cefbea51 100644 --- a/external/unbound/doc/libunbound.3.in +++ b/external/unbound/doc/libunbound.3.in @@ -175,6 +175,7 @@ to read them. Before you call this, use the openssl functions CRYPTO_set_id_callback and CRYPTO_set_locking_callback to set up asyncronous operation if you use lib openssl (the application calls these functions once for initialisation). +Openssl 1.0.0 or later uses the CRYPTO_THREADID_set_callback function. .TP .B ub_ctx_delete Delete validation context and free associated resources. diff --git a/external/unbound/doc/unbound-checkconf.8.in b/external/unbound/doc/unbound-checkconf.8.in index 6d0e54516..f38049a03 100644 --- a/external/unbound/doc/unbound-checkconf.8.in +++ b/external/unbound/doc/unbound-checkconf.8.in @@ -13,6 +13,7 @@ unbound\-checkconf .SH "SYNOPSIS" .B unbound\-checkconf .RB [ \-h ] +.RB [ \-f ] .RB [ \-o .IR option ] .RI [ cfgfile ] @@ -29,6 +30,9 @@ The available options are: .B \-h Show the version and commandline option help. .TP +.B \-f +Print full pathname, with chroot applied to it. Use with the \-o option. +.TP .B \-o\fI option If given, after checking the config file the value of this option is printed to stdout. For "" (disabled) options an empty line is printed. diff --git a/external/unbound/doc/unbound-control.8.in b/external/unbound/doc/unbound-control.8.in index b050ac7b4..259eee1d0 100644 --- a/external/unbound/doc/unbound-control.8.in +++ b/external/unbound/doc/unbound-control.8.in @@ -322,6 +322,11 @@ less than this time. Because of big outliers (usually queries to non responsive servers), the average can be bigger than the median. This median has been calculated by interpolation from a histogram. .TP +.I threadX.tcpusage +The currently held tcp buffers for incoming connections. A spot value on +the time of the request. This helps you spot if the incoming\-num\-tcp +buffers are full. +.TP .I total.num.queries summed over threads. .TP @@ -355,6 +360,9 @@ summed over threads. .I total.recursion.time.median averaged over threads. .TP +.I total.tcpusage +summed over threads. +.TP .I time.now current time in seconds since 1970. .TP diff --git a/external/unbound/doc/unbound.conf.5.in b/external/unbound/doc/unbound.conf.5.in index 67ff89b0c..91b8b24ae 100644 --- a/external/unbound/doc/unbound.conf.5.in +++ b/external/unbound/doc/unbound.conf.5.in @@ -164,12 +164,14 @@ By default only ports above 1024 that have not been assigned by IANA are used. Give a port number or a range of the form "low\-high", without spaces. .TP .B outgoing\-num\-tcp: \fI<number> -Number of outgoing TCP buffers to allocate per thread. Default is 10. If set -to 0, or if do\-tcp is "no", no TCP queries to authoritative servers are done. +Number of outgoing TCP buffers to allocate per thread. Default is 10. If +set to 0, or if do\-tcp is "no", no TCP queries to authoritative servers +are done. For larger installations increasing this value is a good idea. .TP .B incoming\-num\-tcp: \fI<number> -Number of incoming TCP buffers to allocate per thread. Default is 10. If set -to 0, or if do\-tcp is "no", no TCP queries from clients are accepted. +Number of incoming TCP buffers to allocate per thread. Default is +10. If set to 0, or if do\-tcp is "no", no TCP queries from clients are +accepted. For larger installations increasing this value is a good idea. .TP .B edns\-buffer\-size: \fI<number> Number of bytes size to advertise as the EDNS reassembly buffer size. @@ -265,6 +267,16 @@ it then attempts to open the port and passes the option if it was available at compile time, if that works it is used, if it fails, it continues silently (unless verbosity 3) without the option. .TP +.B ip\-transparent: \fI<yes or no> +If yes, then use IP_TRANSPARENT socket option on sockets where unbound +is listening for incoming traffic. Default no. Allows you to bind to +non\-local interfaces. For example for non\-existant IP addresses that +are going to exist later on, with host failover configuration. This is +a lot like interface\-automatic, but that one services all interfaces +and with this option you can select which (future) interfaces unbound +provides service on. This option needs unbound to be started with root +permissions on some systems. +.TP .B rrset\-cache\-size: \fI<number> Number of bytes size of the RRset cache. Default is 4 megabytes. A plain number is in bytes, append 'k', 'm' or 'g' for kilobytes, megabytes @@ -301,6 +313,11 @@ by threads. Must be set to a power of 2. .B infra\-cache\-numhosts: \fI<number> Number of hosts for which information is cached. Default is 10000. .TP +.B infra\-cache\-min\-rtt: \fI<msec> +Lower limit for dynamic retransmit timeout calculation in infrastructure +cache. Default is 50 milliseconds. Increase this value if using forwarders +needing more time to do recursive name resolution. +.TP .B do\-ip4: \fI<yes or no> Enable or disable whether ip4 queries are answered or issued. Default is yes. .TP @@ -543,6 +560,13 @@ extra query load that is generated. Experimental option. If you enable it consider adding more numbers after the target\-fetch\-policy to increase the max depth that is checked to. .TP +.B harden\-algo\-downgrade: \fI<yes or no> +Harden against algorithm downgrade when multiple algorithms are +advertised in the DS record. If no, allows the weakest algorithm to +validate the zone. Default is yes. Zone signers must produce zones +that allow this feature to work, but sometimes they do not, and turning +this option off avoids that validation failure. +.TP .B use\-caps\-for\-id: \fI<yes or no> Use 0x20\-encoded random bits in the query to foil spoof attempts. This perturbs the lowercase and uppercase of query names sent to @@ -791,7 +815,7 @@ data leakage about the local network to the upstream DNS servers. .B local\-zone: \fI<zone> <type> Configure a local zone. The type determines the answer to give if there is no match from local\-data. The types are deny, refuse, static, -transparent, redirect, nodefault, typetransparent, and are explained +transparent, redirect, nodefault, typetransparent, inform, and are explained below. After that the default settings are listed. Use local\-data: to enter data into the local zone. Answers for local zones are authoritative DNS answers. By default the zones are class IN. @@ -841,6 +865,13 @@ local\-data: "example.com. A 127.0.0.1" queries for www.example.com and www.foo.example.com are redirected, so that users with web browsers cannot access sites with suffix example.com. .TP 10 +\h'5'\fIinform\fR +The query is answered normally. The client IP address (@portnumber) +is printed to the logfile. The log message is: timestamp, unbound-pid, +info: zonename inform IP@port queryname type class. This option can be +used for normal resolution, but machines looking up infected names are +logged, eg. to run antivirus on them. +.TP 10 \h'5'\fInodefault\fR Used to turn off default contents for AS112 zones. The other types also turn off default contents for the zone. The 'nodefault' option @@ -958,36 +989,47 @@ to setup SSLv3 / TLSv1 security for the connection. The section for options. To setup the correct self\-signed certificates use the \fIunbound\-control\-setup\fR(8) utility. .TP 5 -.B control\-enable: \fI<yes or no> +.B control\-enable: \fI<yes or no> The option is used to enable remote control, default is "no". If turned off, the server does not listen for control commands. .TP 5 -.B control\-interface: <ip address> -Give IPv4 or IPv6 addresses to listen on for control commands. +.B control\-interface: \fI<ip address or path> +Give IPv4 or IPv6 addresses or local socket path to listen on for +control commands. By default localhost (127.0.0.1 and ::1) is listened to. Use 0.0.0.0 and ::0 to listen to all interfaces. +If you change this and permissions have been dropped, you must restart +the server for the change to take effect. +.TP 5 +.B control\-port: \fI<port number> +The port number to listen on for IPv4 or IPv6 control interfaces, +default is 8953. +If you change this and permissions have been dropped, you must restart +the server for the change to take effect. .TP 5 -.B control\-port: <port number> -The port number to listen on for control commands, default is 8953. -If you change this port number, and permissions have been dropped, -a reload is not sufficient to open the port again, you must then restart. +.B control\-use\-cert: \fI<yes or no> +Whether to require certificate authentication of control connections. +The default is "yes". +This should not be changed unless there are other mechanisms in place +to prevent untrusted users from accessing the remote control +interface. .TP 5 -.B server\-key\-file: "<private key file>" +.B server\-key\-file: \fI<private key file> Path to the server private key, by default unbound_server.key. This file is generated by the \fIunbound\-control\-setup\fR utility. This file is used by the unbound server, but not by \fIunbound\-control\fR. .TP 5 -.B server\-cert\-file: "<certificate file.pem>" +.B server\-cert\-file: \fI<certificate file.pem> Path to the server self signed certificate, by default unbound_server.pem. This file is generated by the \fIunbound\-control\-setup\fR utility. This file is used by the unbound server, and also by \fIunbound\-control\fR. .TP 5 -.B control\-key\-file: "<private key file>" +.B control\-key\-file: \fI<private key file> Path to the control client private key, by default unbound_control.key. This file is generated by the \fIunbound\-control\-setup\fR utility. This file is used by \fIunbound\-control\fR. .TP 5 -.B control\-cert\-file: "<certificate file.pem>" +.B control\-cert\-file: \fI<certificate file.pem> Path to the control client certificate, by default unbound_control.pem. This certificate has to be signed with the server certificate. This file is generated by the \fIunbound\-control\-setup\fR utility. diff --git a/external/unbound/iterator/iter_delegpt.c b/external/unbound/iterator/iter_delegpt.c index b212ec077..0e251ff58 100644 --- a/external/unbound/iterator/iter_delegpt.c +++ b/external/unbound/iterator/iter_delegpt.c @@ -47,8 +47,8 @@ #include "util/data/packed_rrset.h" #include "util/data/msgreply.h" #include "util/net_help.h" -#include "ldns/rrdef.h" -#include "ldns/sbuffer.h" +#include "sldns/rrdef.h" +#include "sldns/sbuffer.h" struct delegpt* delegpt_create(struct regional* region) diff --git a/external/unbound/iterator/iter_fwd.c b/external/unbound/iterator/iter_fwd.c index 012121241..0feee032c 100644 --- a/external/unbound/iterator/iter_fwd.c +++ b/external/unbound/iterator/iter_fwd.c @@ -46,8 +46,8 @@ #include "util/config_file.h" #include "util/net_help.h" #include "util/data/dname.h" -#include "ldns/rrdef.h" -#include "ldns/str2wire.h" +#include "sldns/rrdef.h" +#include "sldns/str2wire.h" int fwd_cmp(const void* k1, const void* k2) diff --git a/external/unbound/iterator/iter_hints.c b/external/unbound/iterator/iter_hints.c index 57b57c2e0..25cae0723 100644 --- a/external/unbound/iterator/iter_hints.c +++ b/external/unbound/iterator/iter_hints.c @@ -46,9 +46,9 @@ #include "util/config_file.h" #include "util/net_help.h" #include "util/data/dname.h" -#include "ldns/rrdef.h" -#include "ldns/str2wire.h" -#include "ldns/wire2str.h" +#include "sldns/rrdef.h" +#include "sldns/str2wire.h" +#include "sldns/wire2str.h" struct iter_hints* hints_create(void) diff --git a/external/unbound/iterator/iter_priv.c b/external/unbound/iterator/iter_priv.c index 9e09a84bd..90bea1746 100644 --- a/external/unbound/iterator/iter_priv.c +++ b/external/unbound/iterator/iter_priv.c @@ -49,8 +49,8 @@ #include "util/data/msgparse.h" #include "util/net_help.h" #include "util/storage/dnstree.h" -#include "ldns/str2wire.h" -#include "ldns/sbuffer.h" +#include "sldns/str2wire.h" +#include "sldns/sbuffer.h" struct iter_priv* priv_create(void) { diff --git a/external/unbound/iterator/iter_resptype.c b/external/unbound/iterator/iter_resptype.c index 45f919387..f146a2b6b 100644 --- a/external/unbound/iterator/iter_resptype.c +++ b/external/unbound/iterator/iter_resptype.c @@ -45,8 +45,8 @@ #include "services/cache/dns.h" #include "util/net_help.h" #include "util/data/dname.h" -#include "ldns/rrdef.h" -#include "ldns/pkthdr.h" +#include "sldns/rrdef.h" +#include "sldns/pkthdr.h" enum response_type response_type_from_cache(struct dns_msg* msg, diff --git a/external/unbound/iterator/iter_scrub.c b/external/unbound/iterator/iter_scrub.c index b2248bc0c..e9db19482 100644 --- a/external/unbound/iterator/iter_scrub.c +++ b/external/unbound/iterator/iter_scrub.c @@ -53,7 +53,7 @@ #include "util/data/dname.h" #include "util/data/msgreply.h" #include "util/alloc.h" -#include "ldns/sbuffer.h" +#include "sldns/sbuffer.h" /** RRset flag used during scrubbing. The RRset is OK. */ #define RRSET_SCRUB_OK 0x80 @@ -680,7 +680,9 @@ scrub_sanitize(sldns_buffer* pkt, struct msg_parse* msg, * (we dont want its glue that was approved * during the normalize action) */ del_addi = 1; - } else if(!env->cfg->harden_glue) { + } else if(!env->cfg->harden_glue && ( + rrset->type == LDNS_RR_TYPE_A || + rrset->type == LDNS_RR_TYPE_AAAA)) { /* store in cache! Since it is relevant * (from normalize) it will be picked up * from the cache to be used later */ diff --git a/external/unbound/iterator/iter_utils.c b/external/unbound/iterator/iter_utils.c index 9d0aa698f..5ec5752bf 100644 --- a/external/unbound/iterator/iter_utils.c +++ b/external/unbound/iterator/iter_utils.c @@ -64,7 +64,7 @@ #include "validator/val_kentry.h" #include "validator/val_utils.h" #include "validator/val_sigcrypt.h" -#include "ldns/sbuffer.h" +#include "sldns/sbuffer.h" /** time when nameserver glue is said to be 'recent' */ #define SUSPICION_RECENT_EXPIRY 86400 @@ -715,6 +715,48 @@ reply_equal(struct reply_info* p, struct reply_info* q, struct regional* region) } void +caps_strip_reply(struct reply_info* rep) +{ + size_t i; + if(!rep) return; + /* see if message is a referral, in which case the additional and + * NS record cannot be removed */ + /* referrals have the AA flag unset (strict check, not elsewhere in + * unbound, but for 0x20 this is very convenient). */ + if(!(rep->flags&BIT_AA)) + return; + /* remove the additional section from the reply */ + if(rep->ar_numrrsets != 0) { + verbose(VERB_ALGO, "caps fallback: removing additional section"); + rep->rrset_count -= rep->ar_numrrsets; + rep->ar_numrrsets = 0; + } + /* is there an NS set in the authority section to remove? */ + /* the failure case (Cisco firewalls) only has one rrset in authsec */ + for(i=rep->an_numrrsets; i<rep->an_numrrsets+rep->ns_numrrsets; i++) { + struct ub_packed_rrset_key* s = rep->rrsets[i]; + if(ntohs(s->rk.type) == LDNS_RR_TYPE_NS) { + /* remove NS rrset and break from loop (loop limits + * have changed) */ + /* move last rrset into this position (there is no + * additional section any more) */ + verbose(VERB_ALGO, "caps fallback: removing NS rrset"); + if(i < rep->rrset_count-1) + rep->rrsets[i]=rep->rrsets[rep->rrset_count-1]; + rep->rrset_count --; + rep->ns_numrrsets --; + break; + } + } +} + +int caps_failed_rcode(struct reply_info* rep) +{ + return !(FLAGS_GET_RCODE(rep->flags) == LDNS_RCODE_NOERROR || + FLAGS_GET_RCODE(rep->flags) == LDNS_RCODE_NXDOMAIN); +} + +void iter_store_parentside_rrset(struct module_env* env, struct ub_packed_rrset_key* rrset) { diff --git a/external/unbound/iterator/iter_utils.h b/external/unbound/iterator/iter_utils.h index d7c2b68af..3a4df3e45 100644 --- a/external/unbound/iterator/iter_utils.h +++ b/external/unbound/iterator/iter_utils.h @@ -223,6 +223,23 @@ int iter_msg_from_zone(struct dns_msg* msg, struct delegpt* dp, int reply_equal(struct reply_info* p, struct reply_info* q, struct regional* region); /** + * Remove unused bits from the reply if possible. + * So that caps-for-id (0x20) fallback is more likely to be successful. + * This removes like, the additional section, and NS record in the authority + * section if those records are gratuitous (not for a referral). + * @param rep: the reply to strip stuff out of. + */ +void caps_strip_reply(struct reply_info* rep); + +/** + * see if reply has a 'useful' rcode for capsforid comparison, so + * not SERVFAIL or REFUSED, and thus NOERROR or NXDOMAIN. + * @param rep: reply to check. + * @return true if the rcode is a bad type of message. + */ +int caps_failed_rcode(struct reply_info* rep); + +/** * Store parent-side rrset in seperate rrset cache entries for later * last-resort * lookups in case the child-side versions of this information * fails. diff --git a/external/unbound/iterator/iterator.c b/external/unbound/iterator/iterator.c index 843948a2e..02de86e12 100644 --- a/external/unbound/iterator/iterator.c +++ b/external/unbound/iterator/iterator.c @@ -61,10 +61,10 @@ #include "util/data/msgencode.h" #include "util/fptr_wlist.h" #include "util/config_file.h" -#include "ldns/rrdef.h" -#include "ldns/wire2str.h" -#include "ldns/parseutil.h" -#include "ldns/sbuffer.h" +#include "sldns/rrdef.h" +#include "sldns/wire2str.h" +#include "sldns/parseutil.h" +#include "sldns/sbuffer.h" int iter_init(struct module_env* env, int id) @@ -120,6 +120,7 @@ iter_new(struct module_qstate* qstate, int id) iq->query_restart_count = 0; iq->referral_count = 0; iq->sent_count = 0; + iq->target_count = NULL; iq->wait_priming_stub = 0; iq->refetch_glue = 0; iq->dnssec_expected = 0; @@ -307,6 +308,8 @@ iter_prepend(struct iter_qstate* iq, struct dns_msg* msg, if(num_an + num_ns == 0) return 1; verbose(VERB_ALGO, "prepending %d rrsets", (int)num_an + (int)num_ns); + if(num_an > RR_COUNT_MAX || num_ns > RR_COUNT_MAX || + msg->rep->rrset_count > RR_COUNT_MAX) return 0; /* overflow */ sets = regional_alloc(region, (num_an+num_ns+msg->rep->rrset_count) * sizeof(struct ub_packed_rrset_key*)); if(!sets) @@ -454,6 +457,26 @@ handle_cname_response(struct module_qstate* qstate, struct iter_qstate* iq, return 1; } +/** create target count structure for this query */ +static void +target_count_create(struct iter_qstate* iq) +{ + if(!iq->target_count) { + iq->target_count = (int*)calloc(2, sizeof(int)); + /* if calloc fails we simply do not track this number */ + if(iq->target_count) + iq->target_count[0] = 1; + } +} + +static void +target_count_increase(struct iter_qstate* iq, int num) +{ + target_count_create(iq); + if(iq->target_count) + iq->target_count[1] += num; +} + /** * Generate a subrequest. * Generate a local request event. Local events are tied to this module, and @@ -529,6 +552,10 @@ generate_sub_request(uint8_t* qname, size_t qnamelen, uint16_t qtype, subiq = (struct iter_qstate*)subq->minfo[id]; memset(subiq, 0, sizeof(*subiq)); subiq->num_target_queries = 0; + target_count_create(iq); + subiq->target_count = iq->target_count; + if(iq->target_count) + iq->target_count[0] ++; /* extra reference */ subiq->num_current_queries = 0; subiq->depth = iq->depth+1; outbound_list_init(&subiq->outlist); @@ -1356,6 +1383,14 @@ query_for_targets(struct module_qstate* qstate, struct iter_qstate* iq, if(iq->depth == ie->max_dependency_depth) return 0; + if(iq->depth > 0 && iq->target_count && + iq->target_count[1] > MAX_TARGET_COUNT) { + char s[LDNS_MAX_DOMAINLEN+1]; + dname_str(qstate->qinfo.qname, s); + verbose(VERB_QUERY, "request %s has exceeded the maximum " + "number of glue fetches %d", s, iq->target_count[1]); + return 0; + } iter_mark_cycle_targets(qstate, iq->dp); missing = (int)delegpt_count_missing_targets(iq->dp); @@ -1538,6 +1573,7 @@ processLastResort(struct module_qstate* qstate, struct iter_qstate* iq, return error_response(qstate, id, LDNS_RCODE_SERVFAIL); } iq->num_target_queries += qs; + target_count_increase(iq, qs); if(qs != 0) { qstate->ext_state[id] = module_wait_subquery; return 0; /* and wait for them */ @@ -1547,6 +1583,14 @@ processLastResort(struct module_qstate* qstate, struct iter_qstate* iq, verbose(VERB_QUERY, "maxdepth and need more nameservers, fail"); return error_response_cache(qstate, id, LDNS_RCODE_SERVFAIL); } + if(iq->depth > 0 && iq->target_count && + iq->target_count[1] > MAX_TARGET_COUNT) { + char s[LDNS_MAX_DOMAINLEN+1]; + dname_str(qstate->qinfo.qname, s); + verbose(VERB_QUERY, "request %s has exceeded the maximum " + "number of glue fetches %d", s, iq->target_count[1]); + return error_response_cache(qstate, id, LDNS_RCODE_SERVFAIL); + } /* mark cycle targets for parent-side lookups */ iter_mark_pside_cycle_targets(qstate, iq->dp); /* see if we can issue queries to get nameserver addresses */ @@ -1576,6 +1620,7 @@ processLastResort(struct module_qstate* qstate, struct iter_qstate* iq, if(query_count != 0) { /* suspend to await results */ verbose(VERB_ALGO, "try parent-side glue lookup"); iq->num_target_queries += query_count; + target_count_increase(iq, query_count); qstate->ext_state[id] = module_wait_subquery; return 0; } @@ -1731,6 +1776,7 @@ processQueryTargets(struct module_qstate* qstate, struct iter_qstate* iq, return error_response(qstate, id, LDNS_RCODE_SERVFAIL); } iq->num_target_queries += extra; + target_count_increase(iq, extra); if(iq->num_target_queries > 0) { /* wait to get all targets, we want to try em */ verbose(VERB_ALGO, "wait for all targets for fallback"); @@ -1743,11 +1789,13 @@ processQueryTargets(struct module_qstate* qstate, struct iter_qstate* iq, * the original query is one that matched too, so we have * caps_server+1 number of matching queries now */ if(iq->caps_server+1 >= naddr*3 || - iq->caps_server+1 >= MAX_SENT_COUNT) { + iq->caps_server*2+2 >= MAX_SENT_COUNT) { + /* *2 on sentcount check because ipv6 may fail */ /* we're done, process the response */ verbose(VERB_ALGO, "0x20 fallback had %d responses " "match for %d wanted, done.", (int)iq->caps_server+1, (int)naddr*3); + iq->response = iq->caps_response; iq->caps_fallback = 0; iter_dec_attempts(iq->dp, 3); /* space for fallback */ iq->num_current_queries++; /* RespState decrements it*/ @@ -1771,6 +1819,7 @@ processQueryTargets(struct module_qstate* qstate, struct iter_qstate* iq, /* errors ignored, these targets are not strictly necessary for * this result, we do not have to reply with SERVFAIL */ iq->num_target_queries += extra; + target_count_increase(iq, extra); } /* Add the current set of unused targets to our queue. */ @@ -1816,10 +1865,29 @@ processQueryTargets(struct module_qstate* qstate, struct iter_qstate* iq, return 1; } iq->num_target_queries += qs; + target_count_increase(iq, qs); } /* Since a target query might have been made, we * need to check again. */ if(iq->num_target_queries == 0) { + /* if in capsforid fallback, instead of last + * resort, we agree with the current reply + * we have (if any) (our count of addrs bad)*/ + if(iq->caps_fallback && iq->caps_reply) { + /* we're done, process the response */ + verbose(VERB_ALGO, "0x20 fallback had %d responses, " + "but no more servers except " + "last resort, done.", + (int)iq->caps_server+1); + iq->response = iq->caps_response; + iq->caps_fallback = 0; + iter_dec_attempts(iq->dp, 3); /* space for fallback */ + iq->num_current_queries++; /* RespState decrements it*/ + iq->referral_count++; /* make sure we don't loop */ + iq->sent_count = 0; + iq->state = QUERY_RESP_STATE; + return 1; + } return processLastResort(qstate, iq, ie, id); } } @@ -2483,6 +2551,12 @@ processClassResponse(struct module_qstate* qstate, int id, /* copy appropriate rcode */ to->rep->flags = from->rep->flags; /* copy rrsets */ + if(from->rep->rrset_count > RR_COUNT_MAX || + to->rep->rrset_count > RR_COUNT_MAX) { + log_err("malloc failed (too many rrsets) in collect ANY"); + foriq->state = FINISHED_STATE; + return; /* integer overflow protection */ + } dest = regional_alloc(forq->region, sizeof(dest[0])*n); if(!dest) { log_err("malloc failed in collect ANY"); @@ -2779,6 +2853,7 @@ process_response(struct module_qstate* qstate, struct iter_qstate* iq, iq->caps_fallback = 1; iq->caps_server = 0; iq->caps_reply = NULL; + iq->caps_response = NULL; iq->state = QUERYTARGETS_STATE; iq->num_current_queries--; /* need fresh attempts for the 0x20 fallback, if @@ -2821,8 +2896,19 @@ process_response(struct module_qstate* qstate, struct iter_qstate* iq, /* normalize and sanitize: easy to delete items from linked lists */ if(!scrub_message(pkt, prs, &iq->qchase, iq->dp->name, - qstate->env->scratch, qstate->env, ie)) + qstate->env->scratch, qstate->env, ie)) { + /* if 0x20 enabled, start fallback, but we have no message */ + if(event == module_event_capsfail && !iq->caps_fallback) { + iq->caps_fallback = 1; + iq->caps_server = 0; + iq->caps_reply = NULL; + iq->caps_response = NULL; + iq->state = QUERYTARGETS_STATE; + iq->num_current_queries--; + verbose(VERB_DETAIL, "Capsforid: scrub failed, starting fallback with no response"); + } goto handle_it; + } /* allocate response dns_msg in region */ iq->response = dns_alloc_msg(pkt, prs, qstate->region); @@ -2836,11 +2922,15 @@ process_response(struct module_qstate* qstate, struct iter_qstate* iq, iq->response->rep); if(event == module_event_capsfail || iq->caps_fallback) { + /* for fallback we care about main answer, not additionals */ + /* removing that makes comparison more likely to succeed */ + caps_strip_reply(iq->response->rep); if(!iq->caps_fallback) { /* start fallback */ iq->caps_fallback = 1; iq->caps_server = 0; iq->caps_reply = iq->response->rep; + iq->caps_response = iq->response; iq->state = QUERYTARGETS_STATE; iq->num_current_queries--; verbose(VERB_DETAIL, "Capsforid: starting fallback"); @@ -2849,8 +2939,24 @@ process_response(struct module_qstate* qstate, struct iter_qstate* iq, /* check if reply is the same, otherwise, fail */ if(!iq->caps_reply) { iq->caps_reply = iq->response->rep; + iq->caps_response = iq->response; iq->caps_server = -1; /*become zero at ++, so that we start the full set of trials */ + } else if(caps_failed_rcode(iq->caps_reply) && + !caps_failed_rcode(iq->response->rep)) { + /* prefer to upgrade to non-SERVFAIL */ + iq->caps_reply = iq->response->rep; + iq->caps_response = iq->response; + } else if(!caps_failed_rcode(iq->caps_reply) && + caps_failed_rcode(iq->response->rep)) { + /* if we have non-SERVFAIL as answer then + * we can ignore SERVFAILs for the equality + * comparison */ + /* no instructions here, skip other else */ + } else if(caps_failed_rcode(iq->caps_reply) && + caps_failed_rcode(iq->response->rep)) { + /* failure is same as other failure in fallbk*/ + /* no instructions here, skip other else */ } else if(!reply_equal(iq->response->rep, iq->caps_reply, qstate->env->scratch)) { verbose(VERB_DETAIL, "Capsforid fallback: " @@ -2928,6 +3034,8 @@ iter_clear(struct module_qstate* qstate, int id) iq = (struct iter_qstate*)qstate->minfo[id]; if(iq) { outbound_list_clear(&iq->outlist); + if(iq->target_count && --iq->target_count[0] == 0) + free(iq->target_count); iq->num_current_queries = 0; } qstate->minfo[id] = NULL; diff --git a/external/unbound/iterator/iterator.h b/external/unbound/iterator/iterator.h index 0b91760d4..cb8043fd5 100644 --- a/external/unbound/iterator/iterator.h +++ b/external/unbound/iterator/iterator.h @@ -52,6 +52,8 @@ struct iter_donotq; struct iter_prep_list; struct iter_priv; +/** max number of targets spawned for a query and its subqueries */ +#define MAX_TARGET_COUNT 32 /** max number of query restarts. Determines max number of CNAME chain. */ #define MAX_RESTART_COUNT 8 /** max number of referrals. Makes sure resolver does not run away */ @@ -233,6 +235,7 @@ struct iter_qstate { /** state for capsfail: stored query for comparisons. Can be NULL if * no response had been seen prior to starting the fallback. */ struct reply_info* caps_reply; + struct dns_msg* caps_response; /** Current delegation message - returned for non-RD queries */ struct dns_msg* deleg_msg; @@ -251,6 +254,10 @@ struct iter_qstate { /** number of queries fired off */ int sent_count; + + /** number of target queries spawned in [1], for this query and its + * subqueries, the malloced-array is shared, [0] refcount. */ + int* target_count; /** * The query must store NS records from referrals as parentside RRs diff --git a/external/unbound/libunbound/context.c b/external/unbound/libunbound/context.c index c21f94184..4469b5bb4 100644 --- a/external/unbound/libunbound/context.c +++ b/external/unbound/libunbound/context.c @@ -49,7 +49,7 @@ #include "services/cache/infra.h" #include "util/data/msgreply.h" #include "util/storage/slabhash.h" -#include "ldns/sbuffer.h" +#include "sldns/sbuffer.h" int context_finalize(struct ub_ctx* ctx) @@ -360,7 +360,7 @@ context_serialize_cancel(struct ctx_query* q, uint32_t* len) /* format of cancel: * o uint32 cmd * o uint32 async-id */ - uint8_t* p = (uint8_t*)malloc(2*sizeof(uint32_t)); + uint8_t* p = (uint8_t*)reallocarray(NULL, sizeof(uint32_t), 2); if(!p) return NULL; *len = 2*sizeof(uint32_t); sldns_write_uint32(p, UB_LIBCMD_CANCEL); diff --git a/external/unbound/libunbound/libunbound.c b/external/unbound/libunbound/libunbound.c index 91a663a77..a37da9ee4 100644 --- a/external/unbound/libunbound/libunbound.c +++ b/external/unbound/libunbound/libunbound.c @@ -61,7 +61,7 @@ #include "services/localzone.h" #include "services/cache/infra.h" #include "services/cache/rrset.h" -#include "ldns/sbuffer.h" +#include "sldns/sbuffer.h" #ifdef HAVE_PTHREAD #include <signal.h> #endif @@ -1028,7 +1028,6 @@ ub_ctx_hosts(struct ub_ctx* ctx, const char* fname) "\\hosts"); retval=ub_ctx_hosts(ctx, buf); } - free(name); return retval; } return UB_READFILE; diff --git a/external/unbound/libunbound/libworker.c b/external/unbound/libunbound/libworker.c index e388e7956..72b615313 100644 --- a/external/unbound/libunbound/libworker.c +++ b/external/unbound/libunbound/libworker.c @@ -70,8 +70,8 @@ #include "util/tube.h" #include "iterator/iter_fwd.h" #include "iterator/iter_hints.h" -#include "ldns/sbuffer.h" -#include "ldns/str2wire.h" +#include "sldns/sbuffer.h" +#include "sldns/str2wire.h" /** handle new query command for bg worker */ static void handle_newq(struct libworker* w, uint8_t* buf, uint32_t len); @@ -606,7 +606,7 @@ int libworker_fg(struct ub_ctx* ctx, struct ctx_query* q) sldns_buffer_write_u16_at(w->back->udp_buff, 0, qid); sldns_buffer_write_u16_at(w->back->udp_buff, 2, qflags); if(local_zones_answer(ctx->local_zones, &qinfo, &edns, - w->back->udp_buff, w->env->scratch)) { + w->back->udp_buff, w->env->scratch, NULL)) { regional_free_all(w->env->scratch); libworker_fillup_fg(q, LDNS_RCODE_NOERROR, w->back->udp_buff, sec_status_insecure, NULL); @@ -676,7 +676,7 @@ int libworker_attach_mesh(struct ub_ctx* ctx, struct ctx_query* q, sldns_buffer_write_u16_at(w->back->udp_buff, 0, qid); sldns_buffer_write_u16_at(w->back->udp_buff, 2, qflags); if(local_zones_answer(ctx->local_zones, &qinfo, &edns, - w->back->udp_buff, w->env->scratch)) { + w->back->udp_buff, w->env->scratch, NULL)) { regional_free_all(w->env->scratch); free(qinfo.qname); libworker_event_done_cb(q, LDNS_RCODE_NOERROR, @@ -796,7 +796,7 @@ handle_newq(struct libworker* w, uint8_t* buf, uint32_t len) sldns_buffer_write_u16_at(w->back->udp_buff, 0, qid); sldns_buffer_write_u16_at(w->back->udp_buff, 2, qflags); if(local_zones_answer(w->ctx->local_zones, &qinfo, &edns, - w->back->udp_buff, w->env->scratch)) { + w->back->udp_buff, w->env->scratch, NULL)) { regional_free_all(w->env->scratch); q->msg_security = sec_status_insecure; add_bg_result(w, q, w->back->udp_buff, UB_NOERROR, NULL); diff --git a/external/unbound/libunbound/python/libunbound.i b/external/unbound/libunbound/python/libunbound.i index 313c74862..1bef79f22 100644 --- a/external/unbound/libunbound/python/libunbound.i +++ b/external/unbound/libunbound/python/libunbound.i @@ -44,6 +44,10 @@ %pythoncode %{ import encodings.idna + try: + import builtins + except ImportError: + import __builtin__ as builtins # Ensure compatibility with older python versions if 'bytes' not in vars(): @@ -52,7 +56,7 @@ def ord(s): if isinstance(s, int): return s - return __builtins__.ord(s) + return builtins.ord(s) %} //%include "doc.i" @@ -699,7 +703,7 @@ Result: ['74.125.43.147', '74.125.43.99', '74.125.43.103', '74.125.43.104'] while (idx < slen): complen = ord(s[idx]) # In python 3.x `str()` converts the string to unicode which is the expected text string type - res.append(str(s[idx+1:idx+1+complen])) + res.append(str(s[idx+1:idx+1+complen].decode())) idx += complen + 1 return res diff --git a/external/unbound/libunbound/worker.h b/external/unbound/libunbound/worker.h index 824012a01..a53150199 100644 --- a/external/unbound/libunbound/worker.h +++ b/external/unbound/libunbound/worker.h @@ -42,7 +42,7 @@ #ifndef LIBUNBOUND_WORKER_H #define LIBUNBOUND_WORKER_H -#include "ldns/sbuffer.h" +#include "sldns/sbuffer.h" #include "util/data/packed_rrset.h" /* for enum sec_status */ struct comm_reply; struct comm_point; diff --git a/external/unbound/makedist.sh b/external/unbound/makedist.sh index 6ece32605..f06017668 100755 --- a/external/unbound/makedist.sh +++ b/external/unbound/makedist.sh @@ -232,6 +232,7 @@ if [ "$DOWIN" = "yes" ]; then cd .. fi + info "SVNROOT is $SVNROOT" info "Exporting source from SVN." svn export "$SVNROOT" unbound || error_cleanup "SVN command failed" cd unbound || error_cleanup "Unbound not exported correctly from SVN" diff --git a/external/unbound/pythonmod/interface.i b/external/unbound/pythonmod/interface.i index 4f1a25f21..b2dd08904 100644 --- a/external/unbound/pythonmod/interface.i +++ b/external/unbound/pythonmod/interface.i @@ -26,6 +26,9 @@ #include "util/storage/lruhash.h" #include "services/cache/dns.h" #include "services/mesh.h" + #include "iterator/iter_delegpt.h" + #include "iterator/iter_hints.h" + #include "iterator/iter_utils.h" #include "ldns/wire2str.h" #include "ldns/str2wire.h" #include "ldns/pkthdr.h" @@ -672,6 +675,99 @@ struct config_file { }; /* ************************************************************************************ * + ASN: Adding structures related to forwards_lookup and dns_cache_find_delegation + * ************************************************************************************ */ +struct delegpt_ns { + struct delegpt_ns* next; + int resolved; + uint8_t got4; + uint8_t got6; + uint8_t lame; + uint8_t done_pside4; + uint8_t done_pside6; +}; + +struct delegpt_addr { + struct delegpt_addr* next_result; + struct delegpt_addr* next_usable; + struct delegpt_addr* next_target; + int attempts; + int sel_rtt; + int bogus; + int lame; +}; + +struct delegpt { + int namelabs; + struct delegpt_ns* nslist; + struct delegpt_addr* target_list; + struct delegpt_addr* usable_list; + struct delegpt_addr* result_list; + int bogus; + uint8_t has_parent_side_NS; + uint8_t dp_type_mlc; +}; + + +%inline %{ + PyObject* _get_dp_dname(struct delegpt* dp) { + return PyString_FromStringAndSize((char*)dp->name, dp->namelen); + } + PyObject* _get_dp_dname_components(struct delegpt* dp) { + return GetNameAsLabelList((char*)dp->name, dp->namelen); + } + PyObject* _get_dpns_dname(struct delegpt_ns* dpns) { + return PyString_FromStringAndSize((char*)dpns->name, dpns->namelen); + } + PyObject* _get_dpns_dname_components(struct delegpt_ns* dpns) { + return GetNameAsLabelList((char*)dpns->name, dpns->namelen); + } + + PyObject* _delegpt_addr_addr_get(struct delegpt_addr* target) { + char dest[64]; + delegpt_addr_addr2str(target, dest, 64); + if (dest[0] == 0) + return Py_None; + return PyString_FromString(dest); + } + +%} + +%extend delegpt { + %pythoncode %{ + __swig_getmethods__["dname"] = _unboundmodule._get_dp_dname + if _newclass:dname = _swig_property(_unboundmodule._get_dp_dname) + + __swig_getmethods__["dname_list"] = _unboundmodule._get_dp_dname_components + if _newclass:dname_list = _swig_property(_unboundmodule._get_dp_dname_components) + + def _get_dname_str(self): return dnameAsStr(self.dname) + __swig_getmethods__["dname_str"] = _get_dname_str + if _newclass:dname_str = _swig_property(_get_dname_str) + %} +} +%extend delegpt_ns { + %pythoncode %{ + __swig_getmethods__["dname"] = _unboundmodule._get_dpns_dname + if _newclass:dname = _swig_property(_unboundmodule._get_dpns_dname) + + __swig_getmethods__["dname_list"] = _unboundmodule._get_dpns_dname_components + if _newclass:dname_list = _swig_property(_unboundmodule._get_dpns_dname_components) + + def _get_dname_str(self): return dnameAsStr(self.dname) + __swig_getmethods__["dname_str"] = _get_dname_str + if _newclass:dname_str = _swig_property(_get_dname_str) + %} +} +%extend delegpt_addr { + %pythoncode %{ + def _addr_get(self): return _delegpt_addr_addr_get(self) + __swig_getmethods__["addr"] = _addr_get + if _newclass:addr = _swig_property(_addr_get) + %} +} + +/* ************************************************************************************ * Enums * ************************************************************************************ */ %rename ("MODULE_STATE_INITIAL") "module_state_initial"; @@ -880,6 +976,65 @@ int set_return_msg(struct module_qstate* qstate, %} /* ************************************************************************************ * + ASN: Delegation pointer related functions + * ************************************************************************************ */ + +/* Functions which we will need to lookup delegations */ +struct delegpt* dns_cache_find_delegation(struct module_env* env, + uint8_t* qname, size_t qnamelen, uint16_t qtype, uint16_t qclass, + struct regional* region, struct dns_msg** msg, uint32_t timenow); +int iter_dp_is_useless(struct query_info* qinfo, uint16_t qflags, + struct delegpt* dp); +struct iter_hints_stub* hints_lookup_stub(struct iter_hints* hints, + uint8_t* qname, uint16_t qclass, struct delegpt* dp); + +/* Custom function to perform logic similar to the one in daemon/cachedump.c */ +struct delegpt* find_delegation(struct module_qstate* qstate, char *nm, size_t nmlen); + +%{ +#define BIT_RD 0x100 + +struct delegpt* find_delegation(struct module_qstate* qstate, char *nm, size_t nmlen) +{ + struct delegpt *dp; + struct dns_msg *msg = NULL; + struct regional* region = qstate->env->scratch; + char b[260]; + struct query_info qinfo; + struct iter_hints_stub* stub; + uint32_t timenow = *qstate->env->now; + + regional_free_all(region); + qinfo.qname = (uint8_t*)nm; + qinfo.qname_len = nmlen; + qinfo.qtype = LDNS_RR_TYPE_A; + qinfo.qclass = LDNS_RR_CLASS_IN; + + while(1) { + dp = dns_cache_find_delegation(qstate->env, (uint8_t*)nm, nmlen, qinfo.qtype, qinfo.qclass, region, &msg, timenow); + if(!dp) + return NULL; + if(iter_dp_is_useless(&qinfo, BIT_RD, dp)) { + if (dname_is_root((uint8_t*)nm)) + return NULL; + nm = (char*)dp->name; + nmlen = dp->namelen; + dname_remove_label((uint8_t**)&nm, &nmlen); + dname_str((uint8_t*)nm, b); + continue; + } + stub = hints_lookup_stub(qstate->env->hints, qinfo.qname, qinfo.qclass, dp); + if (stub) { + return stub->dp; + } else { + return dp; + } + } + return NULL; +} +%} + +/* ************************************************************************************ * Functions * ************************************************************************************ */ diff --git a/external/unbound/pythonmod/pythonmod.c b/external/unbound/pythonmod/pythonmod.c index 359eea0c6..190d41a81 100644 --- a/external/unbound/pythonmod/pythonmod.c +++ b/external/unbound/pythonmod/pythonmod.c @@ -45,7 +45,7 @@ #endif #include "config.h" -#include "ldns/sbuffer.h" +#include "sldns/sbuffer.h" #undef _POSIX_C_SOURCE #undef _XOPEN_SOURCE diff --git a/external/unbound/pythonmod/pythonmod_utils.c b/external/unbound/pythonmod/pythonmod_utils.c index 1091dcf10..5120074e8 100644 --- a/external/unbound/pythonmod/pythonmod_utils.c +++ b/external/unbound/pythonmod/pythonmod_utils.c @@ -48,7 +48,8 @@ #include "util/data/msgreply.h" #include "util/storage/slabhash.h" #include "util/regional.h" -#include "ldns/sbuffer.h" +#include "iterator/iter_delegpt.h" +#include "sldns/sbuffer.h" #undef _POSIX_C_SOURCE #undef _XOPEN_SOURCE @@ -176,3 +177,17 @@ void reply_addr2str(struct comm_reply* reply, char* dest, int maxlen) return; dest[maxlen-1] = 0; } + +/* Convert target->addr to string */ +void delegpt_addr_addr2str(struct delegpt_addr* target, char *dest, int maxlen) +{ + int af = (int)((struct sockaddr_in*) &(target->addr))->sin_family; + void* sinaddr = &((struct sockaddr_in*) &(target->addr))->sin_addr; + + if(af == AF_INET6) + sinaddr = &((struct sockaddr_in6*)&(target->addr))->sin6_addr; + dest[0] = 0; + if (inet_ntop(af, sinaddr, dest, (socklen_t)maxlen) == 0) + return; + dest[maxlen-1] = 0; +} diff --git a/external/unbound/pythonmod/pythonmod_utils.h b/external/unbound/pythonmod/pythonmod_utils.h index a901f391a..768eb46de 100644 --- a/external/unbound/pythonmod/pythonmod_utils.h +++ b/external/unbound/pythonmod/pythonmod_utils.h @@ -42,6 +42,7 @@ #define PYTHONMOD_UTILS_H #include "util/module.h" +struct delegpt_addr; /** * Store the reply_info and query_info pair in message cache (qstate->msg_cache) @@ -86,4 +87,7 @@ int createResponse(struct module_qstate* qstate, sldns_buffer* pkt); */ void reply_addr2str(struct comm_reply* reply, char* dest, int maxlen); +/* Convert target->addr to string */ +void delegpt_addr_addr2str(struct delegpt_addr* target, char *dest, int maxlen); + #endif /* PYTHONMOD_UTILS_H */ diff --git a/external/unbound/services/cache/dns.c b/external/unbound/services/cache/dns.c index 4692744a1..cec2629e1 100644 --- a/external/unbound/services/cache/dns.c +++ b/external/unbound/services/cache/dns.c @@ -50,7 +50,7 @@ #include "util/net_help.h" #include "util/regional.h" #include "util/config_file.h" -#include "ldns/sbuffer.h" +#include "sldns/sbuffer.h" /** store rrsets in the rrset cache. * @param env: module environment with caches. @@ -366,6 +366,8 @@ dns_msg_create(uint8_t* qname, size_t qnamelen, uint16_t qtype, sizeof(struct reply_info)-sizeof(struct rrset_ref)); if(!msg->rep) return NULL; + if(capacity > RR_COUNT_MAX) + return NULL; /* integer overflow protection */ msg->rep->flags = BIT_QR; /* with QR, no AA */ msg->rep->qdcount = 1; msg->rep->rrsets = (struct ub_packed_rrset_key**) @@ -453,6 +455,8 @@ gen_dns_msg(struct regional* region, struct query_info* q, size_t num) sizeof(struct reply_info) - sizeof(struct rrset_ref)); if(!msg->rep) return NULL; + if(num > RR_COUNT_MAX) + return NULL; /* integer overflow protection */ msg->rep->rrsets = (struct ub_packed_rrset_key**) regional_alloc(region, num * sizeof(struct ub_packed_rrset_key*)); diff --git a/external/unbound/services/cache/infra.c b/external/unbound/services/cache/infra.c index 07f2103d7..61bab3fe5 100644 --- a/external/unbound/services/cache/infra.c +++ b/external/unbound/services/cache/infra.c @@ -39,7 +39,7 @@ * This file contains the infrastructure cache. */ #include "config.h" -#include "ldns/rrdef.h" +#include "sldns/rrdef.h" #include "services/cache/infra.h" #include "util/storage/slabhash.h" #include "util/storage/lookup3.h" diff --git a/external/unbound/services/cache/rrset.c b/external/unbound/services/cache/rrset.c index 5f52dbce1..2c8552953 100644 --- a/external/unbound/services/cache/rrset.c +++ b/external/unbound/services/cache/rrset.c @@ -40,7 +40,7 @@ */ #include "config.h" #include "services/cache/rrset.h" -#include "ldns/rrdef.h" +#include "sldns/rrdef.h" #include "util/storage/slabhash.h" #include "util/config_file.h" #include "util/data/packed_rrset.h" @@ -304,10 +304,11 @@ rrset_array_unlock_touch(struct rrset_cache* r, struct regional* scratch, { hashvalue_t* h; size_t i; - if(!(h = (hashvalue_t*)regional_alloc(scratch, - sizeof(hashvalue_t)*count))) + if(count > RR_COUNT_MAX || !(h = (hashvalue_t*)regional_alloc(scratch, + sizeof(hashvalue_t)*count))) { log_warn("rrset LRU: memory allocation failed"); - else /* store hash values */ + h = NULL; + } else /* store hash values */ for(i=0; i<count; i++) h[i] = ref[i].key->entry.hash; /* unlock */ diff --git a/external/unbound/services/listen_dnsport.c b/external/unbound/services/listen_dnsport.c index b7ffb6d3f..276c0fb32 100644 --- a/external/unbound/services/listen_dnsport.c +++ b/external/unbound/services/listen_dnsport.c @@ -49,13 +49,17 @@ #include "util/log.h" #include "util/config_file.h" #include "util/net_help.h" -#include "ldns/sbuffer.h" +#include "sldns/sbuffer.h" #ifdef HAVE_NETDB_H #include <netdb.h> #endif #include <fcntl.h> +#ifdef HAVE_SYS_UN_H +#include <sys/un.h> +#endif + /** number of queued TCP connections for listen() */ #define TCP_BACKLOG 256 @@ -92,10 +96,10 @@ verbose_print_addr(struct addrinfo *addr) int create_udp_sock(int family, int socktype, struct sockaddr* addr, socklen_t addrlen, int v6only, int* inuse, int* noproto, - int rcv, int snd, int listen, int* reuseport) + int rcv, int snd, int listen, int* reuseport, int transparent) { int s; -#if defined(SO_REUSEADDR) || defined(SO_REUSEPORT) || defined(IPV6_USE_MIN_MTU) +#if defined(SO_REUSEADDR) || defined(SO_REUSEPORT) || defined(IPV6_USE_MIN_MTU) || defined(IP_TRANSPARENT) int on=1; #endif #ifdef IPV6_MTU @@ -110,6 +114,9 @@ create_udp_sock(int family, int socktype, struct sockaddr* addr, #ifndef IPV6_V6ONLY (void)v6only; #endif +#ifndef IP_TRANSPARENT + (void)transparent; +#endif if((s = socket(family, socktype, 0)) == -1) { *inuse = 0; #ifndef USE_WINSOCK @@ -173,6 +180,14 @@ create_udp_sock(int family, int socktype, struct sockaddr* addr, #else (void)reuseport; #endif /* defined(SO_REUSEPORT) */ +#ifdef IP_TRANSPARENT + if (transparent && + setsockopt(s, IPPROTO_IP, IP_TRANSPARENT, (void*)&on, + (socklen_t)sizeof(on)) < 0) { + log_warn("setsockopt(.. IP_TRANSPARENT ..) failed: %s", + strerror(errno)); + } +#endif /* IP_TRANSPARENT */ } if(rcv) { #ifdef SO_RCVBUF @@ -368,29 +383,47 @@ create_udp_sock(int family, int socktype, struct sockaddr* addr, * (and also uses the interface mtu to determine the size of the packets). * So there won't be any EMSGSIZE error. Against DNS fragmentation attacks. * FreeBSD already has same semantics without setting the option. */ -# if defined(IP_PMTUDISC_OMIT) - int action = IP_PMTUDISC_OMIT; -# else - int action = IP_PMTUDISC_DONT; -# endif + int omit_set = 0; + int action; +# if defined(IP_PMTUDISC_OMIT) + action = IP_PMTUDISC_OMIT; if (setsockopt(s, IPPROTO_IP, IP_MTU_DISCOVER, &action, (socklen_t)sizeof(action)) < 0) { - log_err("setsockopt(..., IP_MTU_DISCOVER, " -# if defined(IP_PMTUDISC_OMIT) - "IP_PMTUDISC_OMIT" + + if (errno != EINVAL) { + log_err("setsockopt(..., IP_MTU_DISCOVER, IP_PMTUDISC_OMIT...) failed: %s", + strerror(errno)); + +# ifndef USE_WINSOCK + close(s); # else - "IP_PMTUDISC_DONT" + closesocket(s); # endif - "...) failed: %s", - strerror(errno)); + *noproto = 0; + *inuse = 0; + return -1; + } + } + else + { + omit_set = 1; + } +# endif + if (omit_set == 0) { + action = IP_PMTUDISC_DONT; + if (setsockopt(s, IPPROTO_IP, IP_MTU_DISCOVER, + &action, (socklen_t)sizeof(action)) < 0) { + log_err("setsockopt(..., IP_MTU_DISCOVER, IP_PMTUDISC_DONT...) failed: %s", + strerror(errno)); # ifndef USE_WINSOCK - close(s); + close(s); # else - closesocket(s); + closesocket(s); # endif - *noproto = 0; - *inuse = 0; - return -1; + *noproto = 0; + *inuse = 0; + return -1; + } } # elif defined(IP_DONTFRAG) int off = 0; @@ -450,12 +483,15 @@ create_udp_sock(int family, int socktype, struct sockaddr* addr, int create_tcp_accept_sock(struct addrinfo *addr, int v6only, int* noproto, - int* reuseport) + int* reuseport, int transparent) { int s; -#if defined(SO_REUSEADDR) || defined(SO_REUSEPORT) || defined(IPV6_V6ONLY) +#if defined(SO_REUSEADDR) || defined(SO_REUSEPORT) || defined(IPV6_V6ONLY) || defined(IP_TRANSPARENT) int on = 1; -#endif /* SO_REUSEADDR || IPV6_V6ONLY */ +#endif +#ifndef IP_TRANSPARENT + (void)transparent; +#endif verbose_print_addr(addr); *noproto = 0; if((s = socket(addr->ai_family, addr->ai_socktype, 0)) == -1) { @@ -530,6 +566,14 @@ create_tcp_accept_sock(struct addrinfo *addr, int v6only, int* noproto, #else (void)v6only; #endif /* IPV6_V6ONLY */ +#ifdef IP_TRANSPARENT + if (transparent && + setsockopt(s, IPPROTO_IP, IP_TRANSPARENT, (void*)&on, + (socklen_t)sizeof(on)) < 0) { + log_warn("setsockopt(.. IP_TRANSPARENT ..) failed: %s", + strerror(errno)); + } +#endif /* IP_TRANSPARENT */ if(bind(s, addr->ai_addr, addr->ai_addrlen) != 0) { #ifndef USE_WINSOCK /* detect freebsd jail with no ipv6 permission */ @@ -571,13 +615,70 @@ create_tcp_accept_sock(struct addrinfo *addr, int v6only, int* noproto, return s; } +int +create_local_accept_sock(const char *path, int* noproto) +{ +#ifdef HAVE_SYS_UN_H + int s; + struct sockaddr_un usock; + + verbose(VERB_ALGO, "creating unix socket %s", path); +#ifdef HAVE_STRUCT_SOCKADDR_UN_SUN_LEN + /* this member exists on BSDs, not Linux */ + usock.sun_len = (socklen_t)sizeof(usock); +#endif + usock.sun_family = AF_LOCAL; + /* length is 92-108, 104 on FreeBSD */ + (void)strlcpy(usock.sun_path, path, sizeof(usock.sun_path)); + + if ((s = socket(AF_LOCAL, SOCK_STREAM, 0)) == -1) { + log_err("Cannot create local socket %s (%s)", + path, strerror(errno)); + return -1; + } + + if (unlink(path) && errno != ENOENT) { + /* The socket already exists and cannot be removed */ + log_err("Cannot remove old local socket %s (%s)", + path, strerror(errno)); + return -1; + } + + if (bind(s, (struct sockaddr *)&usock, + (socklen_t)sizeof(struct sockaddr_un)) == -1) { + log_err("Cannot bind local socket %s (%s)", + path, strerror(errno)); + return -1; + } + + if (!fd_set_nonblock(s)) { + log_err("Cannot set non-blocking mode"); + return -1; + } + + if (listen(s, TCP_BACKLOG) == -1) { + log_err("can't listen: %s", strerror(errno)); + return -1; + } + + (void)noproto; /*unused*/ + return s; +#else + (void)path; + log_err("Local sockets are not supported"); + *noproto = 1; + return -1; +#endif +} + + /** * Create socket from getaddrinfo results */ static int make_sock(int stype, const char* ifname, const char* port, struct addrinfo *hints, int v6only, int* noip6, size_t rcv, size_t snd, - int* reuseport) + int* reuseport, int transparent) { struct addrinfo *res = NULL; int r, s, inuse, noproto; @@ -605,14 +706,15 @@ make_sock(int stype, const char* ifname, const char* port, s = create_udp_sock(res->ai_family, res->ai_socktype, (struct sockaddr*)res->ai_addr, res->ai_addrlen, v6only, &inuse, &noproto, (int)rcv, (int)snd, 1, - reuseport); + reuseport, transparent); if(s == -1 && inuse) { log_err("bind: address already in use"); } else if(s == -1 && noproto && hints->ai_family == AF_INET6){ *noip6 = 1; } } else { - s = create_tcp_accept_sock(res, v6only, &noproto, reuseport); + s = create_tcp_accept_sock(res, v6only, &noproto, reuseport, + transparent); if(s == -1 && noproto && hints->ai_family == AF_INET6){ *noip6 = 1; } @@ -625,7 +727,7 @@ make_sock(int stype, const char* ifname, const char* port, static int make_sock_port(int stype, const char* ifname, const char* port, struct addrinfo *hints, int v6only, int* noip6, size_t rcv, size_t snd, - int* reuseport) + int* reuseport, int transparent) { char* s = strchr(ifname, '@'); if(s) { @@ -647,10 +749,10 @@ make_sock_port(int stype, const char* ifname, const char* port, (void)strlcpy(p, s+1, sizeof(p)); p[strlen(s+1)]=0; return make_sock(stype, newif, p, hints, v6only, noip6, - rcv, snd, reuseport); + rcv, snd, reuseport, transparent); } return make_sock(stype, ifname, port, hints, v6only, noip6, rcv, snd, - reuseport); + reuseport, transparent); } /** @@ -744,19 +846,20 @@ set_recvpktinfo(int s, int family) * @param ssl_port: ssl service port number * @param reuseport: try to set SO_REUSEPORT if nonNULL and true. * set to false on exit if reuseport failed due to no kernel support. + * @param transparent: set IP_TRANSPARENT socket option. * @return: returns false on error. */ static int ports_create_if(const char* ifname, int do_auto, int do_udp, int do_tcp, struct addrinfo *hints, const char* port, struct listen_port** list, - size_t rcv, size_t snd, int ssl_port, int* reuseport) + size_t rcv, size_t snd, int ssl_port, int* reuseport, int transparent) { int s, noip6=0; if(!do_udp && !do_tcp) return 0; if(do_auto) { if((s = make_sock_port(SOCK_DGRAM, ifname, port, hints, 1, - &noip6, rcv, snd, reuseport)) == -1) { + &noip6, rcv, snd, reuseport, transparent)) == -1) { if(noip6) { log_warn("IPv6 protocol not available"); return 1; @@ -783,7 +886,7 @@ ports_create_if(const char* ifname, int do_auto, int do_udp, int do_tcp, } else if(do_udp) { /* regular udp socket */ if((s = make_sock_port(SOCK_DGRAM, ifname, port, hints, 1, - &noip6, rcv, snd, reuseport)) == -1) { + &noip6, rcv, snd, reuseport, transparent)) == -1) { if(noip6) { log_warn("IPv6 protocol not available"); return 1; @@ -804,7 +907,7 @@ ports_create_if(const char* ifname, int do_auto, int do_udp, int do_tcp, atoi(strchr(ifname, '@')+1) == ssl_port) || (!strchr(ifname, '@') && atoi(port) == ssl_port)); if((s = make_sock_port(SOCK_STREAM, ifname, port, hints, 1, - &noip6, 0, 0, reuseport)) == -1) { + &noip6, 0, 0, reuseport, transparent)) == -1) { if(noip6) { /*log_warn("IPv6 protocol not available");*/ return 1; @@ -960,7 +1063,8 @@ listening_ports_open(struct config_file* cfg, int* reuseport) do_auto, cfg->do_udp, do_tcp, &hints, portbuf, &list, cfg->so_rcvbuf, cfg->so_sndbuf, - cfg->ssl_port, reuseport)) { + cfg->ssl_port, reuseport, + cfg->ip_transparent)) { listening_ports_free(list); return NULL; } @@ -971,7 +1075,8 @@ listening_ports_open(struct config_file* cfg, int* reuseport) do_auto, cfg->do_udp, do_tcp, &hints, portbuf, &list, cfg->so_rcvbuf, cfg->so_sndbuf, - cfg->ssl_port, reuseport)) { + cfg->ssl_port, reuseport, + cfg->ip_transparent)) { listening_ports_free(list); return NULL; } @@ -984,7 +1089,8 @@ listening_ports_open(struct config_file* cfg, int* reuseport) if(!ports_create_if(cfg->ifs[i], 0, cfg->do_udp, do_tcp, &hints, portbuf, &list, cfg->so_rcvbuf, cfg->so_sndbuf, - cfg->ssl_port, reuseport)) { + cfg->ssl_port, reuseport, + cfg->ip_transparent)) { listening_ports_free(list); return NULL; } @@ -995,7 +1101,8 @@ listening_ports_open(struct config_file* cfg, int* reuseport) if(!ports_create_if(cfg->ifs[i], 0, cfg->do_udp, do_tcp, &hints, portbuf, &list, cfg->so_rcvbuf, cfg->so_sndbuf, - cfg->ssl_port, reuseport)) { + cfg->ssl_port, reuseport, + cfg->ip_transparent)) { listening_ports_free(list); return NULL; } diff --git a/external/unbound/services/listen_dnsport.h b/external/unbound/services/listen_dnsport.h index 075f6d281..676f0c638 100644 --- a/external/unbound/services/listen_dnsport.h +++ b/external/unbound/services/listen_dnsport.h @@ -189,11 +189,12 @@ void listen_start_accept(struct listen_dnsport* listen); * set SO_REUSEADDR on it. * @param reuseport: if nonNULL and true, try to set SO_REUSEPORT on * listening UDP port. Set to false on return if it failed to do so. + * @param transparent: set IP_TRANSPARENT socket option. * @return: the socket. -1 on error. */ int create_udp_sock(int family, int socktype, struct sockaddr* addr, socklen_t addrlen, int v6only, int* inuse, int* noproto, int rcv, - int snd, int listen, int* reuseport); + int snd, int listen, int* reuseport, int transparent); /** * Create and bind TCP listening socket @@ -202,9 +203,19 @@ int create_udp_sock(int family, int socktype, struct sockaddr* addr, * @param noproto: if error caused by lack of protocol support. * @param reuseport: if nonNULL and true, try to set SO_REUSEPORT on * listening UDP port. Set to false on return if it failed to do so. + * @param transparent: set IP_TRANSPARENT socket option. * @return: the socket. -1 on error. */ int create_tcp_accept_sock(struct addrinfo *addr, int v6only, int* noproto, - int* reuseport); + int* reuseport, int transparent); + +/** + * Create and bind local listening socket + * @param path: path to the socket. + * @param noproto: on error, this is set true if cause is that local sockets + * are not supported. + * @return: the socket. -1 on error. + */ +int create_local_accept_sock(const char* path, int* noproto); #endif /* LISTEN_DNSPORT_H */ diff --git a/external/unbound/services/localzone.c b/external/unbound/services/localzone.c index d285a127c..51491656f 100644 --- a/external/unbound/services/localzone.c +++ b/external/unbound/services/localzone.c @@ -40,14 +40,15 @@ */ #include "config.h" #include "services/localzone.h" -#include "ldns/str2wire.h" -#include "ldns/sbuffer.h" +#include "sldns/str2wire.h" +#include "sldns/sbuffer.h" #include "util/regional.h" #include "util/config_file.h" #include "util/data/dname.h" #include "util/data/packed_rrset.h" #include "util/data/msgencode.h" #include "util/net_help.h" +#include "util/netevent.h" #include "util/data/msgreply.h" #include "util/data/msgparse.h" @@ -1022,6 +1023,10 @@ void local_zones_print(struct local_zones* zones) log_nametypeclass(0, "static zone", z->name, 0, z->dclass); break; + case local_zone_inform: + log_nametypeclass(0, "inform zone", + z->name, 0, z->dclass); + break; default: log_nametypeclass(0, "badtyped zone", z->name, 0, z->dclass); @@ -1169,9 +1174,25 @@ lz_zone_answer(struct local_zone* z, struct query_info* qinfo, return 0; } +/** print log information for an inform zone query */ +static void +lz_inform_print(struct local_zone* z, struct query_info* qinfo, + struct comm_reply* repinfo) +{ + char ip[128], txt[512]; + char zname[LDNS_MAX_DOMAINLEN+1]; + uint16_t port = ntohs(((struct sockaddr_in*)&repinfo->addr)->sin_port); + dname_str(z->name, zname); + addr_to_str(&repinfo->addr, repinfo->addrlen, ip, sizeof(ip)); + snprintf(txt, sizeof(txt), "%s inform %s@%u", zname, ip, + (unsigned)port); + log_nametypeclass(0, txt, qinfo->qname, qinfo->qtype, qinfo->qclass); +} + int local_zones_answer(struct local_zones* zones, struct query_info* qinfo, - struct edns_data* edns, sldns_buffer* buf, struct regional* temp) + struct edns_data* edns, sldns_buffer* buf, struct regional* temp, + struct comm_reply* repinfo) { /* see if query is covered by a zone, * if so: - try to match (exact) local data @@ -1190,6 +1211,9 @@ local_zones_answer(struct local_zones* zones, struct query_info* qinfo, lock_rw_rdlock(&z->lock); lock_rw_unlock(&zones->lock); + if(z->type == local_zone_inform && repinfo) + lz_inform_print(z, qinfo, repinfo); + if(local_data_answer(z, qinfo, edns, buf, temp, labs, &ld)) { lock_rw_unlock(&z->lock); return 1; @@ -1209,6 +1233,7 @@ const char* local_zone_type2str(enum localzone_type t) case local_zone_typetransparent: return "typetransparent"; case local_zone_static: return "static"; case local_zone_nodefault: return "nodefault"; + case local_zone_inform: return "inform"; } return "badtyped"; } @@ -1227,6 +1252,8 @@ int local_zone_str2type(const char* type, enum localzone_type* t) *t = local_zone_typetransparent; else if(strcmp(type, "redirect") == 0) *t = local_zone_redirect; + else if(strcmp(type, "inform") == 0) + *t = local_zone_inform; else return 0; return 1; } diff --git a/external/unbound/services/localzone.h b/external/unbound/services/localzone.h index 788fbfb3b..29ba8663f 100644 --- a/external/unbound/services/localzone.h +++ b/external/unbound/services/localzone.h @@ -49,6 +49,7 @@ struct config_file; struct edns_data; struct query_info; struct sldns_buffer; +struct comm_reply; /** * Local zone type @@ -70,7 +71,9 @@ enum localzone_type { local_zone_redirect, /** remove default AS112 blocking contents for zone * nodefault is used in config not during service. */ - local_zone_nodefault + local_zone_nodefault, + /** log client address, but no block (transparent) */ + local_zone_inform }; /** @@ -220,12 +223,14 @@ void local_zones_print(struct local_zones* zones); * @param edns: edns info (parsed). * @param buf: buffer with query ID and flags, also for reply. * @param temp: temporary storage region. + * @param repinfo: source address for checks. may be NULL. * @return true if answer is in buffer. false if query is not answered * by authority data. If the reply should be dropped altogether, the return * value is true, but the buffer is cleared (empty). */ int local_zones_answer(struct local_zones* zones, struct query_info* qinfo, - struct edns_data* edns, struct sldns_buffer* buf, struct regional* temp); + struct edns_data* edns, struct sldns_buffer* buf, struct regional* temp, + struct comm_reply* repinfo); /** * Parse the string into localzone type. diff --git a/external/unbound/services/mesh.c b/external/unbound/services/mesh.c index a69aced22..8076874ae 100644 --- a/external/unbound/services/mesh.c +++ b/external/unbound/services/mesh.c @@ -55,7 +55,7 @@ #include "util/fptr_wlist.h" #include "util/alloc.h" #include "util/config_file.h" -#include "ldns/sbuffer.h" +#include "sldns/sbuffer.h" /** subtract timers and the values do not overflow or become negative */ static void diff --git a/external/unbound/services/outside_network.c b/external/unbound/services/outside_network.c index 5bb52ff9f..dc3d2f404 100644 --- a/external/unbound/services/outside_network.c +++ b/external/unbound/services/outside_network.c @@ -57,7 +57,7 @@ #include "util/net_help.h" #include "util/random.h" #include "util/fptr_wlist.h" -#include "ldns/sbuffer.h" +#include "sldns/sbuffer.h" #include "dnstap/dnstap.h" #ifdef HAVE_OPENSSL_SSL_H #include <openssl/ssl.h> @@ -893,13 +893,13 @@ udp_sockport(struct sockaddr_storage* addr, socklen_t addrlen, int port, sa->sin6_port = (in_port_t)htons((uint16_t)port); fd = create_udp_sock(AF_INET6, SOCK_DGRAM, (struct sockaddr*)addr, addrlen, 1, inuse, &noproto, - 0, 0, 0, NULL); + 0, 0, 0, NULL, 0); } else { struct sockaddr_in* sa = (struct sockaddr_in*)addr; sa->sin_port = (in_port_t)htons((uint16_t)port); fd = create_udp_sock(AF_INET, SOCK_DGRAM, (struct sockaddr*)addr, addrlen, 1, inuse, &noproto, - 0, 0, 0, NULL); + 0, 0, 0, NULL, 0); } return fd; } diff --git a/external/unbound/ldns/keyraw.c b/external/unbound/sldns/keyraw.c index 1ff07742b..59e8000f5 100644 --- a/external/unbound/ldns/keyraw.c +++ b/external/unbound/sldns/keyraw.c @@ -11,8 +11,8 @@ */ #include "config.h" -#include "ldns/keyraw.h" -#include "ldns/rrdef.h" +#include "sldns/keyraw.h" +#include "sldns/rrdef.h" #ifdef HAVE_SSL #include <openssl/ssl.h> diff --git a/external/unbound/ldns/keyraw.h b/external/unbound/sldns/keyraw.h index 8abe23509..8abe23509 100644 --- a/external/unbound/ldns/keyraw.h +++ b/external/unbound/sldns/keyraw.h diff --git a/external/unbound/ldns/parse.c b/external/unbound/sldns/parse.c index a605e549f..35dee7196 100644 --- a/external/unbound/ldns/parse.c +++ b/external/unbound/sldns/parse.c @@ -8,9 +8,9 @@ * See the file LICENSE for the license */ #include "config.h" -#include "ldns/parse.h" -#include "ldns/parseutil.h" -#include "ldns/sbuffer.h" +#include "sldns/parse.h" +#include "sldns/parseutil.h" +#include "sldns/sbuffer.h" #include <limits.h> #include <strings.h> diff --git a/external/unbound/ldns/parse.h b/external/unbound/sldns/parse.h index 7b7456dd2..7b7456dd2 100644 --- a/external/unbound/ldns/parse.h +++ b/external/unbound/sldns/parse.h diff --git a/external/unbound/ldns/parseutil.c b/external/unbound/sldns/parseutil.c index 28b344ede..2a2ebbb08 100644 --- a/external/unbound/ldns/parseutil.c +++ b/external/unbound/sldns/parseutil.c @@ -13,7 +13,7 @@ */ #include "config.h" -#include "ldns/parseutil.h" +#include "sldns/parseutil.h" #include <sys/time.h> #include <time.h> #include <ctype.h> diff --git a/external/unbound/ldns/parseutil.h b/external/unbound/sldns/parseutil.h index dfa1c2a2b..dfa1c2a2b 100644 --- a/external/unbound/ldns/parseutil.h +++ b/external/unbound/sldns/parseutil.h diff --git a/external/unbound/ldns/pkthdr.h b/external/unbound/sldns/pkthdr.h index de9952ea7..de9952ea7 100644 --- a/external/unbound/ldns/pkthdr.h +++ b/external/unbound/sldns/pkthdr.h diff --git a/external/unbound/ldns/rrdef.c b/external/unbound/sldns/rrdef.c index 9b4bf7cfe..72161d7b6 100644 --- a/external/unbound/ldns/rrdef.c +++ b/external/unbound/sldns/rrdef.c @@ -13,8 +13,8 @@ * Defines resource record types and constants. */ #include "config.h" -#include "ldns/rrdef.h" -#include "ldns/parseutil.h" +#include "sldns/rrdef.h" +#include "sldns/parseutil.h" /* classes */ static sldns_lookup_table sldns_rr_classes_data[] = { diff --git a/external/unbound/ldns/rrdef.h b/external/unbound/sldns/rrdef.h index 933bcdfbf..933bcdfbf 100644 --- a/external/unbound/ldns/rrdef.h +++ b/external/unbound/sldns/rrdef.h diff --git a/external/unbound/ldns/sbuffer.c b/external/unbound/sldns/sbuffer.c index 3d087bfe2..a7fe53aa0 100644 --- a/external/unbound/ldns/sbuffer.c +++ b/external/unbound/sldns/sbuffer.c @@ -12,7 +12,7 @@ * This file contains the definition of sldns_buffer, and functions to manipulate those. */ #include "config.h" -#include "ldns/sbuffer.h" +#include "sldns/sbuffer.h" #include <stdarg.h> sldns_buffer * diff --git a/external/unbound/ldns/sbuffer.h b/external/unbound/sldns/sbuffer.h index 3ce874fc7..3ce874fc7 100644 --- a/external/unbound/ldns/sbuffer.h +++ b/external/unbound/sldns/sbuffer.h diff --git a/external/unbound/ldns/str2wire.c b/external/unbound/sldns/str2wire.c index 931e28f84..8cda8c750 100644 --- a/external/unbound/ldns/str2wire.c +++ b/external/unbound/sldns/str2wire.c @@ -12,11 +12,11 @@ * Parses text to wireformat. */ #include "config.h" -#include "ldns/str2wire.h" -#include "ldns/wire2str.h" -#include "ldns/sbuffer.h" -#include "ldns/parse.h" -#include "ldns/parseutil.h" +#include "sldns/str2wire.h" +#include "sldns/wire2str.h" +#include "sldns/sbuffer.h" +#include "sldns/parse.h" +#include "sldns/parseutil.h" #include <ctype.h> #ifdef HAVE_TIME_H #include <time.h> diff --git a/external/unbound/ldns/str2wire.h b/external/unbound/sldns/str2wire.h index 94c893389..527074a15 100644 --- a/external/unbound/ldns/str2wire.h +++ b/external/unbound/sldns/str2wire.h @@ -16,7 +16,7 @@ #define LDNS_STR2WIRE_H /* include rrdef for MAX_DOMAINLEN constant */ -#include <ldns/rrdef.h> +#include <sldns/rrdef.h> #ifdef __cplusplus extern "C" { diff --git a/external/unbound/ldns/wire2str.c b/external/unbound/sldns/wire2str.c index 81e173c78..cec3bc7b0 100644 --- a/external/unbound/ldns/wire2str.c +++ b/external/unbound/sldns/wire2str.c @@ -15,13 +15,13 @@ * representation, as well as functions to print them. */ #include "config.h" -#include "ldns/wire2str.h" -#include "ldns/str2wire.h" -#include "ldns/rrdef.h" -#include "ldns/pkthdr.h" -#include "ldns/parseutil.h" -#include "ldns/sbuffer.h" -#include "ldns/keyraw.h" +#include "sldns/wire2str.h" +#include "sldns/str2wire.h" +#include "sldns/rrdef.h" +#include "sldns/pkthdr.h" +#include "sldns/parseutil.h" +#include "sldns/sbuffer.h" +#include "sldns/keyraw.h" #ifdef HAVE_TIME_H #include <time.h> #endif diff --git a/external/unbound/ldns/wire2str.h b/external/unbound/sldns/wire2str.h index 67f543566..67f543566 100644 --- a/external/unbound/ldns/wire2str.h +++ b/external/unbound/sldns/wire2str.h diff --git a/external/unbound/smallapp/unbound-anchor.c b/external/unbound/smallapp/unbound-anchor.c index 9df0d95b4..576a30f64 100644 --- a/external/unbound/smallapp/unbound-anchor.c +++ b/external/unbound/smallapp/unbound-anchor.c @@ -116,7 +116,7 @@ #include "config.h" #include "libunbound/unbound.h" -#include "ldns/rrdef.h" +#include "sldns/rrdef.h" #include <expat.h> #ifndef HAVE_EXPAT_H #error "need libexpat to parse root-anchors.xml file." @@ -915,7 +915,10 @@ read_data_chunk(SSL* ssl, size_t len) { size_t got = 0; int r; - char* data = malloc(len+1); + char* data; + if(len >= 0xfffffff0) + return NULL; /* to protect against integer overflow in malloc*/ + data = malloc(len+1); if(!data) { if(verb) printf("out of memory\n"); return NULL; diff --git a/external/unbound/smallapp/unbound-checkconf.c b/external/unbound/smallapp/unbound-checkconf.c index e83867f26..0524edeaa 100644 --- a/external/unbound/smallapp/unbound-checkconf.c +++ b/external/unbound/smallapp/unbound-checkconf.c @@ -53,7 +53,7 @@ #include "iterator/iter_hints.h" #include "validator/validator.h" #include "services/localzone.h" -#include "ldns/sbuffer.h" +#include "sldns/sbuffer.h" #ifdef HAVE_GETOPT_H #include <getopt.h> #endif @@ -78,6 +78,7 @@ usage() printf(" Checks unbound configuration file for errors.\n"); printf("file if omitted %s is used.\n", CONFIGFILE); printf("-o option print value of option to stdout.\n"); + printf("-f output full pathname with chroot applied, eg. with -o pidfile.\n"); printf("-h show this usage help.\n"); printf("Version %s\n", PACKAGE_VERSION); printf("BSD licensed, see LICENSE in source package for details.\n"); @@ -90,10 +91,15 @@ usage() * @param cfg: config * @param opt: option name without trailing :. * This is different from config_set_option. + * @param final: if final pathname with chroot applied has to be printed. */ static void -print_option(struct config_file* cfg, const char* opt) +print_option(struct config_file* cfg, const char* opt, int final) { + if(strcmp(opt, "pidfile") == 0 && final) { + printf("%s\n", fname_after_chroot(cfg->pidfile, cfg, 1)); + return; + } if(!config_get_option(cfg, opt, config_print_func, stdout)) fatal_exit("cannot print option '%s'", opt); } @@ -416,7 +422,7 @@ morechecks(struct config_file* cfg, const char* fname) endpwent(); } #endif - if(cfg->remote_control_enable) { + if(cfg->remote_control_enable && cfg->remote_control_use_cert) { check_chroot_string("server-key-file", &cfg->server_key_file, cfg->chrootdir, cfg); check_chroot_string("server-cert-file", &cfg->server_cert_file, @@ -456,7 +462,7 @@ check_hints(struct config_file* cfg) /** check config file */ static void -checkconf(const char* cfgfile, const char* opt) +checkconf(const char* cfgfile, const char* opt, int final) { struct config_file* cfg = config_create(); if(!cfg) @@ -467,7 +473,7 @@ checkconf(const char* cfgfile, const char* opt) exit(1); } if(opt) { - print_option(cfg, opt); + print_option(cfg, opt, final); config_delete(cfg); return; } @@ -493,6 +499,7 @@ extern char* optarg; int main(int argc, char* argv[]) { int c; + int final = 0; const char* f; const char* opt = NULL; const char* cfgfile = CONFIGFILE; @@ -505,8 +512,11 @@ int main(int argc, char* argv[]) cfgfile = CONFIGFILE; #endif /* USE_WINSOCK */ /* parse the options */ - while( (c=getopt(argc, argv, "ho:")) != -1) { + while( (c=getopt(argc, argv, "fho:")) != -1) { switch(c) { + case 'f': + final = 1; + break; case 'o': opt = optarg; break; @@ -523,7 +533,7 @@ int main(int argc, char* argv[]) if(argc == 1) f = argv[0]; else f = cfgfile; - checkconf(f, opt); + checkconf(f, opt, final); checklock_stop(); return 0; } diff --git a/external/unbound/smallapp/unbound-control-setup.sh.in b/external/unbound/smallapp/unbound-control-setup.sh.in index 79605dc6f..682ab260a 100644 --- a/external/unbound/smallapp/unbound-control-setup.sh.in +++ b/external/unbound/smallapp/unbound-control-setup.sh.in @@ -36,8 +36,7 @@ # settings: # directory for files -prefix=@prefix@ -DESTDIR=@sysconfdir@/unbound +DESTDIR=@ub_conf_dir@ # issuer and subject name for certificates SERVERNAME=unbound @@ -47,7 +46,7 @@ CLIENTNAME=unbound-control DAYS=7200 # size of keys in bits -BITS=1536 +BITS=3072 # hash algorithm HASH=sha256 diff --git a/external/unbound/smallapp/unbound-control.c b/external/unbound/smallapp/unbound-control.c index ff86184a8..3b47d3bf8 100644 --- a/external/unbound/smallapp/unbound-control.c +++ b/external/unbound/smallapp/unbound-control.c @@ -59,6 +59,10 @@ #include "util/locks.h" #include "util/net_help.h" +#ifdef HAVE_SYS_UN_H +#include <sys/un.h> +#endif + /** Give unbound-control usage, and exit (1). */ static void usage() @@ -136,32 +140,40 @@ static void ssl_err(const char* s) static SSL_CTX* setup_ctx(struct config_file* cfg) { - char* s_cert, *c_key, *c_cert; + char* s_cert=NULL, *c_key=NULL, *c_cert=NULL; SSL_CTX* ctx; - s_cert = fname_after_chroot(cfg->server_cert_file, cfg, 1); - c_key = fname_after_chroot(cfg->control_key_file, cfg, 1); - c_cert = fname_after_chroot(cfg->control_cert_file, cfg, 1); - if(!s_cert || !c_key || !c_cert) - fatal_exit("out of memory"); + if(cfg->remote_control_use_cert) { + s_cert = fname_after_chroot(cfg->server_cert_file, cfg, 1); + c_key = fname_after_chroot(cfg->control_key_file, cfg, 1); + c_cert = fname_after_chroot(cfg->control_cert_file, cfg, 1); + if(!s_cert || !c_key || !c_cert) + fatal_exit("out of memory"); + } ctx = SSL_CTX_new(SSLv23_client_method()); if(!ctx) ssl_err("could not allocate SSL_CTX pointer"); if(!(SSL_CTX_set_options(ctx, SSL_OP_NO_SSLv2) & SSL_OP_NO_SSLv2)) ssl_err("could not set SSL_OP_NO_SSLv2"); - if(!(SSL_CTX_set_options(ctx, SSL_OP_NO_SSLv3) & SSL_OP_NO_SSLv3)) - ssl_err("could not set SSL_OP_NO_SSLv3"); - if(!SSL_CTX_use_certificate_file(ctx,c_cert,SSL_FILETYPE_PEM) || - !SSL_CTX_use_PrivateKey_file(ctx,c_key,SSL_FILETYPE_PEM) - || !SSL_CTX_check_private_key(ctx)) - ssl_err("Error setting up SSL_CTX client key and cert"); - if (SSL_CTX_load_verify_locations(ctx, s_cert, NULL) != 1) - ssl_err("Error setting up SSL_CTX verify, server cert"); - SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER, NULL); + if(cfg->remote_control_use_cert) { + if(!(SSL_CTX_set_options(ctx, SSL_OP_NO_SSLv3) & SSL_OP_NO_SSLv3)) + ssl_err("could not set SSL_OP_NO_SSLv3"); + if(!SSL_CTX_use_certificate_file(ctx,c_cert,SSL_FILETYPE_PEM) || + !SSL_CTX_use_PrivateKey_file(ctx,c_key,SSL_FILETYPE_PEM) + || !SSL_CTX_check_private_key(ctx)) + ssl_err("Error setting up SSL_CTX client key and cert"); + if (SSL_CTX_load_verify_locations(ctx, s_cert, NULL) != 1) + ssl_err("Error setting up SSL_CTX verify, server cert"); + SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER, NULL); - free(s_cert); - free(c_key); - free(c_cert); + free(s_cert); + free(c_key); + free(c_cert); + } else { + /* Use ciphers that don't require authentication */ + if(!SSL_CTX_set_cipher_list(ctx, "aNULL")) + ssl_err("Error setting NULL cipher!"); + } return ctx; } @@ -171,6 +183,7 @@ contact_server(const char* svr, struct config_file* cfg, int statuscmd) { struct sockaddr_storage addr; socklen_t addrlen; + int addrfamily = 0; int fd; /* use svr or the first config entry */ if(!svr) { @@ -189,12 +202,25 @@ contact_server(const char* svr, struct config_file* cfg, int statuscmd) if(strchr(svr, '@')) { if(!extstrtoaddr(svr, &addr, &addrlen)) fatal_exit("could not parse IP@port: %s", svr); +#ifdef HAVE_SYS_UN_H + } else if(svr[0] == '/') { + struct sockaddr_un* usock = (struct sockaddr_un *) &addr; + usock->sun_family = AF_LOCAL; +#ifdef HAVE_STRUCT_SOCKADDR_UN_SUN_LEN + usock->sun_len = (socklen_t)sizeof(usock); +#endif + (void)strlcpy(usock->sun_path, svr, sizeof(usock->sun_path)); + addrlen = (socklen_t)sizeof(struct sockaddr_un); + addrfamily = AF_LOCAL; +#endif } else { if(!ipstrtoaddr(svr, cfg->control_port, &addr, &addrlen)) fatal_exit("could not parse IP: %s", svr); } - fd = socket(addr_is_ip6(&addr, addrlen)?AF_INET6:AF_INET, - SOCK_STREAM, 0); + + if(addrfamily == 0) + addrfamily = addr_is_ip6(&addr, addrlen)?AF_INET6:AF_INET; + fd = socket(addrfamily, SOCK_STREAM, 0); if(fd == -1) { #ifndef USE_WINSOCK fatal_exit("socket: %s", strerror(errno)); @@ -223,7 +249,7 @@ contact_server(const char* svr, struct config_file* cfg, int statuscmd) /** setup SSL on the connection */ static SSL* -setup_ssl(SSL_CTX* ctx, int fd) +setup_ssl(SSL_CTX* ctx, int fd, struct config_file* cfg) { SSL* ssl; X509* x; @@ -249,10 +275,13 @@ setup_ssl(SSL_CTX* ctx, int fd) /* check authenticity of server */ if(SSL_get_verify_result(ssl) != X509_V_OK) ssl_err("SSL verification failed"); - x = SSL_get_peer_certificate(ssl); - if(!x) - ssl_err("Server presented no peer certificate"); - X509_free(x); + if(cfg->remote_control_use_cert) { + x = SSL_get_peer_certificate(ssl); + if(!x) + ssl_err("Server presented no peer certificate"); + X509_free(x); + } + return ssl; } @@ -330,11 +359,11 @@ go(const char* cfgfile, char* svr, int quiet, int argc, char* argv[]) if(!cfg->remote_control_enable) log_warn("control-enable is 'no' in the config file."); ctx = setup_ctx(cfg); - + /* contact server */ fd = contact_server(svr, cfg, argc>0&&strcmp(argv[0],"status")==0); - ssl = setup_ssl(ctx, fd); - + ssl = setup_ssl(ctx, fd, cfg); + /* send command */ ret = go_cmd(ssl, quiet, argc, argv); diff --git a/external/unbound/smallapp/unbound-host.c b/external/unbound/smallapp/unbound-host.c index 959734109..30fef51fd 100644 --- a/external/unbound/smallapp/unbound-host.c +++ b/external/unbound/smallapp/unbound-host.c @@ -60,8 +60,8 @@ #define unbound_lite_wrapstr(s) s #endif #include "libunbound/unbound.h" -#include "ldns/rrdef.h" -#include "ldns/wire2str.h" +#include "sldns/rrdef.h" +#include "sldns/wire2str.h" #ifdef HAVE_NSS /* nss3 */ #include "nss.h" diff --git a/external/unbound/testcode/asynclook.c b/external/unbound/testcode/asynclook.c index 7e9ee7758..534489735 100644 --- a/external/unbound/testcode/asynclook.c +++ b/external/unbound/testcode/asynclook.c @@ -48,7 +48,7 @@ #include "libunbound/context.h" #include "util/locks.h" #include "util/log.h" -#include "ldns/rrdef.h" +#include "sldns/rrdef.h" #ifdef UNBOUND_ALLOC_LITE #undef malloc #undef calloc diff --git a/external/unbound/testcode/delayer.c b/external/unbound/testcode/delayer.c index 7a90fc04c..050cbd23e 100644 --- a/external/unbound/testcode/delayer.c +++ b/external/unbound/testcode/delayer.c @@ -50,7 +50,7 @@ #include <sys/time.h> #include "util/net_help.h" #include "util/config_file.h" -#include "ldns/sbuffer.h" +#include "sldns/sbuffer.h" #include <signal.h> /** number of reads per select for delayer */ diff --git a/external/unbound/testcode/do-tests.sh b/external/unbound/testcode/do-tests.sh index 84d2ef566..6ea12cd2f 100755 --- a/external/unbound/testcode/do-tests.sh +++ b/external/unbound/testcode/do-tests.sh @@ -14,7 +14,7 @@ NEED_NOMINGW='tcp_sigpipe.tpkg 07-confroot.tpkg 08-host-lib.tpkg fwd_ancil.tpkg' test_tool_avail "dig" test_tool_avail "ldns-testns" -# test for ipv6, uses streamptcp peculiarity. +# test for ipv6, uses streamtcp peculiarity. if ./streamtcp -f ::1 2>&1 | grep "not supported" >/dev/null 2>&1; then HAVE_IPV6=no else diff --git a/external/unbound/testcode/fake_event.c b/external/unbound/testcode/fake_event.c index de453aaa2..4335a8f22 100644 --- a/external/unbound/testcode/fake_event.c +++ b/external/unbound/testcode/fake_event.c @@ -60,9 +60,9 @@ #include "testcode/testpkts.h" #include "util/log.h" #include "util/fptr_wlist.h" -#include "ldns/sbuffer.h" -#include "ldns/wire2str.h" -#include "ldns/str2wire.h" +#include "sldns/sbuffer.h" +#include "sldns/wire2str.h" +#include "sldns/str2wire.h" #include <signal.h> struct worker; struct daemon_remote; diff --git a/external/unbound/testcode/perf.c b/external/unbound/testcode/perf.c index c51eee4b1..320cbc933 100644 --- a/external/unbound/testcode/perf.c +++ b/external/unbound/testcode/perf.c @@ -50,9 +50,9 @@ #include "util/data/msgencode.h" #include "util/data/msgreply.h" #include "util/data/msgparse.h" -#include "ldns/sbuffer.h" -#include "ldns/wire2str.h" -#include "ldns/str2wire.h" +#include "sldns/sbuffer.h" +#include "sldns/wire2str.h" +#include "sldns/str2wire.h" #include <sys/time.h> /** usage information for perf */ diff --git a/external/unbound/testcode/pktview.c b/external/unbound/testcode/pktview.c index e59283fa8..12e0d8edb 100644 --- a/external/unbound/testcode/pktview.c +++ b/external/unbound/testcode/pktview.c @@ -45,8 +45,8 @@ #include "util/data/msgparse.h" #include "testcode/unitmain.h" #include "testcode/readhex.h" -#include "ldns/sbuffer.h" -#include "ldns/parseutil.h" +#include "sldns/sbuffer.h" +#include "sldns/parseutil.h" /** usage information for pktview */ static void usage(char* argv[]) diff --git a/external/unbound/testcode/readhex.c b/external/unbound/testcode/readhex.c index d9aba09b7..e871def0e 100644 --- a/external/unbound/testcode/readhex.c +++ b/external/unbound/testcode/readhex.c @@ -41,8 +41,8 @@ #include <ctype.h> #include "testcode/readhex.h" #include "util/log.h" -#include "ldns/sbuffer.h" -#include "ldns/parseutil.h" +#include "sldns/sbuffer.h" +#include "sldns/parseutil.h" /** skip whitespace */ static void diff --git a/external/unbound/testcode/replay.c b/external/unbound/testcode/replay.c index 5c1197146..01b17a7f7 100644 --- a/external/unbound/testcode/replay.c +++ b/external/unbound/testcode/replay.c @@ -50,7 +50,7 @@ #include "testcode/replay.h" #include "testcode/testpkts.h" #include "testcode/fake_event.h" -#include "ldns/str2wire.h" +#include "sldns/str2wire.h" /** max length of lines in file */ #define MAX_LINE_LEN 10240 diff --git a/external/unbound/testcode/streamtcp.c b/external/unbound/testcode/streamtcp.c index d93ab966d..c5919428a 100644 --- a/external/unbound/testcode/streamtcp.c +++ b/external/unbound/testcode/streamtcp.c @@ -51,9 +51,9 @@ #include "util/data/msgparse.h" #include "util/data/msgreply.h" #include "util/data/dname.h" -#include "ldns/sbuffer.h" -#include "ldns/str2wire.h" -#include "ldns/wire2str.h" +#include "sldns/sbuffer.h" +#include "sldns/str2wire.h" +#include "sldns/wire2str.h" #include <openssl/ssl.h> #include <openssl/rand.h> #include <openssl/err.h> diff --git a/external/unbound/testcode/testbed.sh b/external/unbound/testcode/testbed.sh deleted file mode 100755 index 62ce2059b..000000000 --- a/external/unbound/testcode/testbed.sh +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env bash -# Testbed for NSD. -# By Wouter Wijngaards, NLnet Labs, 2006. -# BSD License. - -# this version prefers gmake if available. -# adds variable LDNS for the LDNS path to use. - -# global settings -CONFIGURE_FLAGS="" -REPORT_FILE=testdata/testbed.report -LOG_FILE=testdata/testbed.log -HOST_FILE=testdata/host_file.$USER - -if test ! -f $HOST_FILE; then - echo "No such file: $HOST_FILE" - exit 1 -fi - -function echossh() # like ssh but echos. -{ - echo "> ssh $*" - ssh $* -} - -# Compile and run NSD on platforms -function dotest() -# parameters: <host> <dir> -# host is name of ssh host -# dir is directory of nsd trunk on host -{ - echo "$1 begin on "`date` | tee -a $REPORT_FILE - - DISABLE="" - if test $IP6 = no; then - DISABLE="--disable-ipv6" - fi - if test x$LDNS != x; then - DISABLE="--with-ldns=$LDNS $DISABLE" - fi - if test x$LIBEVENT != x; then - DISABLE="--with-libevent=$LIBEVENT $DISABLE" - fi - - cat >makeconf.mak.$$ << EOF -#configure: configure.ac -# $AC_CMD -# touch configure -Makefile: Makefile.in #configure - ./configure $CONFIGURE_FLAGS $DISABLE - touch Makefile -EOF - scp makeconf.mak.$$ $1:$2 - # determine make to use - tempx=`ssh $1 "cd $2; which gmake"` - MAKE_CMD=`ssh $1 "cd $2; if test -f '$tempx'; then echo $tempx; else echo $MAKE_CMD; fi"` - - if test $SVN = yes; then - echossh $1 "cd $2; svn up" - echossh $1 "cd $2; $MAKE_CMD -f makeconf.mak.$$ configure" - else - # svn and autoconf locally - echo "fake svn via svnexport, tar, autoconf, bison, flex." - svn export svn+ssh://open.nlnetlabs.nl/svn/nsd/trunk unbound_ttt - (cd unbound_ttt; $AC_CMD; rm -r autom4te* .c-mode-rc.el .cvsignore) - if test $FIXCONFIGURE = yes; then - echo fixing up configure length test. - (cd unbound_ttt; mv configure oldconf; sed -e 's?while (test "X"?lt_cv_sys_max_cmd_len=65500; echo skip || while (test "X"?' <oldconf >configure; chmod +x ./configure) - fi - du unbound_ttt - rsync -vrcpz --rsync-path=/home/wouter/bin/rsync unbound_ttt $1:unbound_ttt - # tar czf unbound_ttt.tgz unbound_ttt - rm -rf unbound_ttt - # ls -al unbound_ttt.tgz - # scp unbound_ttt.tgz $1:unbound_ttt.tar.gz - # rm unbound_ttt.tgz - # echossh $1 "gtar xzf unbound_ttt.tar.gz && rm unbound_ttt.tar.gz" - fi - echossh $1 "cd $2; $MAKE_CMD -f makeconf.mak.$$ Makefile" - echossh $1 "cd $2; $MAKE_CMD all tests" - echossh $1 "cd $2; $MAKE_CMD doc" - if test $RUN_TEST = yes; then - echossh $1 "cd $2; bash testcode/do-tests.sh" - echossh $1 "cd $2/testdata; sh ../testcode/mini_tpkg.sh -q report" | tee -a $REPORT_FILE - fi - echossh $1 "cd $2; rm -f makeconf.mak.$$" - rm -f makeconf.mak.$$ - echo "$1 end on "`date` | tee -a $REPORT_FILE -} - -echo "on "`date`" by $USER." > $REPORT_FILE -echo "on "`date`" by $USER." > $LOG_FILE - -# read host names -declare -a hostname desc dir vars -IFS=' ' -i=0 -while read a b c d; do - if echo $a | grep "^#" >/dev/null; then - continue # skip it - fi - # append after arrays - hostname[$i]=$a - desc[$i]=$b - dir[$i]=$c - vars[$i]=$d - i=$(($i+1)) -done <$HOST_FILE -echo "testing on $i hosts" - -# do the test -for((i=0; i<${#hostname[*]}; i=$i+1)); do - if echo ${hostname[$i]} | grep "^#" >/dev/null; then - continue # skip it - fi - # echo "hostname=[${hostname[$i]}]" - # echo "desc=[${desc[$i]}]" - # echo "dir=[${dir[$i]}]" - # echo "vars=[${vars[$i]}]" - AC_CMD="libtoolize -c --force; autoconf && autoheader" - MAKE_CMD="make" - SVN=yes - IP6=yes - FIXCONFIGURE=no - RUN_TEST=yes - LDNS= - LIBEVENT= - eval ${vars[$i]} - echo "*** ${hostname[$i]} ${desc[$i]} ***" | tee -a $LOG_FILE | tee -a $REPORT_FILE - dotest ${hostname[$i]} ${dir[$i]} 2>&1 | tee -a $LOG_FILE -done - -echo "done" diff --git a/external/unbound/testcode/testbed.txt b/external/unbound/testcode/testbed.txt deleted file mode 100644 index b0175049e..000000000 --- a/external/unbound/testcode/testbed.txt +++ /dev/null @@ -1,38 +0,0 @@ -Testbed.sh help page. - -Testbed helps in running the test packages (using tpkg(1)) on several systems. -The script is specially written for unbound (edit it to change to different -software). It is licensed BSD. - -The hosts to run on are listed in host_file.<username>. You need to have -public-key authorized ssh access to these systems (or type your password lots -and lots of times). The host_file describes the directories and environment -of each host. You need only user-level access to the host. - -The host_file is very restrictive in formatting. Comments are lines starting -with the # mark. The entries must be separated by tabs. Please list the -hostname<tab>description<tab>checkoutdir<tab>variables - -hostname: network hostname to ssh to. -desc: pretty text to describe the machine architecture. -checkoutdir: directory on the remote host where a svn checkout is present. -variables: zero or more variables separated by spaces. BLA=value BAR=val. - -Only important variable for unbound is the LDNS=<dir> variable that if present -forces --with-ldns=<dir> to be passed to ./configure. In case LDNS is not -installed on the system itself, but present somewhere else. - -You can also set LIBEVENT=<dir> for the libevent directory, if it is -installed in a nonstandard location. - -*** Running the testbed - -Run by executing the script. It will take all the hosts from the file in -turn and update the svn directory there, possible autoreconf if necessary, -possibly ./configure <args> if necessary, make the executables. -Then it will run the testcode/do-tests script. This script should execute -the tests that this host is capable of running. - -in testdata/testbed.log has a line-by-line log. See your make errors here. -in testdata/testbed.report has only the tpkg reports. Summary. - diff --git a/external/unbound/testcode/testbound.c b/external/unbound/testcode/testbound.c index daf8ddd41..fa361c4ea 100644 --- a/external/unbound/testcode/testbound.c +++ b/external/unbound/testcode/testbound.c @@ -47,7 +47,7 @@ #include "testcode/fake_event.h" #include "daemon/remote.h" #include "util/config_file.h" -#include "ldns/keyraw.h" +#include "sldns/keyraw.h" #include <ctype.h> /** signal that this is a testbound compile */ diff --git a/external/unbound/testcode/testpkts.c b/external/unbound/testcode/testpkts.c index a494d9f01..d1960a410 100644 --- a/external/unbound/testcode/testpkts.c +++ b/external/unbound/testcode/testpkts.c @@ -27,11 +27,11 @@ struct sockaddr_storage; #include <ctype.h> #include "testcode/testpkts.h" #include "util/net_help.h" -#include "ldns/sbuffer.h" -#include "ldns/rrdef.h" -#include "ldns/pkthdr.h" -#include "ldns/str2wire.h" -#include "ldns/wire2str.h" +#include "sldns/sbuffer.h" +#include "sldns/rrdef.h" +#include "sldns/pkthdr.h" +#include "sldns/str2wire.h" +#include "sldns/wire2str.h" /** max size of a packet */ #define MAX_PACKETLEN 65536 diff --git a/external/unbound/testcode/unitanchor.c b/external/unbound/testcode/unitanchor.c index 8047eb2bf..8819c5ab6 100644 --- a/external/unbound/testcode/unitanchor.c +++ b/external/unbound/testcode/unitanchor.c @@ -43,8 +43,8 @@ #include "util/data/dname.h" #include "testcode/unitmain.h" #include "validator/val_anchor.h" -#include "ldns/sbuffer.h" -#include "ldns/rrdef.h" +#include "sldns/sbuffer.h" +#include "sldns/rrdef.h" /** test empty set */ static void diff --git a/external/unbound/testcode/unitdname.c b/external/unbound/testcode/unitdname.c index 83d829fae..238c3edf7 100644 --- a/external/unbound/testcode/unitdname.c +++ b/external/unbound/testcode/unitdname.c @@ -42,8 +42,8 @@ #include "util/log.h" #include "testcode/unitmain.h" #include "util/data/dname.h" -#include "ldns/sbuffer.h" -#include "ldns/str2wire.h" +#include "sldns/sbuffer.h" +#include "sldns/str2wire.h" /** put dname into buffer */ static sldns_buffer* diff --git a/external/unbound/testcode/unitldns.c b/external/unbound/testcode/unitldns.c index 65170a825..e27e46eaa 100644 --- a/external/unbound/testcode/unitldns.c +++ b/external/unbound/testcode/unitldns.c @@ -41,9 +41,9 @@ #include "config.h" #include "util/log.h" #include "testcode/unitmain.h" -#include "ldns/sbuffer.h" -#include "ldns/str2wire.h" -#include "ldns/wire2str.h" +#include "sldns/sbuffer.h" +#include "sldns/str2wire.h" +#include "sldns/wire2str.h" /** verbose this unit test */ static int vbmp = 0; diff --git a/external/unbound/testcode/unitmain.c b/external/unbound/testcode/unitmain.c index 4673214ad..09ebba329 100644 --- a/external/unbound/testcode/unitmain.c +++ b/external/unbound/testcode/unitmain.c @@ -61,8 +61,8 @@ #include "nss.h" #endif -#include "ldns/rrdef.h" -#include "ldns/keyraw.h" +#include "sldns/rrdef.h" +#include "sldns/keyraw.h" #include "util/log.h" #include "testcode/unitmain.h" diff --git a/external/unbound/testcode/unitmsgparse.c b/external/unbound/testcode/unitmsgparse.c index b33a2408d..ba4e234c8 100644 --- a/external/unbound/testcode/unitmsgparse.c +++ b/external/unbound/testcode/unitmsgparse.c @@ -51,9 +51,9 @@ #include "util/net_help.h" #include "testcode/readhex.h" #include "testcode/testpkts.h" -#include "ldns/sbuffer.h" -#include "ldns/str2wire.h" -#include "ldns/wire2str.h" +#include "sldns/sbuffer.h" +#include "sldns/str2wire.h" +#include "sldns/wire2str.h" /** verbose message parse unit test */ static int vbmp = 0; diff --git a/external/unbound/testcode/unitneg.c b/external/unbound/testcode/unitneg.c index d39684095..36fa6b906 100644 --- a/external/unbound/testcode/unitneg.c +++ b/external/unbound/testcode/unitneg.c @@ -45,7 +45,7 @@ #include "util/data/dname.h" #include "testcode/unitmain.h" #include "validator/val_neg.h" -#include "ldns/rrdef.h" +#include "sldns/rrdef.h" /** verbose unit test for negative cache */ static int negverbose = 0; diff --git a/external/unbound/testcode/unitverify.c b/external/unbound/testcode/unitverify.c index 2074f3c4e..078af0a9c 100644 --- a/external/unbound/testcode/unitverify.c +++ b/external/unbound/testcode/unitverify.c @@ -56,10 +56,10 @@ #include "util/net_help.h" #include "util/module.h" #include "util/config_file.h" -#include "ldns/sbuffer.h" -#include "ldns/keyraw.h" -#include "ldns/str2wire.h" -#include "ldns/wire2str.h" +#include "sldns/sbuffer.h" +#include "sldns/keyraw.h" +#include "sldns/str2wire.h" +#include "sldns/wire2str.h" /** verbose signature test */ static int vsig = 0; diff --git a/external/unbound/testdata/ctrl_pipe.tpkg b/external/unbound/testdata/ctrl_pipe.tpkg Binary files differnew file mode 100644 index 000000000..877fcf901 --- /dev/null +++ b/external/unbound/testdata/ctrl_pipe.tpkg diff --git a/external/unbound/testdata/fwd_capsid_strip.tpkg b/external/unbound/testdata/fwd_capsid_strip.tpkg Binary files differnew file mode 100644 index 000000000..c0be8a3c5 --- /dev/null +++ b/external/unbound/testdata/fwd_capsid_strip.tpkg diff --git a/external/unbound/testdata/val_spurious_ns.rpl b/external/unbound/testdata/val_spurious_ns.rpl new file mode 100644 index 000000000..741fd1aff --- /dev/null +++ b/external/unbound/testdata/val_spurious_ns.rpl @@ -0,0 +1,151 @@ +; config options +; The island of trust is at example.com +server: + trust-anchor: "example.com. 3600 IN DS 2854 3 1 46e4ffc6e9a4793b488954bd3f0cc6af0dfb201b" + val-override-date: "20070916134226" + target-fetch-policy: "0 0 0 0 0" + +stub-zone: + name: "." + stub-addr: 193.0.14.129 # K.ROOT-SERVERS.NET. +CONFIG_END + +SCENARIO_BEGIN Test validator with spurious unsigned NS in auth section + +; K.ROOT-SERVERS.NET. +RANGE_BEGIN 0 100 + ADDRESS 193.0.14.129 +ENTRY_BEGIN +MATCH opcode qtype qname +ADJUST copy_id +REPLY QR NOERROR +SECTION QUESTION +. IN NS +SECTION ANSWER +. IN NS K.ROOT-SERVERS.NET. +SECTION ADDITIONAL +K.ROOT-SERVERS.NET. IN A 193.0.14.129 +ENTRY_END + +ENTRY_BEGIN +MATCH opcode qtype qname +ADJUST copy_id +REPLY QR NOERROR +SECTION QUESTION +www.example.com. IN A +SECTION AUTHORITY +com. IN NS a.gtld-servers.net. +SECTION ADDITIONAL +a.gtld-servers.net. IN A 192.5.6.30 +ENTRY_END +RANGE_END + +; a.gtld-servers.net. +RANGE_BEGIN 0 100 + ADDRESS 192.5.6.30 +ENTRY_BEGIN +MATCH opcode qtype qname +ADJUST copy_id +REPLY QR NOERROR +SECTION QUESTION +com. IN NS +SECTION ANSWER +com. IN NS a.gtld-servers.net. +SECTION ADDITIONAL +a.gtld-servers.net. IN A 192.5.6.30 +ENTRY_END + +ENTRY_BEGIN +MATCH opcode qtype qname +ADJUST copy_id +REPLY QR NOERROR +SECTION QUESTION +www.example.com. IN A +SECTION AUTHORITY +example.com. IN NS ns.example.com. +SECTION ADDITIONAL +ns.example.com. IN A 1.2.3.4 +ENTRY_END +RANGE_END + +; ns.example.com. +RANGE_BEGIN 0 100 + ADDRESS 1.2.3.4 +ENTRY_BEGIN +MATCH opcode qtype qname +ADJUST copy_id +REPLY QR NOERROR +SECTION QUESTION +example.com. IN NS +SECTION ANSWER +example.com. IN NS ns.example.com. +example.com. 3600 IN RRSIG NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854} +SECTION ADDITIONAL +ns.example.com. IN A 1.2.3.4 +ns.example.com. 3600 IN RRSIG A 3 3 3600 20070926135752 20070829135752 2854 example.com. MC0CFQCMSWxVehgOQLoYclB9PIAbNP229AIUeH0vNNGJhjnZiqgIOKvs1EhzqAo= ;{id = 2854} +ENTRY_END + +; response to DNSKEY priming query +ENTRY_BEGIN +MATCH opcode qtype qname +ADJUST copy_id +REPLY QR NOERROR +SECTION QUESTION +example.com. IN DNSKEY +SECTION ANSWER +example.com. 3600 IN DNSKEY 256 3 3 ALXLUsWqUrY3JYER3T4TBJII s70j+sDS/UT2QRp61SE7S3E EXopNXoFE73JLRmvpi/UrOO/Vz4Se 6wXv/CYCKjGw06U4WRgR YXcpEhJROyNapmdIKSx hOzfLVE1gqA0PweZR8d tY3aNQSRn3sPpwJr6Mi /PqQKAMMrZ9ckJpf1+b QMOOvxgzz2U1GS18b3y ZKcgTMEaJzd/GZYzi/B N2DzQ0MsrSwYXfsNLFO Bbs8PJMW4LYIxeeOe6rUgkWOF 7CC9Dh/dduQ1QrsJhmZAEFfd6ByYV+ ;{id = 2854 (zsk), size = 1688b} +example.com. 3600 IN RRSIG DNSKEY 3 2 3600 20070926134802 20070829134802 2854 example.com. MCwCFG1yhRNtTEa3Eno2zhVVuy2EJX3wAhQeLyUp6+UXcpC5qGNu9tkrTEgPUg== ;{id = 2854} +SECTION AUTHORITY +example.com. IN NS ns.example.com. +example.com. 3600 IN RRSIG NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854} +SECTION ADDITIONAL +ns.example.com. IN A 1.2.3.4 +ns.example.com. 3600 IN RRSIG A 3 3 3600 20070926135752 20070829135752 2854 example.com. MC0CFQCMSWxVehgOQLoYclB9PIAbNP229AIUeH0vNNGJhjnZiqgIOKvs1EhzqAo= ;{id = 2854} +ENTRY_END + +; response to query of interest +ENTRY_BEGIN +MATCH opcode qtype qname +ADJUST copy_id +REPLY QR NOERROR +SECTION QUESTION +www.example.com. IN A +SECTION ANSWER +www.example.com. IN A 10.20.30.40 +ns.example.com. 3600 IN RRSIG A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCQMyTjn7WWwpwAR1LlVeLpRgZGuQIUCcJDEkwAuzytTDRlYK7nIMwH1CM= ;{id = 2854} +SECTION AUTHORITY +example.com. IN NS ns.example.com. +;example.com. 3600 IN RRSIG NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854} +SECTION ADDITIONAL +ns.example.com. IN A 1.2.3.4 +www.example.com. 3600 IN RRSIG A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFC99iE9K5y2WNgI0gFvBWaTi9wm6AhUAoUqOpDtG5Zct+Qr9F3mSdnbc6V4= ;{id = 2854} +ENTRY_END +RANGE_END + +STEP 1 QUERY +ENTRY_BEGIN +REPLY RD DO +SECTION QUESTION +www.example.com. IN A +ENTRY_END + +; recursion happens here. +STEP 10 CHECK_ANSWER +ENTRY_BEGIN +MATCH all +REPLY QR RD RA AD DO NOERROR +SECTION QUESTION +www.example.com. IN A +SECTION ANSWER +www.example.com. IN A 10.20.30.40 +www.example.com. 3600 IN RRSIG A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFC99iE9K5y2WNgI0gFvBWaTi9wm6AhUAoUqOpDtG5Zct+Qr9F3mSdnbc6V4= ;{id = 2854} +SECTION AUTHORITY +; removed by spurious NS record removal code +;;example.com. IN NS ns.example.com. +;;example.com. 3600 IN RRSIG NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854} +SECTION ADDITIONAL +ns.example.com. IN A 1.2.3.4 +ns.example.com. 3600 IN RRSIG A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCQMyTjn7WWwpwAR1LlVeLpRgZGuQIUCcJDEkwAuzytTDRlYK7nIMwH1CM= ;{id = 2854} +ENTRY_END + +SCENARIO_END diff --git a/external/unbound/testdata/val_ta_algo_dnskey_dp.rpl b/external/unbound/testdata/val_ta_algo_dnskey_dp.rpl new file mode 100644 index 000000000..b23c0f1b0 --- /dev/null +++ b/external/unbound/testdata/val_ta_algo_dnskey_dp.rpl @@ -0,0 +1,182 @@ +; config options +; The island of trust is at example.com +server: + trust-anchor: "example.com. 3600 IN DNSKEY 256 3 3 ALXLUsWqUrY3JYER3T4TBJIIs70j+sDS/UT2QRp61SE7S3EEXopNXoFE73JLRmvpi/UrOO/Vz4Se6wXv/CYCKjGw06U4WRgRYXcpEhJROyNapmdIKSxhOzfLVE1gqA0PweZR8dtY3aNQSRn3sPpwJr6Mi/PqQKAMMrZ9ckJpf1+bQMOOvxgzz2U1GS18b3yZKcgTMEaJzd/GZYzi/BN2DzQ0MsrSwYXfsNLFOBbs8PJMW4LYIxeeOe6rUgkWOF7CC9Dh/dduQ1QrsJhmZAEFfd6ByYV+ ;{id = 2854 (zsk), size = 1688b}" + trust-anchor: "example.com. 3600 IN DS 30899 5 1 d4bf9d2e10f6d76840d42ef5913022abcd0bf512" + val-override-date: "20070916134226" + target-fetch-policy: "0 0 0 0 0" + harden-algo-downgrade: no + +stub-zone: + name: "." + stub-addr: 193.0.14.129 # K.ROOT-SERVERS.NET. +CONFIG_END + +SCENARIO_BEGIN Test validator with multiple algorithm trust anchor without harden + +; K.ROOT-SERVERS.NET. +RANGE_BEGIN 0 100 + ADDRESS 193.0.14.129 +ENTRY_BEGIN +MATCH opcode qtype qname +ADJUST copy_id +REPLY QR NOERROR +SECTION QUESTION +. IN NS +SECTION ANSWER +. IN NS K.ROOT-SERVERS.NET. +SECTION ADDITIONAL +K.ROOT-SERVERS.NET. IN A 193.0.14.129 +ENTRY_END + +ENTRY_BEGIN +MATCH opcode qtype qname +ADJUST copy_id +REPLY QR NOERROR +SECTION QUESTION +www.example.com. IN A +SECTION AUTHORITY +com. IN NS a.gtld-servers.net. +SECTION ADDITIONAL +a.gtld-servers.net. IN A 192.5.6.30 +ENTRY_END +RANGE_END + +; a.gtld-servers.net. +RANGE_BEGIN 0 100 + ADDRESS 192.5.6.30 +ENTRY_BEGIN +MATCH opcode qtype qname +ADJUST copy_id +REPLY QR NOERROR +SECTION QUESTION +com. IN NS +SECTION ANSWER +com. IN NS a.gtld-servers.net. +SECTION ADDITIONAL +a.gtld-servers.net. IN A 192.5.6.30 +ENTRY_END + +ENTRY_BEGIN +MATCH opcode qtype qname +ADJUST copy_id +REPLY QR NOERROR +SECTION QUESTION +www.example.com. IN A +SECTION AUTHORITY +example.com. IN NS ns.example.com. +SECTION ADDITIONAL +ns.example.com. IN A 1.2.3.4 +ENTRY_END +RANGE_END + +; ns.example.com. +RANGE_BEGIN 0 100 + ADDRESS 1.2.3.4 +ENTRY_BEGIN +MATCH opcode qtype qname +ADJUST copy_id +REPLY QR NOERROR +SECTION QUESTION +example.com. IN NS +SECTION ANSWER +example.com. IN NS ns.example.com. +example.com. 3600 IN RRSIG NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854} +example.com. 3600 IN RRSIG NS 5 2 3600 20070926134150 20070829134150 30899 example.com. YTqtYba73HIOQuPr5oDyIX9pfmz1ybEBjwlD/jUgcPmFINUOZ9FeqG6ywgRKwn4AizkKTK00p1sxZYMKxl91wg== ;{id = 30899} +SECTION ADDITIONAL +ns.example.com. IN A 1.2.3.4 +ns.example.com. 3600 IN RRSIG A 3 3 3600 20070926135752 20070829135752 2854 example.com. MC0CFQCMSWxVehgOQLoYclB9PIAbNP229AIUeH0vNNGJhjnZiqgIOKvs1EhzqAo= ;{id = 2854} +ns.example.com. 3600 IN RRSIG A 5 3 3600 20070926134150 20070829134150 30899 example.com. Dn1ziMKrc3NdJkSv8g61Y9WNk3+BAuwCwnYzAZiHmkejkSCPViLJN7+f4Conp9l8LkTl50ZnLgoYrrUYNhMj6w== ;{id = 30899} +ENTRY_END + +ENTRY_BEGIN +MATCH opcode qtype qname +ADJUST copy_id +REPLY QR AA NOERROR +SECTION QUESTION +ns.example.com. IN AAAA +SECTION ANSWER +SECTION AUTHORITY +example.com. IN NS ns.example.com. +example.com. 3600 IN RRSIG NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854} +example.com. 3600 IN RRSIG NS 5 2 3600 20070926134150 20070829134150 30899 example.com. YTqtYba73HIOQuPr5oDyIX9pfmz1ybEBjwlD/jUgcPmFINUOZ9FeqG6ywgRKwn4AizkKTK00p1sxZYMKxl91wg== ;{id = 30899} +SECTION ADDITIONAL +ns.example.com. IN A 1.2.3.4 +ns.example.com. 3600 IN RRSIG A 3 3 3600 20070926135752 20070829135752 2854 example.com. MC0CFQCMSWxVehgOQLoYclB9PIAbNP229AIUeH0vNNGJhjnZiqgIOKvs1EhzqAo= ;{id = 2854} +ns.example.com. 3600 IN RRSIG A 5 3 3600 20070926134150 20070829134150 30899 example.com. Dn1ziMKrc3NdJkSv8g61Y9WNk3+BAuwCwnYzAZiHmkejkSCPViLJN7+f4Conp9l8LkTl50ZnLgoYrrUYNhMj6w== ;{id = 30899} +ENTRY_END + + +; response to DNSKEY priming query +ENTRY_BEGIN +MATCH opcode qtype qname +ADJUST copy_id +REPLY QR NOERROR +SECTION QUESTION +example.com. IN DNSKEY +SECTION ANSWER +example.com. 3600 IN DNSKEY 256 3 5 AQPQ41chR9DEHt/aIzIFAqanbDlRflJoRs5yz1jFsoRIT7dWf0r+PeDuewdxkszNH6wnU4QL8pfKFRh5PIYVBLK3 ;{id = 30899 (zsk), size = 512b} +example.com. 3600 IN DNSKEY 256 3 3 ALXLUsWqUrY3JYER3T4TBJIIs70j+sDS/UT2QRp61SE7S3EEXopNXoFE73JLRmvpi/UrOO/Vz4Se6wXv/CYCKjGw06U4WRgRYXcpEhJROyNapmdIKSxhOzfLVE1gqA0PweZR8dtY3aNQSRn3sPpwJr6Mi/PqQKAMMrZ9ckJpf1+bQMOOvxgzz2U1GS18b3yZKcgTMEaJzd/GZYzi/BN2DzQ0MsrSwYXfsNLFOBbs8PJMW4LYIxeeOe6rUgkWOF7CC9Dh/dduQ1QrsJhmZAEFfd6ByYV+ ;{id = 2854 (zsk), size = 512b} +example.com. 3600 IN RRSIG DNSKEY 3 2 3600 20070926134150 20070829134150 2854 example.com. AKIIYDOGHogglFqJK94ZtOnF7EfGikgAyloMNRSMCrQgFaFkmcOyjrc= ;{id = 2854} +example.com. 3600 IN RRSIG DNSKEY 5 2 3600 20070926134150 20070829134150 30899 example.com. J55fsz1GGMnngc4r50xvXDUdaVMlfcLKLVsfMhwNLF+ERac5XV/lLRAc/aSER+qQdsSo0CrjYjy1wat7YQpDAA== ;{id = 30899} +SECTION AUTHORITY +example.com. IN NS ns.example.com. +example.com. 3600 IN RRSIG NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854} +example.com. 3600 IN RRSIG NS 5 2 3600 20070926134150 20070829134150 30899 example.com. YTqtYba73HIOQuPr5oDyIX9pfmz1ybEBjwlD/jUgcPmFINUOZ9FeqG6ywgRKwn4AizkKTK00p1sxZYMKxl91wg== ;{id = 30899} +SECTION ADDITIONAL +ns.example.com. IN A 1.2.3.4 +ns.example.com. 3600 IN RRSIG A 3 3 3600 20070926135752 20070829135752 2854 example.com. MC0CFQCMSWxVehgOQLoYclB9PIAbNP229AIUeH0vNNGJhjnZiqgIOKvs1EhzqAo= ;{id = 2854} +ns.example.com. 3600 IN RRSIG A 5 3 3600 20070926134150 20070829134150 30899 example.com. Dn1ziMKrc3NdJkSv8g61Y9WNk3+BAuwCwnYzAZiHmkejkSCPViLJN7+f4Conp9l8LkTl50ZnLgoYrrUYNhMj6w== ;{id = 30899} +ENTRY_END + +; response to query of interest +ENTRY_BEGIN +MATCH opcode qtype qname +ADJUST copy_id +REPLY QR NOERROR +SECTION QUESTION +www.example.com. IN A +SECTION ANSWER +www.example.com. IN A 10.20.30.40 +ns.example.com. 3600 IN RRSIG A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCQMyTjn7WWwpwAR1LlVeLpRgZGuQIUCcJDEkwAuzytTDRlYK7nIMwH1CM= ;{id = 2854} +www.example.com. 3600 IN RRSIG A 5 3 3600 20070926134150 20070829134150 30899 example.com. JNWECShNE+nCLQwOXJJ3xpUkh2G+FCh5nk8uYAHIVQRse/BIvCMSlvRrtVyw9RnXvk5RR2bEgN0pRdLWW7ug5Q== ;{id = 30899} +SECTION AUTHORITY +example.com. IN NS ns.example.com. +example.com. 3600 IN RRSIG NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854} +example.com. 3600 IN RRSIG NS 5 2 3600 20070926134150 20070829134150 30899 example.com. YTqtYba73HIOQuPr5oDyIX9pfmz1ybEBjwlD/jUgcPmFINUOZ9FeqG6ywgRKwn4AizkKTK00p1sxZYMKxl91wg== ;{id = 30899} +SECTION ADDITIONAL +ns.example.com. IN A 1.2.3.4 +www.example.com. 3600 IN RRSIG A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFC99iE9K5y2WNgI0gFvBWaTi9wm6AhUAoUqOpDtG5Zct+Qr9F3mSdnbc6V4= ;{id = 2854} +ns.example.com. 3600 IN RRSIG A 5 3 3600 20070926134150 20070829134150 30899 example.com. Dn1ziMKrc3NdJkSv8g61Y9WNk3+BAuwCwnYzAZiHmkejkSCPViLJN7+f4Conp9l8LkTl50ZnLgoYrrUYNhMj6w== ;{id = 30899} +ENTRY_END +RANGE_END + +STEP 1 QUERY +ENTRY_BEGIN +REPLY RD DO +SECTION QUESTION +www.example.com. IN A +ENTRY_END + +; recursion happens here. +STEP 10 CHECK_ANSWER +ENTRY_BEGIN +MATCH all +REPLY QR RD RA AD DO NOERROR +SECTION QUESTION +www.example.com. IN A +SECTION ANSWER +www.example.com. IN A 10.20.30.40 +www.example.com. 3600 IN RRSIG A 5 3 3600 20070926134150 20070829134150 30899 example.com. JNWECShNE+nCLQwOXJJ3xpUkh2G+FCh5nk8uYAHIVQRse/BIvCMSlvRrtVyw9RnXvk5RR2bEgN0pRdLWW7ug5Q== ;{id = 30899} +www.example.com. 3600 IN RRSIG A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFC99iE9K5y2WNgI0gFvBWaTi9wm6AhUAoUqOpDtG5Zct+Qr9F3mSdnbc6V4= ;{id = 2854} +SECTION AUTHORITY +example.com. IN NS ns.example.com. +example.com. 3600 IN RRSIG NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854} +example.com. 3600 IN RRSIG NS 5 2 3600 20070926134150 20070829134150 30899 example.com. YTqtYba73HIOQuPr5oDyIX9pfmz1ybEBjwlD/jUgcPmFINUOZ9FeqG6ywgRKwn4AizkKTK00p1sxZYMKxl91wg== ;{id = 30899} +SECTION ADDITIONAL +ns.example.com. IN A 1.2.3.4 +ns.example.com. 3600 IN RRSIG A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCQMyTjn7WWwpwAR1LlVeLpRgZGuQIUCcJDEkwAuzytTDRlYK7nIMwH1CM= ;{id = 2854} +ns.example.com. 3600 IN RRSIG A 5 3 3600 20070926134150 20070829134150 30899 example.com. Dn1ziMKrc3NdJkSv8g61Y9WNk3+BAuwCwnYzAZiHmkejkSCPViLJN7+f4Conp9l8LkTl50ZnLgoYrrUYNhMj6w== ;{id = 30899} +ENTRY_END + +SCENARIO_END diff --git a/external/unbound/testdata/val_ta_algo_missing_dp.rpl b/external/unbound/testdata/val_ta_algo_missing_dp.rpl new file mode 100644 index 000000000..2cf0556f5 --- /dev/null +++ b/external/unbound/testdata/val_ta_algo_missing_dp.rpl @@ -0,0 +1,185 @@ +; config options +; The island of trust is at example.com +server: + trust-anchor: "example.com. 3600 IN DNSKEY 256 3 3 ALXLUsWqUrY3JYER3T4TBJIIs70j+sDS/UT2QRp61SE7S3EEXopNXoFE73JLRmvpi/UrOO/Vz4Se6wXv/CYCKjGw06U4WRgRYXcpEhJROyNapmdIKSxhOzfLVE1gqA0PweZR8dtY3aNQSRn3sPpwJr6Mi/PqQKAMMrZ9ckJpf1+bQMOOvxgzz2U1GS18b3yZKcgTMEaJzd/GZYzi/BN2DzQ0MsrSwYXfsNLFOBbs8PJMW4LYIxeeOe6rUgkWOF7CC9Dh/dduQ1QrsJhmZAEFfd6ByYV+ ;{id = 2854 (zsk), size = 1688b}" + trust-anchor: "example.com. 3600 IN DS 30899 5 1 d4bf9d2e10f6d76840d42ef5913022abcd0bf512" + trust-anchor: "example.com. 3600 IN DS 30899 7 1 d4bf9d2e10f6d76840d42ef5913022abcd0bf512" + val-override-date: "20070916134226" + target-fetch-policy: "0 0 0 0 0" + harden-algo-downgrade: no + +stub-zone: + name: "." + stub-addr: 193.0.14.129 # K.ROOT-SERVERS.NET. +CONFIG_END + +SCENARIO_BEGIN Test validator with multiple algorithm missing one + +; K.ROOT-SERVERS.NET. +RANGE_BEGIN 0 100 + ADDRESS 193.0.14.129 +ENTRY_BEGIN +MATCH opcode qtype qname +ADJUST copy_id +REPLY QR NOERROR +SECTION QUESTION +. IN NS +SECTION ANSWER +. IN NS K.ROOT-SERVERS.NET. +SECTION ADDITIONAL +K.ROOT-SERVERS.NET. IN A 193.0.14.129 +ENTRY_END + +ENTRY_BEGIN +MATCH opcode qtype qname +ADJUST copy_id +REPLY QR NOERROR +SECTION QUESTION +www.example.com. IN A +SECTION AUTHORITY +com. IN NS a.gtld-servers.net. +SECTION ADDITIONAL +a.gtld-servers.net. IN A 192.5.6.30 +ENTRY_END +RANGE_END + +; a.gtld-servers.net. +RANGE_BEGIN 0 100 + ADDRESS 192.5.6.30 +ENTRY_BEGIN +MATCH opcode qtype qname +ADJUST copy_id +REPLY QR NOERROR +SECTION QUESTION +com. IN NS +SECTION ANSWER +com. IN NS a.gtld-servers.net. +SECTION ADDITIONAL +a.gtld-servers.net. IN A 192.5.6.30 +ENTRY_END + +ENTRY_BEGIN +MATCH opcode qtype qname +ADJUST copy_id +REPLY QR NOERROR +SECTION QUESTION +www.example.com. IN A +SECTION AUTHORITY +example.com. IN NS ns.example.com. +SECTION ADDITIONAL +ns.example.com. IN A 1.2.3.4 +ENTRY_END +RANGE_END + +; ns.example.com. +RANGE_BEGIN 0 100 + ADDRESS 1.2.3.4 +ENTRY_BEGIN +MATCH opcode qtype qname +ADJUST copy_id +REPLY QR NOERROR +SECTION QUESTION +example.com. IN NS +SECTION ANSWER +example.com. IN NS ns.example.com. +example.com. 3600 IN RRSIG NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854} +example.com. 3600 IN RRSIG NS 5 2 3600 20070926134150 20070829134150 30899 example.com. YTqtYba73HIOQuPr5oDyIX9pfmz1ybEBjwlD/jUgcPmFINUOZ9FeqG6ywgRKwn4AizkKTK00p1sxZYMKxl91wg== ;{id = 30899} +SECTION ADDITIONAL +ns.example.com. IN A 1.2.3.4 +ns.example.com. 3600 IN RRSIG A 3 3 3600 20070926135752 20070829135752 2854 example.com. MC0CFQCMSWxVehgOQLoYclB9PIAbNP229AIUeH0vNNGJhjnZiqgIOKvs1EhzqAo= ;{id = 2854} +ns.example.com. 3600 IN RRSIG A 5 3 3600 20070926134150 20070829134150 30899 example.com. Dn1ziMKrc3NdJkSv8g61Y9WNk3+BAuwCwnYzAZiHmkejkSCPViLJN7+f4Conp9l8LkTl50ZnLgoYrrUYNhMj6w== ;{id = 30899} +ENTRY_END + +ENTRY_BEGIN +MATCH opcode qtype qname +ADJUST copy_id +REPLY QR AA NOERROR +SECTION QUESTION +ns.example.com. IN AAAA +SECTION ANSWER +SECTION AUTHORITY +example.com. IN NS ns.example.com. +example.com. 3600 IN RRSIG NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854} +example.com. 3600 IN RRSIG NS 5 2 3600 20070926134150 20070829134150 30899 example.com. YTqtYba73HIOQuPr5oDyIX9pfmz1ybEBjwlD/jUgcPmFINUOZ9FeqG6ywgRKwn4AizkKTK00p1sxZYMKxl91wg== ;{id = 30899} +SECTION ADDITIONAL +ns.example.com. IN A 1.2.3.4 +ns.example.com. 3600 IN RRSIG A 3 3 3600 20070926135752 20070829135752 2854 example.com. MC0CFQCMSWxVehgOQLoYclB9PIAbNP229AIUeH0vNNGJhjnZiqgIOKvs1EhzqAo= ;{id = 2854} +ns.example.com. 3600 IN RRSIG A 5 3 3600 20070926134150 20070829134150 30899 example.com. Dn1ziMKrc3NdJkSv8g61Y9WNk3+BAuwCwnYzAZiHmkejkSCPViLJN7+f4Conp9l8LkTl50ZnLgoYrrUYNhMj6w== ;{id = 30899} +ENTRY_END + + +; response to DNSKEY priming query +ENTRY_BEGIN +MATCH opcode qtype qname +ADJUST copy_id +REPLY QR NOERROR +SECTION QUESTION +example.com. IN DNSKEY +SECTION ANSWER +example.com. 3600 IN DNSKEY 256 3 5 AQPQ41chR9DEHt/aIzIFAqanbDlRflJoRs5yz1jFsoRIT7dWf0r+PeDuewdxkszNH6wnU4QL8pfKFRh5PIYVBLK3 ;{id = 30899 (zsk), size = 512b} +example.com. 3600 IN DNSKEY 256 3 3 ALXLUsWqUrY3JYER3T4TBJIIs70j+sDS/UT2QRp61SE7S3EEXopNXoFE73JLRmvpi/UrOO/Vz4Se6wXv/CYCKjGw06U4WRgRYXcpEhJROyNapmdIKSxhOzfLVE1gqA0PweZR8dtY3aNQSRn3sPpwJr6Mi/PqQKAMMrZ9ckJpf1+bQMOOvxgzz2U1GS18b3yZKcgTMEaJzd/GZYzi/BN2DzQ0MsrSwYXfsNLFOBbs8PJMW4LYIxeeOe6rUgkWOF7CC9Dh/dduQ1QrsJhmZAEFfd6ByYV+ ;{id = 2854 (zsk), size = 512b} +example.com. 3600 IN RRSIG DNSKEY 3 2 3600 20070926134150 20070829134150 2854 example.com. AKIIYDOGHogglFqJK94ZtOnF7EfGikgAyloMNRSMCrQgFaFkmcOyjrc= ;{id = 2854} +example.com. 3600 IN RRSIG DNSKEY 5 2 3600 20070926134150 20070829134150 30899 example.com. J55fsz1GGMnngc4r50xvXDUdaVMlfcLKLVsfMhwNLF+ERac5XV/lLRAc/aSER+qQdsSo0CrjYjy1wat7YQpDAA== ;{id = 30899} +SECTION AUTHORITY +example.com. IN NS ns.example.com. +example.com. 3600 IN RRSIG NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854} +example.com. 3600 IN RRSIG NS 5 2 3600 20070926134150 20070829134150 30899 example.com. YTqtYba73HIOQuPr5oDyIX9pfmz1ybEBjwlD/jUgcPmFINUOZ9FeqG6ywgRKwn4AizkKTK00p1sxZYMKxl91wg== ;{id = 30899} +SECTION ADDITIONAL +ns.example.com. IN A 1.2.3.4 +ns.example.com. 3600 IN RRSIG A 3 3 3600 20070926135752 20070829135752 2854 example.com. MC0CFQCMSWxVehgOQLoYclB9PIAbNP229AIUeH0vNNGJhjnZiqgIOKvs1EhzqAo= ;{id = 2854} +ns.example.com. 3600 IN RRSIG A 5 3 3600 20070926134150 20070829134150 30899 example.com. Dn1ziMKrc3NdJkSv8g61Y9WNk3+BAuwCwnYzAZiHmkejkSCPViLJN7+f4Conp9l8LkTl50ZnLgoYrrUYNhMj6w== ;{id = 30899} +ENTRY_END + +; response to query of interest +ENTRY_BEGIN +MATCH opcode qtype qname +ADJUST copy_id +REPLY QR NOERROR +SECTION QUESTION +www.example.com. IN A +SECTION ANSWER +www.example.com. IN A 10.20.30.40 +ns.example.com. 3600 IN RRSIG A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCQMyTjn7WWwpwAR1LlVeLpRgZGuQIUCcJDEkwAuzytTDRlYK7nIMwH1CM= ;{id = 2854} +www.example.com. 3600 IN RRSIG A 5 3 3600 20070926134150 20070829134150 30899 example.com. JNWECShNE+nCLQwOXJJ3xpUkh2G+FCh5nk8uYAHIVQRse/BIvCMSlvRrtVyw9RnXvk5RR2bEgN0pRdLWW7ug5Q== ;{id = 30899} +SECTION AUTHORITY +example.com. IN NS ns.example.com. +example.com. 3600 IN RRSIG NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854} +example.com. 3600 IN RRSIG NS 5 2 3600 20070926134150 20070829134150 30899 example.com. YTqtYba73HIOQuPr5oDyIX9pfmz1ybEBjwlD/jUgcPmFINUOZ9FeqG6ywgRKwn4AizkKTK00p1sxZYMKxl91wg== ;{id = 30899} +SECTION ADDITIONAL +ns.example.com. IN A 1.2.3.4 +www.example.com. 3600 IN RRSIG A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFC99iE9K5y2WNgI0gFvBWaTi9wm6AhUAoUqOpDtG5Zct+Qr9F3mSdnbc6V4= ;{id = 2854} +ns.example.com. 3600 IN RRSIG A 5 3 3600 20070926134150 20070829134150 30899 example.com. Dn1ziMKrc3NdJkSv8g61Y9WNk3+BAuwCwnYzAZiHmkejkSCPViLJN7+f4Conp9l8LkTl50ZnLgoYrrUYNhMj6w== ;{id = 30899} +ENTRY_END +RANGE_END + +STEP 1 QUERY +ENTRY_BEGIN +REPLY RD DO +SECTION QUESTION +www.example.com. IN A +ENTRY_END + +; recursion happens here. +STEP 10 CHECK_ANSWER +ENTRY_BEGIN +MATCH all +REPLY QR RD RA AD DO NOERROR +SECTION QUESTION +www.example.com. IN A +SECTION ANSWER +www.example.com. 3600 IN A 10.20.30.40 +www.example.com. 3600 IN RRSIG A 5 3 3600 20070926134150 20070829134150 30899 example.com. JNWECShNE+nCLQwOXJJ3xpUkh2G+FCh5nk8uYAHIVQRse/BIvCMSlvRrtVyw9RnXvk5RR2bEgN0pRdLWW7ug5Q== ;{id = 30899} +www.example.com. 3600 IN RRSIG A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFC99iE9K5y2WNgI0gFvBWaTi9wm6AhUAoUqOpDtG5Zct+Qr9F3mSdnbc6V4= ;{id = 2854} + +SECTION AUTHORITY +example.com. 3600 IN NS ns.example.com. +example.com. 3600 IN RRSIG NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854} +example.com. 3600 IN RRSIG NS 5 2 3600 20070926134150 20070829134150 30899 example.com. YTqtYba73HIOQuPr5oDyIX9pfmz1ybEBjwlD/jUgcPmFINUOZ9FeqG6ywgRKwn4AizkKTK00p1sxZYMKxl91wg== ;{id = 30899} + +SECTION ADDITIONAL +ns.example.com. 3600 IN A 1.2.3.4 +ns.example.com. 3600 IN RRSIG A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCQMyTjn7WWwpwAR1LlVeLpRgZGuQIUCcJDEkwAuzytTDRlYK7nIMwH1CM= ;{id = 2854} +ns.example.com. 3600 IN RRSIG A 5 3 3600 20070926134150 20070829134150 30899 example.com. Dn1ziMKrc3NdJkSv8g61Y9WNk3+BAuwCwnYzAZiHmkejkSCPViLJN7+f4Conp9l8LkTl50ZnLgoYrrUYNhMj6w== ;{id = 30899} +ENTRY_END + +SCENARIO_END diff --git a/external/unbound/util/alloc.c b/external/unbound/util/alloc.c index 4b81beb4c..66bdc7db9 100644 --- a/external/unbound/util/alloc.c +++ b/external/unbound/util/alloc.c @@ -367,8 +367,12 @@ void *unbound_stat_malloc(size_t size) /** calloc with stats */ void *unbound_stat_calloc(size_t nmemb, size_t size) { - size_t s = (nmemb*size==0)?(size_t)1:nmemb*size; - void* res = calloc(1, s+16); + size_t s; + void* res; + if(nmemb != 0 && INT_MAX/nmemb < size) + return NULL; /* integer overflow check */ + s = (nmemb*size==0)?(size_t)1:nmemb*size; + res = calloc(1, s+16); if(!res) return NULL; log_info("stat %p=calloc(%u, %u)", res+16, (unsigned)nmemb, (unsigned)size); unbound_mem_alloc += s; @@ -503,8 +507,12 @@ void *unbound_stat_malloc_lite(size_t size, const char* file, int line, void *unbound_stat_calloc_lite(size_t nmemb, size_t size, const char* file, int line, const char* func) { - size_t req = nmemb * size; - void* res = malloc(req+lite_pad*2+sizeof(size_t)); + size_t req; + void* res; + if(nmemb != 0 && INT_MAX/nmemb < size) + return NULL; /* integer overflow check */ + req = nmemb * size; + res = malloc(req+lite_pad*2+sizeof(size_t)); if(!res) return NULL; memmove(res, lite_pre, lite_pad); memmove(res+lite_pad, &req, sizeof(size_t)); diff --git a/external/unbound/util/alloc.h b/external/unbound/util/alloc.h index ffd605c5d..43fc30f98 100644 --- a/external/unbound/util/alloc.h +++ b/external/unbound/util/alloc.h @@ -177,8 +177,8 @@ void alloc_set_id_cleanup(struct alloc_cache* alloc, void (*cleanup)(void*), void* arg); #ifdef UNBOUND_ALLOC_LITE -# include <ldns/ldns.h> -# include <ldns/packet.h> +# include <sldns/ldns.h> +# include <sldns/packet.h> # ifdef HAVE_OPENSSL_SSL_H # include <openssl/ssl.h> # endif diff --git a/external/unbound/util/config_file.c b/external/unbound/util/config_file.c index 35bc6452a..9c6b43cab 100644 --- a/external/unbound/util/config_file.c +++ b/external/unbound/util/config_file.c @@ -55,11 +55,20 @@ #include "util/regional.h" #include "util/fptr_wlist.h" #include "util/data/dname.h" -#include "ldns/wire2str.h" -#include "ldns/parseutil.h" +#include "util/rtt.h" +#include "sldns/wire2str.h" +#include "sldns/parseutil.h" #ifdef HAVE_GLOB_H # include <glob.h> #endif +#ifdef HAVE_PWD_H +#include <pwd.h> +#endif + +/** from cfg username, after daemonise setup performed */ +uid_t cfg_uid = (uid_t)-1; +/** from cfg username, after daemonise setup performed */ +gid_t cfg_gid = (gid_t)-1; /** global config during parsing */ struct config_parser_state* cfg_parser = 0; @@ -126,6 +135,7 @@ config_create(void) cfg->prefetch_key = 0; cfg->infra_cache_slabs = 4; cfg->infra_cache_numhosts = 10000; + cfg->infra_cache_min_rtt = 50; cfg->delay_close = 0; if(!(cfg->outgoing_avail_ports = (int*)calloc(65536, sizeof(int)))) goto error_exit; @@ -146,6 +156,7 @@ config_create(void) cfg->so_rcvbuf = 0; cfg->so_sndbuf = 0; cfg->so_reuseport = 0; + cfg->ip_transparent = 0; cfg->num_ifs = 0; cfg->ifs = NULL; cfg->num_out_ifs = 0; @@ -159,6 +170,7 @@ config_create(void) cfg->harden_dnssec_stripped = 1; cfg->harden_below_nxdomain = 0; cfg->harden_referral_path = 0; + cfg->harden_algo_downgrade = 1; cfg->use_caps_bits_for_id = 0; cfg->private_address = NULL; cfg->private_domain = NULL; @@ -196,6 +208,7 @@ config_create(void) cfg->remote_control_enable = 0; cfg->control_ifs = NULL; cfg->control_port = UNBOUND_CONTROL_PORT; + cfg->remote_control_use_cert = 1; cfg->minimal_responses = 0; cfg->rrset_roundrobin = 0; cfg->max_udp_size = 4096; @@ -361,6 +374,7 @@ int config_set_option(struct config_file* cfg, const char* opt, else S_MEMSIZE("so-rcvbuf:", so_rcvbuf) else S_MEMSIZE("so-sndbuf:", so_sndbuf) else S_YNO("so-reuseport:", so_reuseport) + else S_YNO("ip-transparent:", ip_transparent) else S_MEMSIZE("rrset-cache-size:", rrset_cache_size) else S_POW2("rrset-cache-slabs:", rrset_cache_slabs) else S_YNO("prefetch:", prefetch) @@ -369,6 +383,10 @@ int config_set_option(struct config_file* cfg, const char* opt, { IS_NUMBER_OR_ZERO; cfg->max_ttl = atoi(val); MAX_TTL=(time_t)cfg->max_ttl;} else if(strcmp(opt, "cache-min-ttl:") == 0) { IS_NUMBER_OR_ZERO; cfg->min_ttl = atoi(val); MIN_TTL=(time_t)cfg->min_ttl;} + else if(strcmp(opt, "infra-cache-min-rtt:") == 0) { + IS_NUMBER_OR_ZERO; cfg->infra_cache_min_rtt = atoi(val); + RTT_MIN_TIMEOUT=cfg->infra_cache_min_rtt; + } else S_NUMBER_OR_ZERO("infra-host-ttl:", host_ttl) else S_POW2("infra-cache-slabs:", infra_cache_slabs) else S_SIZET_NONZERO("infra-cache-numhosts:", infra_cache_numhosts) @@ -389,6 +407,7 @@ int config_set_option(struct config_file* cfg, const char* opt, else S_YNO("harden-dnssec-stripped:", harden_dnssec_stripped) else S_YNO("harden-below-nxdomain:", harden_below_nxdomain) else S_YNO("harden-referral-path:", harden_referral_path) + else S_YNO("harden-algo-downgrade:", harden_algo_downgrade) else S_YNO("use-caps-for-id", use_caps_bits_for_id) else S_SIZET_OR_ZERO("unwanted-reply-threshold:", unwanted_threshold) else S_STRLIST("private-address:", private_address) @@ -437,7 +456,8 @@ int config_set_option(struct config_file* cfg, const char* opt, { IS_NUMBER_OR_ZERO; cfg->val_sig_skew_max = (int32_t)atoi(val); } else if (strcmp(opt, "outgoing-interface:") == 0) { char* d = strdup(val); - char** oi = (char**)malloc((cfg->num_out_ifs+1)*sizeof(char*)); + char** oi = + (char**)reallocarray(NULL, (size_t)cfg->num_out_ifs+1, sizeof(char*)); if(!d || !oi) { free(d); free(oi); return -1; } if(cfg->out_ifs && cfg->num_out_ifs) { memmove(oi, cfg->out_ifs, cfg->num_out_ifs*sizeof(char*)); @@ -609,6 +629,7 @@ config_get_option(struct config_file* cfg, const char* opt, else O_MEM(opt, "so-rcvbuf", so_rcvbuf) else O_MEM(opt, "so-sndbuf", so_sndbuf) else O_YNO(opt, "so-reuseport", so_reuseport) + else O_YNO(opt, "ip-transparent", ip_transparent) else O_MEM(opt, "rrset-cache-size", rrset_cache_size) else O_DEC(opt, "rrset-cache-slabs", rrset_cache_slabs) else O_YNO(opt, "prefetch-key", prefetch_key) @@ -617,6 +638,7 @@ config_get_option(struct config_file* cfg, const char* opt, else O_DEC(opt, "cache-min-ttl", min_ttl) else O_DEC(opt, "infra-host-ttl", host_ttl) else O_DEC(opt, "infra-cache-slabs", infra_cache_slabs) + else O_DEC(opt, "infra-cache-min-rtt", infra_cache_min_rtt) else O_MEM(opt, "infra-cache-numhosts", infra_cache_numhosts) else O_UNS(opt, "delay-close", delay_close) else O_YNO(opt, "do-ip4", do_ip4) @@ -646,6 +668,7 @@ config_get_option(struct config_file* cfg, const char* opt, else O_YNO(opt, "harden-dnssec-stripped", harden_dnssec_stripped) else O_YNO(opt, "harden-below-nxdomain", harden_below_nxdomain) else O_YNO(opt, "harden-referral-path", harden_referral_path) + else O_YNO(opt, "harden-algo-downgrade", harden_algo_downgrade) else O_YNO(opt, "use-caps-for-id", use_caps_bits_for_id) else O_DEC(opt, "unwanted-reply-threshold", unwanted_threshold) else O_YNO(opt, "do-not-query-localhost", donotquery_localhost) @@ -799,6 +822,7 @@ config_read(struct config_file* cfg, const char* filename, const char* chroot) errno=EINVAL; return 0; } + return 1; } @@ -981,7 +1005,7 @@ int cfg_condense_ports(struct config_file* cfg, int** avail) *avail = NULL; if(num == 0) return 0; - *avail = (int*)malloc(sizeof(int)*num); + *avail = (int*)reallocarray(NULL, (size_t)num, sizeof(int)); if(!*avail) return 0; for(i=0; i<65536; i++) { @@ -1181,12 +1205,29 @@ config_apply(struct config_file* config) { MAX_TTL = (time_t)config->max_ttl; MIN_TTL = (time_t)config->min_ttl; + RTT_MIN_TIMEOUT = config->infra_cache_min_rtt; EDNS_ADVERTISED_SIZE = (uint16_t)config->edns_buffer_size; MINIMAL_RESPONSES = config->minimal_responses; RRSET_ROUNDROBIN = config->rrset_roundrobin; log_set_time_asc(config->log_time_ascii); } +void config_lookup_uid(struct config_file* cfg) +{ +#ifdef HAVE_GETPWNAM + /* translate username into uid and gid */ + if(cfg->username && cfg->username[0]) { + struct passwd *pwd; + if((pwd = getpwnam(cfg->username)) != NULL) { + cfg_uid = pwd->pw_uid; + cfg_gid = pwd->pw_gid; + } + } +#else + (void)cfg; +#endif +} + /** * Calculate string length of full pathname in original filesys * @param fname: the path name to convert. diff --git a/external/unbound/util/config_file.h b/external/unbound/util/config_file.h index 49ffbdde4..a3479b28f 100644 --- a/external/unbound/util/config_file.h +++ b/external/unbound/util/config_file.h @@ -119,6 +119,8 @@ struct config_file { size_t infra_cache_slabs; /** max number of hosts in the infra cache */ size_t infra_cache_numhosts; + /** min value for infra cache rtt */ + int infra_cache_min_rtt; /** delay close of udp-timeouted ports, if 0 no delayclose. in msec */ int delay_close; @@ -134,6 +136,8 @@ struct config_file { size_t so_sndbuf; /** SO_REUSEPORT requested on port 53 sockets */ int so_reuseport; + /** IP_TRANSPARENT socket option requested on port 53 sockets */ + int ip_transparent; /** number of interfaces to open. If 0 default all interfaces. */ int num_ifs; @@ -171,6 +175,8 @@ struct config_file { int harden_below_nxdomain; /** harden the referral path, query for NS,A,AAAA and validate */ int harden_referral_path; + /** harden against algorithm downgrade */ + int harden_algo_downgrade; /** use 0x20 bits in query as random ID bits */ int use_caps_bits_for_id; /** strip away these private addrs from answers, no DNS Rebinding */ @@ -282,6 +288,8 @@ struct config_file { struct config_strlist* control_ifs; /** port number for the control port */ int control_port; + /** use certificates for remote control */ + int remote_control_use_cert; /** private key file for server */ char* server_key_file; /** certificate file for server */ @@ -339,6 +347,11 @@ struct config_file { int dnstap_log_forwarder_response_messages; }; +/** from cfg username, after daemonise setup performed */ +extern uid_t cfg_uid; +/** from cfg username, after daemonise setup performed */ +extern gid_t cfg_gid; + /** * Stub config options */ @@ -423,6 +436,12 @@ void config_delete(struct config_file* config); void config_apply(struct config_file* config); /** + * Find username, sets cfg_uid and cfg_gid. + * @param config: the config structure. + */ +void config_lookup_uid(struct config_file* config); + +/** * Set the given keyword to the given value. * @param config: where to store config * @param option: option name, including the ':' character. diff --git a/external/unbound/util/configlexer.c b/external/unbound/util/configlexer.c index 4a0380e6a..fcade1ac3 100644 --- a/external/unbound/util/configlexer.c +++ b/external/unbound/util/configlexer.c @@ -363,8 +363,8 @@ static void yy_fatal_error (yyconst char msg[] ); *yy_cp = '\0'; \ (yy_c_buf_p) = yy_cp; -#define YY_NUM_RULES 162 -#define YY_END_OF_BUFFER 163 +#define YY_NUM_RULES 166 +#define YY_END_OF_BUFFER 167 /* This struct is not used in this scanner, but its presence is necessary. */ struct yy_trans_info @@ -372,185 +372,190 @@ struct yy_trans_info flex_int32_t yy_verify; flex_int32_t yy_nxt; }; -static yyconst flex_int16_t yy_accept[1611] = +static yyconst flex_int16_t yy_accept[1655] = { 0, - 1, 1, 144, 144, 148, 148, 152, 152, 156, 156, - 1, 1, 163, 160, 1, 142, 142, 161, 2, 161, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 144, - 145, 145, 146, 161, 148, 149, 149, 150, 161, 155, - 152, 153, 153, 154, 161, 156, 157, 157, 158, 161, - 159, 143, 2, 147, 161, 159, 160, 0, 1, 2, - 2, 2, 2, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 144, 0, 148, 0, 155, 0, 152, 156, - 0, 159, 0, 2, 2, 159, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 159, 160, 160, 160, 160, 160, 160, - - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 159, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - - 160, 160, 160, 160, 160, 65, 160, 160, 160, 160, - 160, 6, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 159, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 159, 160, 160, 160, 160, 29, 160, 160, 160, - 160, 160, 160, 160, 160, 129, 160, 12, 13, 160, - 15, 14, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 122, - 160, 160, 160, 160, 160, 3, 160, 160, 160, 160, - - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 159, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 151, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 32, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 33, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 80, 151, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 79, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 63, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - - 160, 160, 20, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 30, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 31, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 22, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - - 160, 160, 160, 160, 26, 160, 27, 160, 160, 160, - 66, 160, 67, 160, 64, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 5, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 82, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 23, 160, 160, 160, - 160, 107, 106, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - - 160, 160, 34, 160, 160, 160, 160, 160, 160, 160, - 160, 69, 68, 160, 160, 160, 160, 160, 160, 160, - 103, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 50, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 54, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 105, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 4, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 100, 160, 160, 160, 160, 160, 160, - 160, 116, 101, 160, 127, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 21, 160, 160, 160, 160, - 71, 160, 72, 70, 160, 160, 160, 160, 160, 160, - 78, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 102, 160, 160, 160, 160, 126, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 62, 160, 160, - 160, 160, 160, 160, 160, 160, 28, 160, 160, 17, - - 160, 160, 160, 16, 160, 87, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 41, - 42, 160, 160, 160, 160, 160, 160, 130, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 73, 160, 160, 160, 160, 160, 77, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 81, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 121, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 91, 160, 95, 160, 160, 160, 160, 76, 160, - - 160, 114, 160, 160, 160, 128, 160, 160, 160, 160, - 160, 160, 160, 135, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 94, 160, 160, 160, 160, 43, - 44, 160, 49, 96, 160, 108, 104, 160, 160, 37, - 160, 98, 160, 160, 160, 160, 160, 7, 160, 61, - 113, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 83, 134, 160, 160, - 160, 160, 160, 160, 160, 160, 123, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 97, - - 160, 36, 38, 160, 160, 160, 160, 160, 60, 160, - 160, 160, 160, 117, 18, 19, 160, 160, 160, 160, - 160, 160, 160, 58, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 119, 160, 160, 160, 160, 160, 160, - 160, 160, 35, 160, 160, 160, 160, 160, 160, 11, - 160, 160, 160, 160, 160, 160, 160, 10, 160, 160, - 39, 160, 125, 118, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 90, 89, 160, 120, 115, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 45, 160, 124, 160, - - 160, 160, 160, 40, 160, 160, 160, 84, 86, 109, - 160, 160, 160, 88, 160, 160, 160, 160, 160, 160, - 160, 160, 131, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 24, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 133, 160, 160, 112, 160, 160, 160, 160, 160, - 160, 160, 25, 160, 9, 160, 160, 110, 51, 160, - 160, 160, 93, 160, 160, 160, 160, 160, 160, 132, - 74, 160, 160, 160, 53, 57, 52, 160, 46, 160, - 8, 160, 160, 92, 160, 160, 160, 160, 160, 160, - - 160, 160, 160, 56, 160, 47, 160, 111, 160, 160, - 85, 160, 160, 160, 160, 160, 160, 75, 55, 48, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 59, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 99, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 138, 160, 160, 160, 160, 160, - 160, 160, 160, 160, 160, 160, 160, 160, 136, 160, - - 139, 140, 160, 160, 160, 160, 160, 137, 141, 0 + 1, 1, 148, 148, 152, 152, 156, 156, 160, 160, + 1, 1, 167, 164, 1, 146, 146, 165, 2, 165, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 148, + 149, 149, 150, 165, 152, 153, 153, 154, 165, 159, + 156, 157, 157, 158, 165, 160, 161, 161, 162, 165, + 163, 147, 2, 151, 165, 163, 164, 0, 1, 2, + 2, 2, 2, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 148, 0, 152, 0, 159, 0, 156, 160, + 0, 163, 0, 2, 2, 163, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 163, 164, 164, 164, 164, 164, 164, + + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 163, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + + 164, 164, 164, 164, 164, 164, 164, 68, 164, 164, + 164, 164, 164, 6, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 163, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 163, 164, 164, 164, 164, 30, + 164, 164, 164, 164, 164, 164, 164, 164, 133, 164, + 12, 13, 164, 15, 14, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 126, 164, 164, 164, 164, 164, 3, + + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 163, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 155, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 33, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 34, 164, 164, 164, 164, 164, 164, 164, 164, 164, + + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 83, + 155, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 82, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 66, 164, + + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 20, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 31, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 32, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 22, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 26, 164, 27, 164, + 164, 164, 69, 164, 70, 164, 67, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 5, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 85, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 23, 164, 164, 164, 164, 164, 110, 109, 164, 164, + + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 35, 164, 164, + 164, 164, 164, 164, 164, 164, 72, 71, 164, 164, + 164, 164, 164, 164, 164, 106, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 52, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 56, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 108, 164, 164, + + 164, 164, 164, 164, 164, 164, 164, 4, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 103, 164, 164, 164, 164, 164, 164, 164, 119, + 164, 104, 164, 131, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 21, 164, 164, 164, 164, 74, + 164, 75, 73, 164, 164, 164, 164, 164, 164, 164, + 81, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 105, 164, 164, 164, 164, 130, 164, 164, + + 164, 164, 164, 164, 164, 164, 164, 164, 164, 65, + 164, 164, 164, 164, 164, 164, 164, 164, 28, 164, + 164, 17, 164, 164, 164, 16, 164, 90, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 42, 43, 164, 164, 164, 164, 164, 164, 164, + 134, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 76, 164, 164, 164, 164, 164, + 164, 80, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 84, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + + 125, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 94, 164, 98, + 164, 164, 164, 164, 79, 164, 164, 117, 164, 164, + 164, 164, 132, 164, 164, 164, 164, 164, 164, 164, + 139, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 97, 164, 164, 164, 164, 164, 44, 45, + 164, 29, 51, 99, 164, 111, 107, 164, 164, 38, + 164, 101, 164, 164, 164, 164, 164, 7, 164, 64, + 116, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + + 164, 164, 164, 164, 164, 164, 164, 86, 138, 164, + 164, 164, 164, 164, 164, 164, 164, 127, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 100, 164, 37, 39, 164, 164, 164, 164, + 164, 63, 164, 164, 164, 164, 121, 18, 19, 164, + 164, 164, 164, 164, 164, 164, 61, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 123, 120, 164, 164, + 164, 164, 164, 164, 164, 164, 36, 164, 164, 164, + 164, 164, 164, 164, 11, 164, 164, 164, 164, 164, + 164, 164, 164, 10, 164, 164, 40, 164, 129, 122, + + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 93, 92, 164, 124, 118, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 46, 164, 128, 164, 164, 164, + 164, 41, 164, 164, 164, 87, 89, 112, 164, 164, + 164, 91, 164, 164, 164, 164, 164, 164, 164, 164, + 135, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 24, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 137, 164, 164, 115, 164, 164, 164, 164, 164, + + 164, 164, 49, 164, 25, 164, 9, 164, 164, 113, + 53, 164, 164, 164, 96, 164, 164, 164, 164, 164, + 164, 136, 77, 164, 164, 164, 164, 55, 59, 54, + 164, 47, 164, 8, 164, 164, 95, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 60, 58, 164, 48, + 164, 114, 164, 164, 88, 164, 164, 164, 164, 164, + 164, 78, 57, 50, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 62, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + + 164, 164, 164, 164, 164, 164, 164, 164, 164, 102, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 142, 164, + 164, 164, 164, 164, 164, 164, 164, 164, 164, 164, + 164, 164, 140, 164, 143, 144, 164, 164, 164, 164, + 164, 141, 145, 0 } ; static yyconst flex_int32_t yy_ec[256] = @@ -593,373 +598,381 @@ static yyconst flex_int32_t yy_meta[40] = 1, 1, 1, 1, 1, 1, 1, 1, 1 } ; -static yyconst flex_int16_t yy_base[1625] = +static yyconst flex_int16_t yy_base[1669] = { 0, 0, 0, 37, 40, 44, 51, 63, 75, 56, 68, - 87, 108, 2844, 2315, 50, 3201, 3201, 3201, 129, 94, + 87, 108, 2501, 2145, 50, 3281, 3281, 3281, 129, 94, 70, 104, 130, 90, 92, 115, 127, 95, 84, 111, - 137, 148, 50, 150, 155, 157, 163, 171, 178, 2213, - 3201, 3201, 3201, 70, 2102, 3201, 3201, 3201, 42, 1930, - 1613, 3201, 3201, 3201, 195, 1340, 3201, 3201, 3201, 141, - 1156, 3201, 202, 3201, 206, 122, 1017, 212, 120, 0, + 137, 148, 50, 150, 155, 157, 163, 171, 178, 2001, + 3281, 3281, 3281, 70, 1900, 3281, 3281, 3281, 42, 1856, + 1448, 3281, 3281, 3281, 195, 1202, 3281, 3281, 3281, 141, + 1174, 3281, 202, 3281, 206, 122, 1044, 212, 120, 0, 223, 0, 0, 103, 147, 154, 158, 192, 199, 207, 208, 205, 209, 221, 218, 220, 224, 225, 229, 230, 231, 238, 251, 236, 247, 250, 237, 248, 256, 259, 167, 263, 254, 249, 264, 265, 271, 273, 274, 125, 275, 277, 284, 278, 281, 285, 288, 286, 289, 292, - 296, 298, 987, 308, 860, 315, 829, 326, 692, 342, + 296, 298, 1016, 308, 746, 315, 650, 326, 474, 368, 319, 301, 330, 334, 0, 327, 331, 337, 314, 305, 333, 335, 343, 340, 346, 349, 370, 350, 339, 352, - 49, 356, 354, 353, 361, 363, 362, 366, 364, 368, - 373, 377, 381, 392, 399, 386, 383, 401, 397, 405, - 408, 406, 182, 407, 409, 410, 385, 411, 414, 412, - 416, 418, 424, 420, 421, 422, 430, 428, 431, 441, - 444, 450, 433, 446, 448, 449, 456, 454, 453, 455, - - 461, 460, 463, 464, 474, 475, 477, 470, 472, 479, - 480, 481, 488, 489, 491, 493, 499, 495, 496, 497, - 500, 501, 504, 505, 507, 511, 517, 512, 508, 521, - 510, 516, 523, 534, 527, 538, 532, 533, 542, 540, - 545, 543, 546, 547, 558, 554, 555, 559, 556, 557, - 563, 571, 575, 565, 568, 569, 577, 579, 581, 599, - 585, 583, 587, 590, 589, 596, 610, 597, 606, 607, - 624, 603, 612, 625, 627, 628, 632, 631, 634, 636, - 635, 637, 638, 640, 641, 644, 645, 656, 659, 648, - 665, 662, 667, 664, 670, 593, 677, 673, 675, 674, - - 676, 646, 682, 678, 688, 3201, 690, 679, 685, 692, - 695, 3201, 696, 697, 698, 703, 702, 709, 705, 706, - 711, 714, 715, 719, 713, 720, 740, 722, 721, 731, - 735, 725, 733, 727, 743, 747, 749, 750, 751, 753, - 754, 755, 757, 758, 763, 774, 760, 768, 771, 778, - 779, 780, 782, 781, 787, 783, 794, 788, 798, 804, - 800, 806, 808, 814, 810, 811, 813, 820, 818, 819, - 812, 826, 823, 825, 830, 834, 827, 841, 837, 840, - 844, 847, 849, 850, 851, 853, 859, 857, 858, 866, - 868, 856, 864, 871, 872, 877, 884, 885, 881, 886, - - 892, 893, 888, 894, 897, 898, 899, 900, 901, 902, - 906, 908, 909, 917, 903, 913, 919, 925, 927, 928, - 910, 930, 931, 934, 933, 937, 941, 935, 945, 947, - 948, 949, 955, 957, 950, 959, 3201, 969, 963, 965, - 966, 970, 972, 951, 991, 3201, 973, 3201, 3201, 975, - 3201, 3201, 974, 979, 982, 994, 1014, 993, 981, 980, - 1001, 1007, 995, 1008, 997, 1019, 1022, 1023, 1011, 1015, - 1027, 1029, 1024, 1036, 1037, 1041, 1043, 1049, 1045, 1046, - 1047, 1050, 1051, 1056, 1053, 1063, 1057, 1064, 1066, 3201, - 1067, 1068, 1071, 1073, 1075, 3201, 1074, 1076, 1077, 1079, - - 1080, 1083, 1084, 1087, 1089, 1090, 1092, 1094, 1100, 1096, - 1097, 1110, 1117, 1114, 1104, 1113, 1115, 1119, 1121, 1129, - 1127, 1126, 1128, 1136, 1132, 1134, 1138, 1135, 1139, 1142, - 1145, 1146, 1170, 1147, 1148, 1149, 1153, 1154, 1157, 1158, - 1162, 1165, 1177, 1178, 1176, 1164, 1184, 1187, 1194, 1195, - 1197, 1191, 1155, 1199, 1204, 1206, 1208, 1188, 1211, 1212, - 3201, 1218, 1217, 1215, 1219, 1223, 1224, 1226, 1225, 1231, - 1227, 1237, 1229, 1245, 3201, 1230, 1239, 1233, 1241, 1250, - 1253, 1252, 1259, 1267, 3201, 1269, 1270, 1263, 1265, 1256, - 1272, 1273, 1277, 1278, 1279, 1281, 1284, 1287, 1289, 1288, - - 1292, 1291, 1293, 1296, 1300, 1299, 1302, 1301, 1305, 1303, - 1242, 1315, 1322, 1311, 1321, 1318, 1324, 1323, 1329, 1327, - 1330, 1328, 1331, 3201, 239, 1332, 1333, 1334, 1341, 1349, - 1350, 1343, 1351, 1353, 1352, 1359, 1342, 1363, 1360, 1364, - 1366, 1368, 1371, 1372, 1373, 1377, 1375, 1381, 1383, 1382, - 1384, 1385, 1391, 1394, 1392, 1396, 1397, 1398, 1399, 1400, - 1402, 1406, 1405, 1410, 3201, 1427, 1412, 1415, 1413, 1424, - 1435, 1428, 1431, 1432, 1442, 1438, 1440, 1443, 1449, 1446, - 1451, 1452, 1436, 1458, 1462, 1459, 1461, 1460, 1468, 3201, - 1463, 1470, 1471, 1472, 1473, 1475, 1477, 1480, 1481, 1488, - - 1482, 1494, 3201, 1496, 1490, 1484, 1501, 1502, 1505, 1507, - 1509, 1510, 1511, 1513, 1514, 1517, 1515, 1520, 1524, 3201, - 1528, 1532, 1529, 1540, 1536, 1525, 1537, 1539, 1541, 1542, - 1551, 1543, 1547, 1550, 1552, 1549, 1554, 1555, 1557, 1559, - 3201, 1571, 1572, 1574, 1414, 1562, 1583, 1556, 1575, 1558, - 1584, 1585, 1587, 1588, 1589, 1590, 1593, 1595, 1591, 1597, - 1596, 1599, 1600, 1608, 1598, 1617, 1618, 1619, 1601, 1623, - 1629, 1632, 1620, 3201, 1631, 1634, 1635, 1636, 1642, 1644, - 1638, 1640, 1645, 1646, 1647, 1656, 1648, 1650, 1654, 1658, - 1652, 1660, 1661, 1662, 1670, 1664, 1672, 1674, 1678, 1680, - - 1682, 1684, 1686, 1694, 3201, 1690, 3201, 1691, 1692, 1700, - 3201, 1702, 3201, 1704, 3201, 1707, 1711, 1699, 1697, 1705, - 1713, 1709, 1716, 1717, 1720, 1723, 1725, 1726, 1727, 1728, - 3201, 1729, 1734, 1731, 1735, 1738, 1739, 1740, 1744, 1756, - 1741, 1743, 1753, 1754, 3201, 1750, 1762, 1764, 1765, 1766, - 1773, 1770, 1771, 1777, 1767, 1778, 1788, 1786, 1779, 1789, - 1790, 1791, 1793, 1794, 1801, 1798, 1795, 1802, 1804, 1805, - 1806, 1807, 1814, 1811, 1812, 1815, 3201, 1823, 1821, 1817, - 1825, 3201, 3201, 1834, 1827, 1835, 1837, 1839, 1842, 1843, - 1845, 1849, 1847, 1850, 1851, 1856, 1857, 1858, 1859, 1862, - - 1860, 1866, 3201, 1872, 1868, 1878, 1869, 1865, 1887, 1879, - 1888, 3201, 3201, 1881, 1880, 1890, 1894, 1900, 1896, 1898, - 3201, 1899, 1901, 1902, 1904, 1906, 1914, 1903, 1910, 1918, - 1915, 1920, 1921, 1927, 1924, 1928, 1931, 1935, 1939, 3201, - 1940, 1942, 1943, 1945, 1944, 1947, 1948, 1951, 1952, 1957, - 1954, 1965, 1967, 1958, 1966, 1960, 1971, 1968, 1973, 1976, - 1983, 1981, 3201, 1987, 1986, 1994, 1990, 1992, 1995, 1998, - 1997, 1999, 2000, 2001, 2002, 2006, 2003, 2007, 2010, 3201, - 2013, 2018, 2012, 2024, 2025, 2014, 2032, 2026, 2028, 3201, - 2036, 2042, 2035, 2038, 2047, 2043, 2049, 2039, 2050, 2051, - - 2053, 2054, 2057, 2062, 2056, 2064, 2069, 2071, 2078, 2075, - 2081, 2082, 2079, 2083, 2089, 2086, 2087, 2088, 2092, 2099, - 2093, 2101, 2097, 3201, 2103, 2108, 2111, 2112, 2114, 2116, - 2104, 3201, 3201, 2120, 3201, 2121, 2124, 2125, 2126, 2128, - 2129, 2127, 2134, 2141, 2130, 3201, 2137, 2140, 2142, 2143, - 3201, 2154, 3201, 3201, 2147, 2155, 2156, 2159, 2162, 2166, - 3201, 2163, 2157, 2167, 2169, 2171, 2172, 2174, 2176, 2175, - 3201, 2180, 2179, 2178, 2188, 3201, 2187, 2193, 2182, 2189, - 2198, 2199, 2202, 2203, 2205, 2211, 2209, 3201, 2210, 2212, - 2216, 2218, 2219, 2220, 2223, 2224, 3201, 2233, 2234, 3201, - - 2227, 2217, 2235, 3201, 2241, 3201, 2242, 2243, 2244, 2245, - 2250, 2246, 2253, 2256, 2257, 2259, 2260, 2263, 2266, 3201, - 3201, 2267, 2271, 2277, 2268, 2279, 2282, 3201, 2265, 2283, - 2272, 2285, 2286, 2287, 2292, 2294, 2297, 2289, 2298, 2296, - 2299, 3201, 2302, 2306, 2307, 2309, 2310, 3201, 2312, 2313, - 2314, 2317, 2319, 2324, 2333, 2329, 2335, 2337, 2338, 2340, - 2344, 2341, 3201, 2345, 2347, 2348, 2352, 2353, 2349, 2356, - 2361, 2364, 2354, 2366, 3201, 2369, 2358, 2374, 2370, 2376, - 2378, 2372, 2380, 2381, 2382, 2385, 2386, 2389, 2390, 2396, - 2399, 3201, 2392, 3201, 2401, 2405, 2413, 2414, 3201, 2394, - - 2407, 3201, 2417, 2411, 2422, 3201, 2425, 2418, 2426, 2428, - 2429, 2430, 2433, 3201, 2431, 2436, 2438, 2441, 2442, 2444, - 2445, 2447, 2455, 2451, 3201, 2453, 2454, 2456, 2458, 3201, - 3201, 2466, 3201, 3201, 2471, 3201, 3201, 2463, 2473, 3201, - 2475, 3201, 2481, 2477, 2479, 2461, 2480, 3201, 2487, 3201, - 3201, 2484, 2488, 2482, 2492, 2494, 2499, 2501, 2491, 2502, - 2504, 2505, 2506, 2507, 2508, 2510, 2513, 2511, 2514, 2519, - 2515, 2521, 2523, 2524, 2525, 2527, 3201, 3201, 2531, 2536, - 2533, 2538, 2540, 2542, 2543, 2544, 3201, 2547, 2548, 2550, - 2552, 2553, 2556, 2559, 2564, 2573, 2560, 2566, 2567, 3201, - - 2569, 3201, 3201, 2574, 2576, 2581, 2577, 2582, 3201, 2584, - 2586, 2590, 2596, 3201, 3201, 3201, 2597, 2591, 2593, 2599, - 2601, 2603, 2607, 3201, 2608, 2609, 2611, 2610, 2618, 2620, - 2626, 2628, 2630, 3201, 2636, 2633, 2634, 2632, 2635, 2638, - 2640, 2612, 3201, 2644, 2642, 2643, 2648, 2653, 2649, 3201, - 2651, 2656, 2660, 2662, 2664, 2661, 2667, 3201, 2665, 2666, - 3201, 2677, 3201, 3201, 2668, 2680, 2682, 2684, 2687, 2690, - 2673, 2678, 2698, 2695, 2696, 3201, 3201, 2697, 3201, 3201, - 2699, 2701, 2702, 2704, 2705, 2708, 2709, 2710, 2711, 2712, - 2715, 2721, 2714, 2722, 2723, 2725, 3201, 2733, 3201, 2729, - - 2738, 2735, 2740, 3201, 2741, 2742, 2727, 3201, 3201, 3201, - 2746, 2743, 2753, 3201, 2755, 2756, 2757, 2758, 2759, 2766, - 2764, 2768, 3201, 2769, 2770, 2772, 2773, 2774, 2775, 2776, - 2781, 2779, 2780, 2792, 2793, 3201, 2796, 2783, 2787, 2801, - 2807, 2797, 2802, 2804, 2811, 2808, 2812, 2813, 2823, 2819, - 2822, 3201, 2826, 2815, 3201, 2827, 2828, 2830, 2839, 2843, - 2835, 2845, 3201, 2846, 3201, 2849, 2850, 3201, 3201, 2851, - 2853, 2856, 3201, 2857, 2854, 2858, 2860, 2863, 2864, 3201, - 3201, 2865, 2867, 2871, 3201, 3201, 3201, 2880, 3201, 2882, - 3201, 2888, 2868, 3201, 2873, 2890, 2870, 2881, 2876, 2893, - - 2894, 2892, 2901, 3201, 2903, 3201, 2905, 3201, 2906, 2907, - 3201, 2914, 2910, 2912, 2913, 2915, 2919, 3201, 3201, 3201, - 2916, 2921, 2923, 2925, 2922, 2926, 2927, 2928, 2930, 2937, - 2934, 2945, 2931, 2938, 2954, 2947, 3201, 2948, 2951, 2960, - 2962, 2958, 2964, 2959, 2965, 2966, 2967, 2968, 2969, 2971, - 2982, 2973, 2974, 2984, 2986, 2990, 2983, 2998, 2997, 2994, - 2999, 3000, 3008, 3004, 3007, 3201, 3005, 3006, 3009, 3011, - 3012, 3016, 3014, 3026, 3029, 3017, 3031, 3015, 3036, 3032, - 3037, 3040, 3041, 3042, 3201, 3043, 3045, 3047, 3049, 3051, - 3053, 3054, 3055, 3058, 3060, 3063, 3065, 3069, 3201, 3070, - - 3201, 3201, 3074, 3071, 3077, 3081, 3083, 3201, 3201, 3201, - 3109, 3116, 3123, 3130, 3137, 94, 3144, 3151, 3158, 3165, - 3172, 3179, 3186, 3193 + 49, 356, 354, 353, 361, 363, 362, 366, 372, 365, + 377, 386, 382, 394, 401, 388, 387, 403, 399, 407, + 410, 406, 182, 395, 411, 342, 412, 409, 414, 416, + 418, 419, 427, 423, 426, 425, 435, 429, 433, 441, + 436, 444, 447, 446, 450, 451, 457, 455, 453, 456, + + 462, 461, 464, 471, 476, 477, 479, 465, 475, 482, + 483, 484, 494, 488, 491, 495, 501, 497, 498, 499, + 505, 503, 506, 507, 508, 509, 515, 519, 513, 512, + 514, 527, 528, 530, 531, 536, 537, 538, 539, 543, + 545, 546, 549, 547, 553, 561, 557, 559, 560, 566, + 567, 569, 575, 571, 568, 572, 579, 581, 582, 584, + 602, 591, 587, 590, 574, 597, 599, 608, 600, 607, + 609, 615, 613, 623, 630, 626, 627, 631, 634, 635, + 636, 639, 638, 641, 643, 644, 646, 647, 657, 659, + 660, 670, 666, 667, 648, 673, 664, 674, 681, 679, + + 680, 682, 683, 684, 686, 688, 692, 3281, 695, 690, + 697, 698, 701, 3281, 699, 702, 703, 710, 704, 721, + 706, 708, 714, 723, 718, 720, 726, 728, 748, 731, + 729, 739, 737, 732, 757, 736, 759, 738, 743, 750, + 762, 741, 766, 767, 768, 769, 773, 775, 778, 779, + 772, 783, 780, 786, 788, 789, 793, 794, 800, 801, + 803, 806, 812, 809, 816, 822, 811, 818, 820, 826, + 825, 828, 819, 835, 831, 827, 834, 843, 838, 840, + 850, 848, 849, 847, 853, 855, 859, 860, 862, 866, + 863, 867, 872, 877, 865, 871, 879, 878, 886, 893, + + 894, 887, 895, 902, 903, 880, 904, 890, 900, 907, + 909, 910, 911, 912, 915, 916, 922, 919, 923, 921, + 928, 930, 932, 934, 936, 937, 940, 943, 941, 942, + 951, 952, 954, 956, 955, 958, 962, 957, 964, 3281, + 972, 971, 968, 974, 976, 977, 979, 1005, 3281, 981, + 3281, 3281, 982, 3281, 3281, 985, 983, 986, 993, 1028, + 997, 991, 987, 998, 1007, 1008, 1018, 1011, 1020, 1000, + 1031, 1022, 1013, 1023, 1037, 1038, 1034, 1025, 1048, 1051, + 1057, 1059, 1055, 1056, 1060, 1061, 1063, 1065, 1064, 1067, + 1072, 1078, 1074, 3281, 1076, 1075, 1082, 1077, 1084, 3281, + + 1085, 1086, 1088, 1090, 1094, 1097, 1096, 1098, 1100, 1101, + 1099, 1107, 1113, 1103, 1111, 1127, 1129, 1126, 1112, 1128, + 1132, 1114, 1133, 1141, 1139, 1138, 1144, 1148, 1145, 1147, + 1150, 1151, 1154, 1152, 1153, 1155, 1178, 1159, 1160, 1161, + 1164, 1162, 1168, 1170, 1172, 1173, 1180, 1191, 1190, 1192, + 1193, 1197, 1205, 1208, 1198, 1201, 1211, 1212, 1217, 1215, + 1219, 1221, 1222, 1229, 1226, 3281, 1236, 1237, 1223, 1232, + 1233, 1239, 1243, 1244, 1246, 1247, 1248, 1249, 1251, 1255, + 3281, 1254, 1256, 1257, 1261, 1262, 1264, 1270, 1275, 1282, + 3281, 1285, 1286, 1278, 1283, 1271, 1289, 1288, 1294, 1295, + + 1296, 1297, 1301, 1304, 1305, 1293, 1312, 1306, 1308, 1310, + 1316, 1317, 1318, 1319, 1325, 1104, 1320, 1326, 1336, 1333, + 1335, 1338, 1339, 1340, 1345, 1342, 1346, 1343, 1344, 3281, + 239, 1347, 1349, 1350, 1356, 1359, 1358, 1365, 1366, 1367, + 1368, 1371, 1373, 1374, 1376, 1377, 1378, 1381, 1385, 1382, + 1389, 1391, 1394, 1392, 1398, 1399, 1400, 1403, 1402, 1409, + 1406, 1411, 1412, 1416, 1417, 1414, 1418, 1420, 1422, 1425, + 1424, 1432, 3281, 1435, 1431, 1434, 1436, 1443, 1451, 1447, + 1452, 1449, 1454, 1460, 1457, 1459, 1465, 1467, 1469, 1471, + 1461, 1472, 1479, 1482, 1480, 1483, 1484, 1481, 3281, 1491, + + 1473, 1486, 1489, 1494, 1497, 1496, 1499, 1501, 1507, 1508, + 1512, 3281, 1515, 1509, 1516, 1520, 1521, 1525, 1522, 1529, + 1530, 1531, 1532, 1533, 1542, 1534, 1539, 1537, 3281, 1551, + 1553, 1549, 1560, 1547, 1557, 1558, 1562, 1559, 1563, 1570, + 1564, 1567, 1571, 1572, 1568, 1566, 1574, 1575, 1577, 1578, + 3281, 1595, 1593, 1596, 1597, 1598, 1606, 1581, 1603, 1605, + 1582, 1607, 1610, 1609, 1611, 1613, 1616, 1618, 1619, 1620, + 1621, 1622, 1623, 1632, 1638, 1626, 1640, 1630, 1642, 1629, + 1646, 1652, 1658, 1654, 3281, 1657, 1659, 1660, 1661, 1662, + 1668, 1670, 1664, 1666, 1671, 1672, 1673, 1687, 1674, 1676, + + 1680, 1684, 1678, 1686, 1689, 1700, 1705, 1690, 1696, 1702, + 1688, 1713, 1714, 1716, 1698, 1722, 3281, 1706, 3281, 1721, + 1727, 1730, 3281, 1732, 3281, 1734, 3281, 1735, 1736, 1729, + 1715, 1739, 1741, 1742, 1748, 1723, 1749, 1745, 1751, 1752, + 1754, 1755, 3281, 1758, 1761, 1763, 1759, 1765, 1766, 1768, + 1769, 1777, 1771, 1783, 1780, 1782, 1784, 3281, 1787, 1788, + 1792, 1793, 1796, 1800, 1798, 1797, 1805, 1801, 1809, 1820, + 1807, 1816, 1817, 1818, 1819, 1827, 1821, 1823, 1830, 1832, + 1825, 1831, 1833, 1835, 1840, 1841, 1847, 1839, 1845, 1846, + 3281, 1850, 1848, 1857, 1860, 1858, 3281, 3281, 1862, 1868, + + 1870, 1866, 1876, 1873, 1877, 1878, 1884, 1881, 1883, 1888, + 1890, 1891, 1893, 1894, 1895, 1897, 1898, 3281, 1904, 1902, + 1911, 1909, 1907, 1919, 1910, 1921, 3281, 3281, 1917, 1922, + 1926, 1928, 1934, 1932, 1931, 3281, 1933, 1936, 1937, 1938, + 1941, 1945, 1939, 1950, 1952, 1955, 1956, 1958, 1964, 1960, + 1957, 1961, 1974, 1963, 1977, 3281, 1971, 1979, 1981, 1982, + 1983, 1984, 1985, 1986, 1988, 1989, 1990, 1995, 2003, 1999, + 2002, 2004, 2007, 2008, 2012, 2014, 2015, 2022, 2023, 3281, + 2030, 2016, 2032, 2018, 2036, 2037, 2044, 2033, 2029, 2031, + 2040, 2041, 2042, 2054, 2048, 2056, 2050, 3281, 2058, 2062, + + 2064, 2059, 2068, 2065, 2071, 2069, 2073, 3281, 2075, 2081, + 2078, 2083, 2085, 2082, 2088, 2086, 2092, 2093, 2094, 2096, + 2099, 2101, 2103, 2105, 2110, 2111, 2118, 2116, 2122, 2128, + 2120, 2124, 2132, 2126, 2129, 2130, 2134, 2140, 2139, 2148, + 2137, 3281, 2141, 2151, 2153, 2154, 2157, 2158, 2147, 3281, + 2163, 3281, 2155, 3281, 2165, 2170, 2166, 2171, 2172, 2173, + 2174, 2176, 2184, 2175, 3281, 2182, 2185, 2187, 2188, 3281, + 2199, 3281, 3281, 2191, 2193, 2200, 2201, 2204, 2208, 2212, + 3281, 2202, 2213, 2214, 2216, 2217, 2219, 2220, 2222, 2221, + 2223, 2224, 3281, 2230, 2226, 2227, 2237, 3281, 2239, 2241, + + 2229, 2242, 2248, 2249, 2251, 2252, 2257, 2263, 2259, 3281, + 2260, 2261, 2266, 2262, 2269, 2271, 2267, 2268, 3281, 2282, + 2283, 3281, 2272, 2284, 2286, 3281, 2289, 3281, 2290, 2291, + 2292, 2294, 2298, 2299, 2301, 2293, 2305, 2310, 2307, 2314, + 2311, 3281, 3281, 2316, 2322, 2323, 2318, 2326, 2327, 2329, + 3281, 2331, 2333, 2334, 2335, 2336, 2337, 2342, 2340, 2343, + 2345, 2346, 2348, 2349, 3281, 2352, 2353, 2354, 2359, 2364, + 2361, 3281, 2363, 2366, 2365, 2367, 2370, 2371, 2385, 2387, + 2379, 2389, 2391, 2394, 2395, 2397, 2399, 2400, 3281, 2401, + 2403, 2404, 2408, 2409, 2405, 2410, 2415, 2418, 2420, 2422, + + 3281, 2425, 2412, 2428, 2426, 2430, 2432, 2433, 2434, 2436, + 2440, 2441, 2443, 2444, 2446, 2454, 2451, 3281, 2448, 3281, + 2450, 2461, 2469, 2465, 3281, 2463, 2467, 3281, 2470, 2471, + 2472, 2481, 3281, 2483, 2475, 2484, 2486, 2477, 2487, 2493, + 3281, 2495, 2497, 2502, 2503, 2490, 2504, 2505, 2496, 2507, + 2519, 2511, 3281, 2513, 2515, 2526, 2522, 2523, 3281, 3281, + 2527, 3281, 3281, 3281, 2532, 3281, 3281, 2533, 2535, 3281, + 2537, 3281, 2544, 2540, 2542, 2524, 2543, 3281, 2550, 3281, + 3281, 2547, 2551, 2545, 2555, 2557, 2562, 2564, 2554, 2565, + 2567, 2568, 2569, 2570, 2571, 2573, 2576, 2574, 2577, 2582, + + 2578, 2584, 2586, 2587, 2588, 2590, 2606, 3281, 3281, 2594, + 2599, 2596, 2601, 2603, 2607, 2608, 2609, 3281, 2616, 2611, + 2622, 2613, 2619, 2623, 2624, 2625, 2631, 2638, 2632, 2634, + 2635, 2636, 3281, 2639, 3281, 3281, 2642, 2643, 2648, 2645, + 2651, 3281, 2653, 2655, 2664, 2666, 3281, 3281, 3281, 2667, + 2657, 2661, 2671, 2672, 2659, 2673, 3281, 2675, 2680, 2682, + 2683, 2689, 2692, 2699, 2696, 2701, 3281, 3281, 2702, 2703, + 2704, 2684, 2705, 2706, 2708, 2710, 3281, 2712, 2711, 2717, + 2716, 2718, 2723, 2719, 3281, 2721, 2726, 2731, 2733, 2734, + 2736, 2737, 2739, 3281, 2738, 2741, 3281, 2749, 3281, 3281, + + 2742, 2751, 2753, 2758, 2760, 2764, 2755, 2761, 2770, 2767, + 2768, 3281, 3281, 2771, 3281, 3281, 2769, 2774, 2776, 2778, + 2781, 2783, 2780, 2784, 2785, 2788, 2790, 2793, 2796, 2797, + 2800, 2802, 2803, 2805, 3281, 2807, 3281, 2806, 2813, 2814, + 2819, 3281, 2820, 2815, 2808, 3281, 3281, 3281, 2816, 2830, + 2832, 3281, 2834, 2835, 2822, 2837, 2836, 2845, 2838, 2847, + 3281, 2848, 2849, 2851, 2852, 2853, 2855, 2857, 2858, 2860, + 2859, 2869, 2861, 2873, 2870, 3281, 2878, 2874, 2879, 2882, + 2884, 2886, 2887, 2888, 2890, 2891, 2892, 2893, 2900, 2896, + 2907, 3281, 2910, 2898, 3281, 2911, 2904, 2915, 2922, 2924, + + 2926, 2927, 3281, 2929, 3281, 2930, 3281, 2934, 2935, 3281, + 3281, 2936, 2938, 2941, 3281, 2942, 2939, 2943, 2945, 2948, + 2949, 3281, 3281, 2950, 2952, 2956, 2960, 3281, 3281, 3281, + 2966, 3281, 2967, 3281, 2973, 2953, 3281, 2958, 2975, 2961, + 2977, 2978, 2979, 2980, 2981, 2983, 3281, 3281, 2988, 3281, + 2990, 3281, 2991, 2993, 3281, 3001, 2995, 2997, 2998, 3004, + 3005, 3281, 3281, 3281, 3006, 3007, 3011, 3012, 3008, 3013, + 3014, 3015, 3019, 3016, 3023, 3032, 3020, 3033, 3040, 3036, + 3281, 3034, 3038, 3046, 3047, 3044, 3043, 3051, 3052, 3053, + 3054, 3055, 3059, 3060, 3064, 3061, 3062, 3071, 3074, 3077, + + 3073, 3086, 3084, 3087, 3085, 3088, 3095, 3089, 3093, 3281, + 3091, 3097, 3098, 3099, 3100, 3105, 3102, 3113, 3118, 3115, + 3122, 3109, 3124, 3125, 3126, 3111, 3131, 3127, 3281, 3134, + 3135, 3137, 3138, 3141, 3143, 3144, 3147, 3150, 3145, 3154, + 3159, 3160, 3281, 3161, 3281, 3281, 3164, 3151, 3155, 3172, + 3176, 3281, 3281, 3281, 3189, 3196, 3203, 3210, 3217, 94, + 3224, 3231, 3238, 3245, 3252, 3259, 3266, 3273 } ; -static yyconst flex_int16_t yy_def[1625] = +static yyconst flex_int16_t yy_def[1669] = { 0, - 1610, 1, 1611, 1611, 1612, 1612, 1613, 1613, 1614, 1614, - 1615, 1615, 1610, 1616, 1610, 1610, 1610, 1610, 1617, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1618, - 1610, 1610, 1610, 1618, 1619, 1610, 1610, 1610, 1619, 1620, - 1610, 1610, 1610, 1610, 1620, 1621, 1610, 1610, 1610, 1621, - 1622, 1610, 1623, 1610, 1622, 1622, 1616, 1616, 1610, 1624, - 1617, 1624, 1617, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1618, 1618, 1619, 1619, 1620, 1620, 1610, 1621, - 1621, 1622, 1622, 1623, 1623, 1622, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1622, 1616, 1616, 1616, 1616, 1616, 1616, - - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1622, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - - 1616, 1616, 1616, 1616, 1616, 1610, 1616, 1616, 1616, 1616, - 1616, 1610, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1622, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1622, 1616, 1616, 1616, 1616, 1610, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1610, 1616, 1610, 1610, 1616, - 1610, 1610, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1610, - 1616, 1616, 1616, 1616, 1616, 1610, 1616, 1616, 1616, 1616, - - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1622, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1610, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1610, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1610, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1610, 1622, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1610, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1610, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - - 1616, 1616, 1610, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1610, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1610, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1610, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - - 1616, 1616, 1616, 1616, 1610, 1616, 1610, 1616, 1616, 1616, - 1610, 1616, 1610, 1616, 1610, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1610, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1610, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1610, 1616, 1616, 1616, - 1616, 1610, 1610, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - - 1616, 1616, 1610, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1610, 1610, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1610, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1610, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1610, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1610, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1610, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1610, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1610, 1610, 1616, 1610, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1610, 1616, 1616, 1616, 1616, - 1610, 1616, 1610, 1610, 1616, 1616, 1616, 1616, 1616, 1616, - 1610, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1610, 1616, 1616, 1616, 1616, 1610, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1610, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1610, 1616, 1616, 1610, - - 1616, 1616, 1616, 1610, 1616, 1610, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1610, - 1610, 1616, 1616, 1616, 1616, 1616, 1616, 1610, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1610, 1616, 1616, 1616, 1616, 1616, 1610, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1610, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1610, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1610, 1616, 1610, 1616, 1616, 1616, 1616, 1610, 1616, - - 1616, 1610, 1616, 1616, 1616, 1610, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1610, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1610, 1616, 1616, 1616, 1616, 1610, - 1610, 1616, 1610, 1610, 1616, 1610, 1610, 1616, 1616, 1610, - 1616, 1610, 1616, 1616, 1616, 1616, 1616, 1610, 1616, 1610, - 1610, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1610, 1610, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1610, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1610, - - 1616, 1610, 1610, 1616, 1616, 1616, 1616, 1616, 1610, 1616, - 1616, 1616, 1616, 1610, 1610, 1610, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1610, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1610, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1610, 1616, 1616, 1616, 1616, 1616, 1616, 1610, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1610, 1616, 1616, - 1610, 1616, 1610, 1610, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1610, 1610, 1616, 1610, 1610, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1610, 1616, 1610, 1616, - - 1616, 1616, 1616, 1610, 1616, 1616, 1616, 1610, 1610, 1610, - 1616, 1616, 1616, 1610, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1610, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1610, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1610, 1616, 1616, 1610, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1610, 1616, 1610, 1616, 1616, 1610, 1610, 1616, - 1616, 1616, 1610, 1616, 1616, 1616, 1616, 1616, 1616, 1610, - 1610, 1616, 1616, 1616, 1610, 1610, 1610, 1616, 1610, 1616, - 1610, 1616, 1616, 1610, 1616, 1616, 1616, 1616, 1616, 1616, - - 1616, 1616, 1616, 1610, 1616, 1610, 1616, 1610, 1616, 1616, - 1610, 1616, 1616, 1616, 1616, 1616, 1616, 1610, 1610, 1610, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1610, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1610, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1610, 1616, 1616, 1616, 1616, 1616, - 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1616, 1610, 1616, - - 1610, 1610, 1616, 1616, 1616, 1616, 1616, 1610, 1610, 0, - 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, - 1610, 1610, 1610, 1610 + 1654, 1, 1655, 1655, 1656, 1656, 1657, 1657, 1658, 1658, + 1659, 1659, 1654, 1660, 1654, 1654, 1654, 1654, 1661, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1662, + 1654, 1654, 1654, 1662, 1663, 1654, 1654, 1654, 1663, 1664, + 1654, 1654, 1654, 1654, 1664, 1665, 1654, 1654, 1654, 1665, + 1666, 1654, 1667, 1654, 1666, 1666, 1660, 1660, 1654, 1668, + 1661, 1668, 1661, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1662, 1662, 1663, 1663, 1664, 1664, 1654, 1665, + 1665, 1666, 1666, 1667, 1667, 1666, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1666, 1660, 1660, 1660, 1660, 1660, 1660, + + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1666, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1654, 1660, 1660, + 1660, 1660, 1660, 1654, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1666, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1666, 1660, 1660, 1660, 1660, 1654, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1654, 1660, + 1654, 1654, 1660, 1654, 1654, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1654, 1660, 1660, 1660, 1660, 1660, 1654, + + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1666, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1654, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1654, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1654, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1654, + 1666, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1654, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1654, 1660, + + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1654, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1654, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1654, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1654, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1654, 1660, 1654, 1660, + 1660, 1660, 1654, 1660, 1654, 1660, 1654, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1654, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1654, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1654, 1660, 1660, 1660, 1660, 1660, 1654, 1654, 1660, 1660, + + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1654, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1654, 1654, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1654, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1654, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1654, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1654, 1660, 1660, + + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1654, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1654, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1654, + 1660, 1654, 1660, 1654, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1654, 1660, 1660, 1660, 1660, 1654, + 1660, 1654, 1654, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1654, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1654, 1660, 1660, 1660, 1660, 1654, 1660, 1660, + + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1654, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1654, 1660, + 1660, 1654, 1660, 1660, 1660, 1654, 1660, 1654, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1654, 1654, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1654, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1654, 1660, 1660, 1660, 1660, 1660, + 1660, 1654, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1654, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + + 1654, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1654, 1660, 1654, + 1660, 1660, 1660, 1660, 1654, 1660, 1660, 1654, 1660, 1660, + 1660, 1660, 1654, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1654, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1654, 1660, 1660, 1660, 1660, 1660, 1654, 1654, + 1660, 1654, 1654, 1654, 1660, 1654, 1654, 1660, 1660, 1654, + 1660, 1654, 1660, 1660, 1660, 1660, 1660, 1654, 1660, 1654, + 1654, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1654, 1654, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1654, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1654, 1660, 1654, 1654, 1660, 1660, 1660, 1660, + 1660, 1654, 1660, 1660, 1660, 1660, 1654, 1654, 1654, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1654, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1654, 1654, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1654, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1654, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1654, 1660, 1660, 1654, 1660, 1654, 1654, + + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1654, 1654, 1660, 1654, 1654, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1654, 1660, 1654, 1660, 1660, 1660, + 1660, 1654, 1660, 1660, 1660, 1654, 1654, 1654, 1660, 1660, + 1660, 1654, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1654, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1654, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1654, 1660, 1660, 1654, 1660, 1660, 1660, 1660, 1660, + + 1660, 1660, 1654, 1660, 1654, 1660, 1654, 1660, 1660, 1654, + 1654, 1660, 1660, 1660, 1654, 1660, 1660, 1660, 1660, 1660, + 1660, 1654, 1654, 1660, 1660, 1660, 1660, 1654, 1654, 1654, + 1660, 1654, 1660, 1654, 1660, 1660, 1654, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1654, 1654, 1660, 1654, + 1660, 1654, 1660, 1660, 1654, 1660, 1660, 1660, 1660, 1660, + 1660, 1654, 1654, 1654, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1654, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1654, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1654, 1660, + 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, 1660, + 1660, 1660, 1654, 1660, 1654, 1654, 1660, 1660, 1660, 1660, + 1660, 1654, 1654, 0, 1654, 1654, 1654, 1654, 1654, 1654, + 1654, 1654, 1654, 1654, 1654, 1654, 1654, 1654 } ; -static yyconst flex_int16_t yy_nxt[3241] = +static yyconst flex_int16_t yy_nxt[3321] = { 0, 14, 15, 16, 17, 18, 19, 18, 14, 14, 14, 14, 18, 20, 21, 14, 22, 23, 24, 25, 14, @@ -983,7 +996,7 @@ static yyconst flex_int16_t yy_nxt[3241] = 107, 103, 110, 68, 113, 111, 139, 108, 114, 115, 68, 121, 169, 118, 68, 122, 127, 119, 127, 127, - 239, 127, 120, 72, 68, 72, 72, 132, 72, 132, + 240, 127, 120, 72, 68, 72, 72, 132, 72, 132, 132, 68, 132, 67, 135, 67, 67, 68, 67, 68, 68, 68, 141, 67, 72, 142, 72, 72, 147, 72, 68, 143, 68, 68, 72, 73, 68, 68, 144, 145, @@ -999,327 +1012,336 @@ static yyconst flex_int16_t yy_nxt[3241] = 192, 125, 130, 130, 190, 198, 68, 127, 193, 127, 127, 132, 127, 132, 132, 72, 132, 72, 72, 133, 72, 197, 194, 68, 196, 68, 135, 68, 195, 68, - 201, 68, 68, 204, 131, 68, 202, 203, 68, 205, + 201, 68, 68, 204, 68, 68, 202, 203, 68, 205, 199, 68, 68, 212, 68, 68, 68, 200, 68, 214, - 213, 217, 216, 68, 68, 68, 68, 224, 68, 218, - 68, 206, 68, 223, 227, 68, 207, 220, 219, 68, - 221, 208, 222, 68, 228, 68, 209, 68, 68, 229, - - 225, 226, 210, 211, 68, 243, 231, 232, 234, 68, - 230, 68, 236, 68, 235, 237, 233, 68, 68, 68, - 68, 68, 68, 68, 68, 238, 68, 241, 68, 246, - 68, 250, 68, 68, 68, 252, 68, 254, 244, 240, - 68, 253, 68, 68, 242, 68, 245, 261, 248, 249, - 247, 258, 251, 68, 257, 256, 68, 260, 133, 255, - 68, 68, 68, 266, 262, 68, 68, 68, 68, 265, - 263, 267, 68, 68, 259, 68, 68, 273, 272, 264, - 268, 270, 68, 274, 68, 269, 68, 68, 275, 68, - 276, 68, 68, 68, 279, 282, 280, 271, 277, 278, - - 68, 68, 281, 68, 284, 68, 286, 68, 68, 68, - 285, 68, 68, 68, 291, 283, 68, 68, 295, 68, - 68, 293, 68, 68, 68, 290, 288, 287, 68, 68, - 289, 294, 296, 68, 292, 68, 297, 298, 306, 68, - 299, 302, 300, 301, 68, 68, 68, 303, 304, 305, - 68, 312, 68, 307, 68, 68, 314, 68, 68, 68, - 310, 308, 313, 311, 309, 317, 68, 68, 68, 68, - 68, 68, 319, 315, 320, 68, 321, 68, 327, 316, - 68, 68, 322, 68, 326, 323, 318, 68, 324, 68, - 325, 68, 328, 68, 333, 68, 331, 68, 329, 133, - - 330, 68, 68, 332, 344, 68, 380, 335, 68, 68, - 334, 68, 343, 336, 337, 68, 347, 349, 68, 68, - 345, 346, 68, 338, 68, 339, 340, 341, 348, 350, - 342, 353, 356, 351, 352, 354, 68, 68, 355, 68, - 68, 359, 360, 68, 68, 358, 68, 68, 68, 68, - 68, 365, 68, 68, 366, 357, 68, 68, 68, 386, - 68, 364, 362, 361, 363, 369, 371, 368, 68, 374, - 367, 68, 375, 372, 68, 373, 68, 68, 370, 68, - 376, 378, 68, 377, 381, 68, 68, 68, 68, 68, - 68, 68, 385, 129, 68, 379, 383, 68, 384, 387, - - 68, 389, 68, 390, 68, 392, 382, 68, 68, 68, - 68, 388, 391, 393, 68, 68, 400, 68, 68, 394, - 398, 68, 399, 68, 397, 68, 68, 68, 395, 396, - 404, 68, 68, 68, 68, 401, 406, 68, 416, 68, - 402, 407, 405, 68, 403, 68, 420, 68, 417, 415, - 418, 408, 68, 409, 414, 68, 422, 419, 410, 68, - 411, 68, 68, 68, 421, 68, 68, 68, 412, 68, - 68, 428, 68, 424, 423, 133, 429, 425, 413, 432, - 68, 433, 426, 68, 434, 431, 68, 427, 430, 437, - 68, 68, 68, 68, 68, 68, 436, 440, 435, 68, - - 68, 445, 442, 438, 439, 446, 68, 450, 441, 448, - 68, 443, 68, 447, 444, 449, 68, 451, 68, 452, - 68, 453, 68, 68, 68, 68, 68, 457, 455, 456, - 68, 68, 68, 461, 458, 68, 459, 68, 68, 68, - 462, 128, 68, 454, 460, 464, 68, 467, 469, 68, - 465, 463, 68, 68, 466, 471, 68, 468, 470, 68, - 472, 68, 68, 68, 474, 68, 479, 476, 68, 68, - 68, 68, 126, 481, 475, 480, 68, 478, 68, 477, - 68, 482, 473, 68, 68, 483, 484, 487, 486, 68, - 485, 489, 491, 68, 488, 490, 68, 68, 68, 495, - - 68, 493, 492, 496, 68, 68, 68, 497, 499, 68, - 68, 68, 68, 68, 68, 68, 494, 504, 68, 498, - 68, 68, 68, 505, 509, 68, 511, 500, 501, 68, - 502, 68, 503, 506, 510, 512, 508, 68, 507, 68, - 68, 516, 68, 68, 514, 68, 68, 68, 513, 68, - 518, 519, 520, 68, 521, 517, 515, 68, 522, 68, - 68, 133, 68, 68, 524, 523, 527, 68, 525, 68, - 528, 68, 531, 529, 526, 68, 533, 68, 68, 534, - 532, 68, 68, 530, 68, 68, 68, 68, 539, 535, - 537, 68, 68, 68, 68, 547, 536, 548, 549, 124, - - 545, 550, 538, 68, 546, 68, 68, 68, 540, 68, - 557, 558, 541, 68, 562, 542, 560, 559, 561, 68, - 68, 563, 543, 68, 564, 544, 68, 68, 551, 68, - 552, 68, 565, 553, 68, 68, 68, 566, 554, 68, - 567, 68, 571, 568, 555, 556, 569, 570, 68, 68, - 576, 572, 575, 68, 574, 68, 577, 68, 68, 68, - 578, 68, 68, 68, 579, 68, 573, 580, 68, 68, - 581, 587, 584, 583, 585, 68, 68, 586, 68, 68, - 68, 582, 589, 68, 591, 68, 68, 68, 68, 68, - 593, 68, 68, 596, 590, 68, 68, 588, 594, 68, - - 592, 68, 68, 602, 68, 601, 68, 595, 68, 68, - 597, 598, 68, 605, 599, 600, 68, 610, 607, 603, - 606, 604, 68, 609, 611, 68, 68, 68, 608, 68, - 612, 68, 615, 68, 617, 613, 618, 614, 68, 68, - 68, 68, 619, 622, 68, 624, 68, 68, 68, 625, - 133, 68, 616, 621, 68, 627, 620, 68, 68, 68, - 68, 68, 626, 636, 623, 68, 68, 68, 133, 68, - 68, 640, 630, 628, 68, 637, 68, 68, 643, 659, - 639, 629, 68, 641, 642, 631, 638, 632, 68, 68, - 68, 633, 644, 634, 645, 647, 68, 649, 635, 68, - - 68, 652, 648, 68, 646, 651, 68, 68, 653, 68, - 650, 68, 660, 654, 657, 655, 68, 658, 68, 664, - 68, 661, 665, 68, 68, 667, 662, 68, 666, 68, - 68, 68, 668, 656, 663, 68, 68, 68, 68, 68, - 671, 68, 68, 68, 670, 68, 669, 673, 675, 68, - 678, 68, 679, 68, 68, 672, 674, 68, 683, 676, - 682, 680, 68, 677, 68, 68, 716, 684, 68, 681, - 685, 68, 687, 686, 688, 68, 689, 68, 693, 68, - 690, 68, 68, 691, 68, 68, 692, 696, 695, 68, - 68, 68, 694, 68, 698, 699, 68, 700, 703, 68, - - 68, 68, 702, 68, 68, 68, 697, 706, 68, 701, - 704, 68, 68, 68, 68, 68, 711, 68, 705, 712, - 714, 707, 717, 68, 708, 710, 715, 68, 709, 718, - 68, 713, 720, 68, 68, 68, 68, 724, 719, 68, - 68, 68, 68, 68, 68, 68, 68, 723, 732, 721, - 722, 726, 131, 68, 68, 68, 733, 725, 729, 727, - 730, 68, 68, 68, 68, 68, 731, 734, 728, 735, - 737, 68, 68, 740, 741, 68, 68, 736, 68, 738, - 68, 745, 743, 68, 68, 68, 739, 68, 746, 68, - 742, 747, 749, 68, 68, 68, 68, 68, 744, 748, - - 752, 754, 751, 68, 68, 750, 68, 756, 68, 68, - 68, 68, 68, 753, 68, 757, 755, 68, 68, 758, - 764, 760, 68, 759, 68, 68, 68, 68, 848, 761, - 763, 767, 762, 765, 768, 770, 68, 766, 769, 68, - 68, 772, 773, 68, 68, 771, 774, 68, 68, 778, - 68, 779, 68, 777, 68, 68, 776, 780, 68, 775, - 781, 68, 782, 68, 68, 787, 786, 783, 784, 788, - 68, 68, 68, 68, 68, 68, 789, 790, 785, 796, - 68, 791, 68, 68, 68, 68, 792, 68, 793, 68, - 794, 795, 68, 68, 68, 801, 68, 797, 799, 805, - - 68, 798, 68, 802, 800, 807, 68, 809, 68, 806, - 803, 808, 811, 68, 68, 810, 813, 68, 804, 68, - 815, 68, 68, 68, 814, 68, 68, 68, 817, 68, - 820, 819, 68, 818, 812, 824, 68, 68, 822, 825, - 68, 68, 816, 823, 68, 821, 826, 827, 68, 68, - 831, 68, 68, 68, 68, 68, 829, 828, 834, 68, - 830, 68, 68, 68, 68, 838, 68, 68, 68, 68, - 68, 68, 851, 832, 68, 835, 843, 833, 844, 836, - 837, 839, 845, 68, 68, 841, 68, 68, 849, 853, - 850, 840, 842, 846, 847, 68, 68, 68, 852, 68, - - 68, 68, 68, 68, 856, 68, 859, 68, 68, 68, - 68, 68, 68, 68, 129, 855, 865, 862, 857, 866, - 68, 858, 854, 860, 863, 867, 861, 864, 868, 68, - 68, 68, 68, 876, 872, 68, 874, 871, 869, 875, - 873, 68, 877, 68, 68, 870, 68, 68, 68, 881, - 68, 878, 68, 882, 68, 883, 68, 68, 68, 68, - 68, 879, 68, 889, 68, 888, 68, 880, 68, 884, - 68, 885, 68, 68, 68, 897, 68, 887, 894, 890, - 899, 891, 68, 886, 68, 892, 68, 898, 896, 893, - 68, 903, 68, 895, 68, 901, 68, 900, 68, 904, - - 905, 907, 68, 68, 68, 910, 68, 911, 909, 68, - 902, 68, 68, 912, 68, 913, 68, 68, 906, 68, - 908, 68, 914, 68, 915, 68, 916, 921, 68, 68, - 919, 918, 68, 923, 917, 68, 920, 68, 68, 68, - 68, 68, 926, 68, 931, 928, 68, 68, 925, 922, - 68, 68, 68, 68, 935, 68, 68, 924, 927, 929, - 930, 936, 68, 937, 940, 68, 68, 932, 68, 939, - 933, 934, 941, 938, 68, 943, 68, 68, 68, 68, - 949, 946, 68, 68, 947, 68, 942, 950, 944, 68, - 68, 68, 951, 954, 952, 955, 948, 945, 68, 956, - - 68, 68, 68, 68, 953, 68, 68, 68, 960, 957, - 68, 962, 963, 68, 68, 964, 68, 68, 68, 68, - 958, 971, 959, 68, 68, 965, 68, 68, 961, 68, - 969, 968, 970, 68, 966, 68, 967, 68, 977, 68, - 976, 978, 981, 972, 973, 980, 68, 68, 974, 68, - 975, 68, 982, 979, 68, 68, 989, 68, 990, 68, - 984, 68, 68, 68, 986, 983, 988, 987, 68, 68, - 68, 68, 68, 985, 68, 997, 992, 68, 68, 1000, - 68, 68, 991, 994, 68, 1002, 993, 1003, 998, 996, - 68, 68, 68, 68, 1005, 995, 999, 1004, 1001, 68, - - 68, 1009, 68, 1007, 1006, 1010, 68, 1013, 68, 1008, - 68, 68, 68, 68, 68, 68, 68, 1017, 68, 1018, - 1011, 1019, 68, 1020, 1014, 1012, 68, 68, 1015, 1024, - 68, 1016, 68, 68, 1028, 1021, 68, 1023, 1022, 68, - 68, 1025, 128, 68, 1026, 1027, 1032, 68, 1029, 1031, - 1033, 68, 68, 1035, 68, 68, 68, 68, 1030, 68, - 68, 1034, 1037, 68, 68, 1038, 68, 1036, 1041, 68, - 68, 1039, 68, 1042, 1047, 1040, 1046, 68, 68, 68, - 68, 1045, 1051, 68, 1053, 68, 1043, 1054, 68, 1044, - 1055, 1048, 1050, 68, 1057, 68, 1056, 1049, 68, 68, - - 1052, 1059, 68, 1061, 68, 1063, 68, 68, 1062, 68, - 68, 68, 68, 68, 68, 68, 1058, 1071, 68, 68, - 1073, 1064, 68, 1065, 68, 68, 68, 1060, 1066, 1076, - 68, 1067, 1068, 1069, 1075, 1070, 68, 68, 68, 1077, - 68, 1072, 1074, 1078, 68, 1081, 1079, 68, 68, 1085, - 68, 68, 1080, 1084, 68, 68, 1083, 1087, 1088, 68, - 1089, 68, 68, 68, 1082, 68, 68, 1086, 68, 68, - 1090, 1091, 1095, 1097, 68, 1100, 68, 1092, 1096, 1098, - 1094, 68, 1093, 68, 1099, 1103, 1104, 68, 1105, 1107, - 68, 68, 1106, 68, 68, 68, 1110, 1102, 68, 68, - - 68, 68, 1101, 1112, 68, 68, 1115, 1109, 1117, 68, - 1111, 68, 1113, 68, 126, 68, 68, 1108, 1119, 1120, - 68, 1118, 1121, 68, 68, 1125, 68, 1116, 68, 1114, - 1122, 1123, 68, 68, 1124, 1128, 68, 68, 68, 68, - 68, 68, 68, 1133, 1126, 1127, 68, 1130, 1135, 68, - 1137, 1134, 68, 68, 68, 68, 1136, 1129, 1131, 68, - 1132, 1138, 1144, 1140, 1141, 1142, 68, 68, 68, 68, - 1139, 68, 1146, 1143, 68, 68, 1147, 1148, 68, 68, - 1151, 68, 1149, 68, 68, 1145, 68, 68, 68, 1150, - 68, 68, 68, 1159, 68, 1153, 1154, 1158, 1155, 68, - - 68, 68, 1152, 1156, 1163, 68, 1165, 1157, 1160, 1161, - 68, 68, 1166, 1162, 68, 68, 1167, 68, 1171, 1168, - 1164, 68, 68, 68, 68, 124, 1172, 1175, 68, 68, - 68, 68, 68, 1169, 1179, 68, 68, 1170, 1174, 68, - 1177, 1173, 1176, 1178, 1180, 68, 68, 68, 1181, 1185, - 1182, 1183, 1184, 68, 68, 68, 68, 68, 68, 1187, - 1188, 1192, 68, 1186, 1194, 68, 1191, 1193, 68, 68, - 1189, 68, 68, 1196, 1199, 68, 1197, 68, 68, 68, - 68, 1190, 1202, 68, 68, 1198, 1200, 1195, 1201, 68, - 1203, 68, 1204, 1206, 68, 68, 1205, 68, 68, 68, - - 1208, 68, 1207, 1211, 68, 1214, 68, 1209, 68, 68, - 68, 68, 1210, 1215, 68, 1217, 1216, 1212, 68, 68, - 1213, 68, 68, 1225, 68, 68, 68, 68, 1226, 68, - 1219, 68, 1229, 1223, 1218, 1230, 68, 1221, 1220, 1227, - 1222, 68, 1228, 1224, 1231, 68, 1233, 68, 1234, 68, - 68, 1236, 68, 68, 1232, 1237, 68, 68, 1240, 68, - 68, 68, 1239, 1242, 68, 68, 68, 1244, 68, 1235, - 68, 1245, 1238, 68, 1246, 1248, 68, 1250, 68, 1241, - 1251, 68, 68, 1243, 68, 1249, 68, 1253, 68, 1247, - 68, 1254, 68, 68, 68, 1256, 1252, 68, 68, 1260, - - 1255, 68, 68, 1265, 68, 1258, 68, 1262, 68, 1257, - 1263, 68, 1261, 68, 1259, 1266, 1264, 68, 1269, 68, - 1270, 1273, 1268, 68, 1267, 68, 68, 1271, 1276, 68, - 68, 1274, 1275, 1277, 68, 1272, 1278, 68, 68, 1280, - 68, 68, 68, 68, 1279, 68, 1284, 1285, 68, 1287, - 68, 1286, 1281, 68, 68, 1282, 68, 68, 1288, 68, - 1290, 1283, 1293, 68, 1292, 68, 68, 68, 68, 1294, - 68, 1296, 1298, 68, 1289, 68, 1297, 1291, 68, 1299, - 1301, 1295, 1300, 68, 1302, 68, 1303, 68, 1304, 68, - 1305, 68, 68, 68, 68, 1307, 68, 1308, 1309, 68, - - 68, 1310, 1311, 68, 68, 1314, 68, 1306, 1312, 1313, - 1315, 68, 1316, 68, 68, 1317, 68, 68, 68, 68, - 68, 1324, 68, 68, 1322, 68, 68, 68, 1319, 1320, - 1321, 68, 1318, 68, 1325, 68, 68, 68, 1334, 68, - 1323, 1332, 1333, 68, 1328, 68, 1327, 1330, 68, 1326, - 68, 1329, 68, 1331, 68, 68, 68, 1342, 1343, 68, - 68, 1339, 68, 1335, 68, 68, 1336, 1337, 68, 1344, - 1338, 68, 68, 1341, 1340, 1350, 68, 1345, 68, 68, - 1351, 68, 1346, 1347, 1348, 68, 68, 1352, 68, 68, - 1349, 1357, 1358, 68, 68, 1361, 68, 1353, 68, 1354, - - 1355, 1363, 68, 68, 1359, 68, 1356, 1364, 68, 68, - 1365, 68, 1360, 68, 1367, 68, 1368, 1362, 1369, 68, - 68, 68, 68, 68, 68, 1366, 1373, 1371, 1374, 1376, - 68, 1377, 68, 1378, 1370, 1372, 1388, 1375, 68, 1379, - 68, 1380, 68, 1381, 68, 68, 68, 68, 68, 1382, - 68, 1383, 68, 1384, 68, 68, 68, 1387, 1386, 1389, - 68, 68, 1385, 68, 1391, 68, 1393, 1390, 68, 1392, - 1394, 1397, 68, 68, 68, 1399, 68, 68, 68, 68, - 68, 1400, 1395, 1398, 1401, 68, 1402, 1396, 1404, 68, - 68, 1403, 68, 1406, 68, 1408, 68, 1407, 1409, 68, - - 1405, 1410, 68, 1411, 1412, 1413, 1414, 68, 68, 68, - 68, 68, 1415, 68, 68, 1416, 68, 68, 1419, 1423, - 68, 68, 68, 68, 68, 1427, 68, 68, 1417, 1418, - 1425, 1422, 1420, 68, 68, 68, 1421, 68, 1429, 68, - 1424, 68, 1426, 1428, 1432, 68, 1430, 68, 1434, 1436, - 68, 1437, 68, 68, 68, 68, 1443, 1433, 68, 1435, - 1431, 1438, 1439, 1442, 1441, 68, 1440, 68, 68, 68, - 68, 68, 1445, 1450, 1444, 1448, 68, 1446, 68, 1452, - 68, 68, 68, 1455, 68, 68, 68, 68, 68, 1449, - 1447, 68, 68, 68, 1456, 68, 1459, 1454, 1460, 68, - - 1453, 1451, 1457, 1463, 68, 68, 1458, 1465, 68, 68, - 1464, 1462, 1468, 68, 68, 1466, 68, 1461, 1469, 68, - 68, 1467, 1473, 68, 68, 68, 1471, 68, 1470, 1475, - 1477, 68, 1474, 1480, 68, 68, 1472, 1481, 68, 68, - 68, 1485, 68, 1610, 1476, 1484, 1482, 68, 1478, 1479, - 1486, 68, 1488, 1483, 1487, 68, 1489, 68, 68, 1490, - 1491, 68, 68, 68, 1494, 68, 68, 1492, 68, 68, - 68, 1493, 68, 1495, 1496, 68, 68, 68, 1504, 68, - 68, 1502, 68, 68, 1497, 68, 1498, 1505, 68, 1499, - 1500, 1506, 68, 68, 68, 1509, 1501, 1503, 1507, 1508, - - 68, 1511, 68, 1510, 68, 68, 68, 1512, 1513, 1514, - 1515, 1516, 1518, 68, 1519, 68, 1520, 68, 68, 68, - 1522, 1523, 68, 1517, 68, 68, 68, 68, 68, 1525, - 1521, 68, 1529, 68, 68, 68, 1524, 68, 68, 68, - 68, 1537, 68, 68, 1526, 1527, 68, 1528, 1531, 68, - 68, 1539, 1533, 1530, 1534, 1536, 1532, 68, 1538, 68, - 68, 1543, 1540, 68, 1535, 1542, 68, 1547, 1541, 1548, - 68, 68, 68, 1544, 68, 1545, 68, 68, 68, 68, - 68, 68, 1546, 68, 1549, 68, 68, 1560, 1557, 1550, - 1551, 1552, 1554, 1555, 68, 68, 68, 1553, 68, 1558, - - 1556, 1561, 68, 1562, 1559, 1565, 68, 1563, 1566, 68, - 68, 68, 68, 1567, 1564, 1570, 68, 68, 68, 68, - 68, 68, 1573, 68, 68, 1577, 68, 68, 68, 68, - 1568, 1569, 1572, 1578, 1582, 1571, 1576, 1574, 68, 1580, - 1575, 68, 1581, 68, 68, 1579, 1584, 1585, 68, 68, - 1583, 1586, 68, 68, 68, 68, 1587, 68, 1589, 68, - 1591, 68, 1592, 68, 1595, 68, 68, 68, 1598, 1599, - 68, 1588, 68, 1590, 1601, 68, 1602, 68, 1593, 1600, - 1594, 68, 68, 68, 1596, 1597, 68, 1604, 1603, 68, - 1610, 1605, 1608, 68, 1609, 68, 1610, 1610, 1610, 1610, - - 1610, 1610, 1606, 1610, 1610, 1610, 1610, 1610, 1607, 40, + 213, 217, 216, 68, 68, 68, 243, 68, 68, 218, + 131, 206, 68, 223, 68, 224, 207, 220, 219, 68, + 221, 208, 222, 228, 68, 229, 209, 226, 68, 68, + + 68, 230, 210, 211, 225, 227, 68, 68, 232, 233, + 235, 68, 231, 68, 237, 68, 236, 238, 68, 68, + 234, 68, 68, 68, 68, 239, 68, 241, 68, 242, + 68, 68, 244, 247, 251, 68, 245, 68, 68, 68, + 253, 68, 255, 259, 254, 68, 246, 68, 68, 249, + 250, 261, 248, 68, 258, 252, 68, 257, 133, 68, + 256, 262, 68, 68, 267, 68, 260, 68, 68, 68, + 264, 266, 268, 68, 68, 129, 68, 68, 263, 273, + 269, 265, 271, 68, 274, 275, 270, 68, 68, 68, + 276, 68, 277, 278, 68, 68, 68, 280, 272, 281, + + 68, 283, 279, 68, 285, 282, 68, 68, 287, 68, + 68, 68, 286, 68, 284, 68, 292, 68, 68, 68, + 68, 68, 297, 294, 68, 68, 68, 68, 289, 288, + 291, 68, 290, 296, 298, 302, 293, 299, 295, 68, + 68, 300, 68, 68, 301, 306, 307, 308, 68, 68, + 68, 68, 309, 304, 305, 68, 314, 68, 68, 68, + 303, 68, 316, 315, 313, 68, 312, 310, 319, 68, + 311, 68, 68, 68, 317, 322, 321, 323, 68, 68, + 68, 68, 329, 68, 68, 318, 68, 68, 330, 320, + 328, 68, 324, 68, 68, 325, 68, 335, 326, 68, + + 327, 331, 133, 68, 332, 348, 333, 334, 346, 68, + 337, 68, 68, 336, 68, 351, 338, 339, 345, 68, + 68, 68, 355, 347, 349, 68, 340, 68, 341, 342, + 343, 350, 352, 344, 353, 68, 354, 358, 68, 68, + 361, 362, 68, 68, 360, 356, 68, 68, 68, 357, + 68, 68, 367, 68, 359, 68, 68, 368, 68, 68, + 68, 380, 128, 364, 365, 366, 363, 371, 373, 68, + 370, 68, 68, 369, 374, 375, 68, 377, 68, 68, + 372, 376, 68, 379, 378, 68, 68, 383, 384, 382, + 381, 68, 68, 68, 68, 68, 68, 389, 68, 388, + + 68, 386, 68, 390, 68, 392, 387, 68, 393, 68, + 68, 68, 385, 68, 68, 68, 68, 395, 68, 396, + 68, 391, 68, 394, 402, 397, 68, 401, 403, 400, + 68, 398, 68, 68, 399, 68, 404, 409, 68, 407, + 68, 68, 405, 68, 68, 408, 419, 406, 68, 68, + 68, 68, 421, 68, 410, 68, 420, 418, 126, 411, + 68, 412, 68, 417, 422, 426, 413, 427, 414, 68, + 423, 68, 425, 424, 68, 430, 415, 428, 68, 68, + 68, 68, 436, 431, 68, 133, 416, 68, 432, 435, + 68, 68, 68, 429, 440, 68, 434, 439, 68, 433, + + 68, 68, 437, 443, 441, 68, 68, 448, 445, 438, + 442, 449, 68, 68, 451, 68, 444, 452, 68, 453, + 454, 68, 446, 68, 68, 447, 450, 455, 68, 456, + 68, 68, 68, 460, 68, 458, 459, 68, 68, 68, + 68, 461, 464, 68, 457, 462, 68, 68, 465, 467, + 68, 463, 68, 466, 468, 68, 470, 473, 469, 68, + 68, 68, 68, 476, 475, 68, 472, 68, 471, 474, + 478, 68, 68, 483, 68, 68, 480, 68, 68, 68, + 479, 484, 485, 68, 68, 477, 482, 486, 481, 68, + 68, 68, 68, 491, 487, 488, 490, 489, 68, 68, + + 493, 495, 68, 492, 494, 68, 68, 68, 496, 499, + 497, 502, 68, 500, 68, 68, 68, 501, 503, 68, + 504, 68, 68, 68, 68, 498, 508, 68, 68, 513, + 505, 68, 509, 68, 68, 68, 515, 516, 506, 510, + 68, 507, 68, 512, 68, 511, 68, 518, 68, 68, + 514, 517, 68, 68, 68, 68, 522, 523, 525, 526, + 519, 521, 524, 68, 68, 520, 68, 133, 68, 68, + 68, 528, 531, 532, 68, 529, 68, 535, 533, 537, + 68, 527, 530, 68, 68, 536, 68, 538, 68, 68, + 534, 68, 539, 68, 68, 68, 541, 68, 68, 68, + + 554, 552, 553, 68, 540, 68, 551, 542, 549, 68, + 68, 550, 68, 565, 562, 571, 543, 68, 566, 68, + 68, 563, 544, 68, 564, 68, 545, 567, 124, 546, + 68, 568, 68, 570, 68, 68, 547, 68, 569, 548, + 68, 555, 556, 68, 557, 574, 68, 558, 572, 68, + 68, 577, 559, 573, 575, 579, 68, 576, 560, 561, + 68, 578, 581, 68, 582, 580, 583, 68, 68, 68, + 584, 68, 68, 68, 585, 68, 68, 68, 591, 68, + 586, 587, 589, 590, 68, 593, 68, 68, 68, 68, + 68, 595, 592, 588, 68, 597, 68, 68, 68, 599, + + 68, 596, 68, 602, 598, 594, 68, 600, 68, 68, + 68, 68, 68, 68, 608, 68, 68, 607, 601, 68, + 611, 603, 604, 68, 68, 68, 68, 724, 605, 606, + 609, 613, 610, 612, 616, 614, 617, 615, 68, 68, + 68, 68, 618, 619, 68, 68, 623, 622, 624, 621, + 68, 68, 620, 68, 625, 628, 68, 68, 630, 68, + 68, 631, 133, 68, 68, 68, 68, 68, 626, 627, + 633, 68, 68, 68, 68, 643, 68, 629, 632, 647, + 68, 636, 68, 634, 68, 68, 133, 644, 650, 635, + 68, 646, 68, 637, 648, 638, 649, 652, 645, 639, + + 651, 640, 68, 68, 68, 68, 641, 653, 654, 68, + 68, 642, 659, 68, 131, 658, 655, 68, 665, 657, + 68, 660, 664, 68, 68, 656, 661, 68, 662, 68, + 668, 68, 669, 68, 68, 68, 667, 666, 68, 670, + 673, 68, 674, 675, 68, 68, 663, 671, 68, 68, + 679, 68, 676, 672, 677, 68, 68, 678, 68, 68, + 68, 68, 688, 68, 684, 682, 68, 68, 68, 68, + 680, 681, 687, 68, 68, 686, 68, 683, 692, 693, + 685, 694, 68, 68, 691, 689, 690, 68, 696, 697, + 68, 695, 698, 702, 68, 68, 699, 68, 68, 700, + + 68, 68, 701, 704, 705, 68, 68, 68, 68, 68, + 703, 707, 708, 68, 709, 712, 68, 68, 68, 711, + 68, 706, 68, 714, 68, 710, 713, 715, 68, 68, + 68, 68, 68, 726, 720, 721, 716, 68, 68, 717, + 723, 719, 718, 727, 725, 68, 729, 68, 68, 722, + 68, 68, 68, 733, 68, 68, 68, 68, 68, 68, + 728, 68, 68, 741, 732, 731, 742, 735, 68, 730, + 68, 68, 734, 738, 736, 743, 739, 68, 68, 68, + 68, 737, 740, 68, 746, 68, 68, 751, 68, 68, + 68, 744, 745, 68, 68, 747, 753, 68, 755, 756, + + 749, 68, 748, 68, 68, 750, 68, 752, 757, 759, + 68, 68, 68, 754, 68, 68, 762, 758, 68, 761, + 764, 68, 760, 68, 68, 766, 68, 767, 68, 68, + 68, 763, 68, 765, 68, 770, 68, 68, 768, 769, + 775, 771, 779, 68, 68, 772, 68, 68, 68, 129, + 774, 773, 776, 778, 781, 68, 777, 780, 784, 68, + 783, 68, 785, 68, 68, 787, 68, 790, 782, 68, + 791, 68, 68, 68, 788, 789, 792, 68, 786, 68, + 794, 68, 793, 68, 68, 68, 799, 797, 796, 800, + 795, 68, 68, 68, 68, 68, 68, 801, 68, 802, + + 809, 68, 798, 68, 807, 803, 68, 808, 68, 68, + 804, 68, 805, 68, 806, 811, 810, 813, 817, 68, + 68, 68, 814, 819, 68, 812, 821, 68, 68, 815, + 820, 823, 68, 68, 68, 818, 825, 68, 816, 826, + 827, 68, 68, 68, 68, 68, 68, 822, 829, 68, + 831, 68, 830, 824, 68, 832, 835, 834, 836, 68, + 837, 68, 828, 68, 833, 68, 838, 839, 840, 68, + 68, 68, 68, 843, 68, 68, 68, 846, 68, 68, + 68, 842, 68, 68, 68, 850, 68, 68, 841, 68, + 68, 844, 854, 68, 68, 856, 847, 864, 845, 848, + + 851, 849, 857, 852, 853, 68, 858, 68, 68, 68, + 68, 861, 855, 863, 859, 68, 860, 68, 68, 68, + 867, 68, 68, 68, 862, 68, 865, 869, 68, 872, + 68, 68, 68, 68, 68, 68, 866, 868, 68, 870, + 879, 68, 68, 871, 68, 875, 873, 876, 877, 874, + 68, 880, 68, 878, 68, 881, 882, 884, 68, 888, + 885, 883, 886, 887, 68, 889, 68, 890, 891, 68, + 68, 68, 68, 68, 68, 896, 68, 893, 68, 897, + 68, 898, 68, 68, 68, 68, 68, 894, 68, 892, + 68, 903, 68, 895, 904, 899, 68, 900, 68, 68, + + 68, 68, 68, 902, 909, 905, 914, 906, 68, 901, + 68, 907, 68, 912, 68, 908, 911, 68, 68, 910, + 917, 915, 913, 916, 918, 68, 68, 68, 68, 922, + 921, 919, 920, 68, 68, 68, 923, 926, 924, 68, + 925, 68, 68, 927, 68, 928, 68, 68, 68, 930, + 929, 68, 932, 68, 68, 937, 931, 68, 934, 936, + 68, 68, 938, 68, 68, 933, 68, 68, 941, 935, + 68, 68, 943, 68, 940, 68, 946, 68, 68, 939, + 68, 68, 950, 68, 952, 942, 951, 945, 944, 68, + 954, 947, 68, 956, 68, 68, 68, 948, 949, 68, + + 68, 959, 957, 953, 68, 68, 955, 965, 68, 68, + 68, 962, 68, 68, 963, 966, 960, 68, 967, 68, + 972, 68, 968, 958, 970, 961, 964, 971, 68, 68, + 68, 68, 68, 68, 977, 68, 976, 68, 969, 68, + 979, 980, 68, 68, 68, 68, 973, 68, 974, 981, + 975, 68, 68, 68, 988, 982, 978, 68, 68, 68, + 68, 985, 68, 983, 986, 984, 987, 992, 128, 68, + 68, 989, 68, 998, 68, 993, 994, 990, 68, 991, + 68, 995, 68, 999, 996, 68, 997, 1000, 68, 68, + 68, 1007, 1008, 68, 1001, 68, 68, 1002, 1004, 1006, + + 68, 1005, 68, 68, 1003, 68, 68, 68, 1015, 68, + 68, 1018, 126, 1010, 68, 1009, 68, 1012, 1020, 68, + 1011, 68, 68, 68, 1014, 1016, 1023, 1021, 1017, 68, + 1013, 68, 1019, 68, 68, 1024, 1025, 1027, 68, 1022, + 68, 1031, 1026, 68, 68, 68, 68, 1028, 68, 68, + 68, 68, 1035, 68, 1036, 1037, 1029, 68, 1038, 1030, + 1032, 1033, 68, 1042, 68, 1034, 1039, 68, 68, 68, + 68, 1046, 68, 68, 1040, 68, 68, 1041, 1051, 1049, + 1044, 1043, 1045, 68, 1047, 1050, 68, 1048, 1052, 68, + 1054, 68, 1053, 68, 68, 68, 68, 68, 68, 1056, + + 68, 68, 68, 1060, 1057, 1055, 1065, 68, 1058, 1061, + 1066, 68, 1059, 124, 68, 68, 68, 1064, 1070, 68, + 68, 1063, 1062, 1072, 68, 1073, 68, 68, 68, 1075, + 68, 1074, 1067, 1068, 68, 68, 1069, 1077, 1076, 1079, + 1071, 68, 68, 68, 68, 68, 1078, 1081, 68, 68, + 1082, 1083, 68, 68, 68, 1080, 68, 1084, 1085, 1086, + 68, 1088, 68, 1089, 1087, 1093, 68, 1091, 68, 1095, + 68, 68, 1090, 1098, 68, 1092, 68, 68, 1100, 1097, + 68, 68, 1096, 68, 1103, 68, 1094, 68, 1107, 1101, + 68, 1099, 1106, 68, 68, 68, 1110, 68, 68, 1111, + + 68, 1105, 1109, 1102, 68, 68, 68, 1104, 68, 1112, + 1108, 68, 1119, 68, 1117, 68, 1122, 68, 1113, 1114, + 1118, 1116, 68, 68, 1115, 1125, 1120, 1126, 68, 1127, + 68, 1121, 68, 1128, 68, 1129, 68, 1124, 68, 1132, + 68, 68, 68, 1123, 68, 1134, 68, 1137, 1131, 68, + 1133, 68, 68, 68, 1135, 1139, 1141, 68, 1130, 68, + 68, 1140, 1142, 68, 1143, 68, 68, 68, 1147, 68, + 68, 1136, 1144, 1138, 1145, 68, 1146, 68, 68, 1149, + 1148, 1151, 68, 68, 68, 68, 68, 68, 68, 1150, + 1156, 1158, 1153, 1157, 68, 1160, 68, 68, 1152, 68, + + 68, 1159, 1154, 68, 1155, 68, 1161, 1168, 1163, 1164, + 1165, 68, 68, 68, 68, 1162, 68, 1170, 1166, 1167, + 68, 1173, 1171, 1172, 68, 68, 68, 1175, 68, 68, + 1169, 68, 68, 68, 68, 68, 68, 1176, 68, 68, + 1185, 68, 68, 1178, 1179, 1174, 1180, 1184, 1181, 68, + 1177, 68, 1189, 68, 68, 1182, 1183, 1186, 1187, 1191, + 68, 68, 1192, 68, 68, 1188, 1193, 1190, 1194, 68, + 1197, 68, 68, 68, 68, 68, 1198, 1201, 68, 68, + 68, 68, 1195, 68, 68, 1205, 1202, 1200, 1206, 1196, + 1203, 1199, 1207, 1204, 68, 68, 68, 1210, 68, 1208, + + 1209, 68, 68, 68, 68, 68, 68, 1213, 1214, 1218, + 68, 68, 1220, 68, 1212, 1217, 1211, 68, 1215, 68, + 1219, 1222, 68, 68, 1221, 1225, 68, 1223, 68, 1216, + 68, 1226, 1224, 1228, 68, 68, 1229, 1227, 68, 68, + 1233, 68, 1230, 68, 1232, 68, 68, 68, 68, 68, + 1235, 1241, 68, 1238, 68, 68, 1231, 68, 68, 1242, + 68, 68, 1237, 1244, 68, 68, 68, 1239, 1234, 1236, + 1240, 68, 1243, 68, 1253, 68, 68, 68, 68, 68, + 1246, 1254, 68, 68, 1258, 1249, 1245, 1247, 1251, 1248, + 1255, 68, 1250, 1256, 1252, 1257, 1259, 68, 1260, 68, + + 1262, 68, 1263, 68, 1261, 1264, 68, 68, 1266, 68, + 1267, 68, 68, 68, 1270, 68, 68, 68, 1269, 1272, + 68, 68, 68, 1274, 68, 1275, 1265, 68, 1276, 1278, + 68, 1268, 68, 1280, 68, 1271, 1281, 68, 68, 1273, + 68, 1283, 68, 1277, 68, 68, 68, 1284, 68, 1286, + 1282, 1279, 68, 68, 1285, 68, 68, 1290, 68, 1288, + 68, 1295, 68, 68, 1292, 1293, 68, 1296, 1291, 1289, + 1287, 1298, 1294, 68, 1299, 68, 1300, 68, 1301, 68, + 1297, 68, 68, 68, 68, 1305, 1302, 68, 1306, 68, + 1303, 1304, 1308, 68, 1309, 68, 68, 1311, 68, 68, + + 1654, 1310, 68, 1313, 1307, 68, 1315, 68, 68, 68, + 1312, 1316, 1317, 1318, 68, 68, 68, 68, 1314, 68, + 1319, 1322, 1320, 68, 1324, 68, 1325, 68, 1323, 1326, + 1321, 68, 1328, 1329, 68, 68, 68, 1331, 68, 68, + 1332, 1327, 1330, 1333, 68, 68, 1335, 68, 1336, 68, + 1334, 1337, 68, 1338, 68, 68, 68, 68, 1340, 68, + 1341, 1342, 68, 68, 1343, 1344, 68, 68, 1347, 68, + 1339, 1345, 1346, 1348, 68, 1349, 68, 68, 1350, 68, + 68, 68, 68, 68, 1357, 68, 68, 1355, 68, 68, + 68, 1352, 1353, 1354, 68, 1351, 68, 1358, 68, 68, + + 68, 1367, 68, 1356, 1365, 1366, 68, 1361, 68, 1360, + 1363, 68, 1359, 68, 1362, 68, 1364, 1368, 68, 68, + 68, 68, 1376, 68, 1373, 68, 1369, 1377, 68, 1370, + 1371, 68, 1378, 1372, 68, 68, 68, 68, 1375, 1374, + 1380, 1379, 1385, 68, 68, 1386, 68, 68, 68, 1381, + 68, 68, 1383, 1382, 68, 68, 1384, 68, 1393, 1394, + 68, 1388, 1387, 68, 1397, 68, 1389, 68, 1390, 68, + 1391, 68, 1395, 68, 1392, 1399, 68, 1400, 68, 68, + 1401, 1396, 1403, 68, 68, 68, 1398, 68, 1404, 1405, + 1406, 1402, 68, 1407, 68, 68, 68, 1409, 1654, 1410, + + 1412, 68, 1408, 1413, 68, 1420, 1414, 1415, 68, 1417, + 1411, 68, 1416, 68, 68, 68, 68, 68, 68, 1418, + 68, 1419, 68, 68, 68, 1423, 1422, 1425, 68, 68, + 68, 68, 1421, 68, 1424, 68, 1430, 1428, 68, 1429, + 1431, 1426, 1427, 68, 1435, 68, 68, 1437, 68, 68, + 68, 68, 1432, 68, 68, 1436, 1439, 1438, 1433, 1440, + 1442, 68, 1434, 68, 1444, 68, 1441, 68, 1445, 1446, + 68, 1447, 68, 68, 1443, 1448, 68, 1451, 1452, 68, + 68, 68, 68, 68, 1453, 1449, 68, 1450, 68, 1454, + 68, 1457, 68, 68, 1461, 68, 68, 68, 1455, 1456, + + 68, 1465, 68, 1466, 1463, 68, 1458, 1460, 68, 68, + 1459, 1462, 68, 1468, 68, 68, 1464, 68, 68, 68, + 68, 1467, 1474, 1471, 1476, 68, 68, 68, 68, 1469, + 1477, 68, 68, 1482, 68, 1472, 1475, 1473, 1470, 1480, + 1478, 1479, 68, 1483, 68, 1481, 68, 68, 68, 68, + 68, 1485, 1490, 1484, 1488, 1487, 1486, 68, 1492, 68, + 68, 68, 1495, 68, 68, 68, 1489, 68, 1496, 68, + 68, 68, 68, 68, 1497, 1491, 1494, 1501, 1500, 1493, + 1503, 68, 68, 1498, 1505, 68, 68, 1506, 1499, 1507, + 68, 68, 1504, 1510, 68, 1511, 68, 1502, 68, 68, + + 68, 1515, 68, 68, 68, 68, 1508, 1519, 68, 1517, + 68, 1513, 68, 1509, 1654, 1516, 68, 1512, 1522, 68, + 1514, 1523, 68, 68, 1518, 1520, 1521, 68, 1525, 1524, + 1526, 1654, 1527, 1528, 68, 1529, 68, 1530, 68, 68, + 1532, 68, 68, 1533, 1531, 1534, 68, 68, 68, 1537, + 68, 68, 1535, 68, 68, 68, 1536, 68, 1538, 1539, + 68, 68, 68, 1547, 68, 68, 1545, 1548, 68, 1540, + 68, 1541, 68, 68, 1542, 1543, 1549, 1550, 68, 68, + 1553, 1544, 1546, 1551, 1552, 68, 1555, 68, 1554, 68, + 68, 68, 68, 68, 1562, 68, 1559, 1560, 1556, 1563, + + 68, 1564, 68, 68, 1557, 68, 1566, 68, 1567, 68, + 68, 1558, 1561, 68, 1569, 1565, 68, 68, 68, 68, + 68, 1568, 1573, 68, 68, 68, 68, 68, 68, 1570, + 1581, 68, 68, 1572, 1571, 68, 1575, 1582, 1577, 1574, + 1583, 1578, 1580, 1576, 68, 68, 68, 1587, 68, 1584, + 68, 1579, 68, 1591, 1592, 68, 68, 1585, 68, 68, + 1586, 1589, 1588, 68, 68, 68, 68, 68, 1594, 1590, + 1593, 68, 68, 68, 68, 1604, 68, 1601, 1596, 1598, + 1599, 1602, 1595, 68, 1597, 68, 68, 1654, 1605, 68, + 1600, 1606, 1603, 1609, 1607, 1610, 68, 68, 68, 68, + + 68, 68, 1614, 68, 1608, 68, 1611, 68, 1617, 68, + 68, 68, 68, 1621, 68, 1654, 1612, 68, 1616, 1613, + 1615, 68, 1622, 68, 1620, 68, 1624, 68, 1618, 1619, + 68, 1625, 1626, 1623, 68, 1629, 68, 68, 68, 68, + 1628, 1627, 1632, 68, 1630, 1631, 68, 68, 1633, 68, + 68, 1635, 1636, 68, 1639, 68, 68, 68, 1634, 68, + 1642, 1643, 68, 68, 1644, 1645, 68, 68, 1637, 1638, + 1646, 68, 68, 68, 1640, 1641, 68, 1654, 1648, 1647, + 1654, 1649, 1650, 1652, 68, 1654, 1651, 1653, 68, 40, 40, 40, 40, 40, 40, 40, 45, 45, 45, 45, + 45, 45, 45, 50, 50, 50, 50, 50, 50, 50, 56, 56, 56, 56, 56, 56, 56, 61, 61, 61, - 61, 61, 61, 61, 71, 71, 1610, 71, 71, 71, - 71, 123, 123, 1610, 1610, 1610, 123, 123, 125, 125, - 1610, 1610, 125, 1610, 125, 127, 1610, 1610, 1610, 1610, - 1610, 127, 130, 130, 1610, 1610, 1610, 130, 130, 132, - 1610, 1610, 1610, 1610, 1610, 132, 134, 134, 1610, 134, - 134, 134, 134, 72, 72, 1610, 72, 72, 72, 72, - - 13, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, - 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, - 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, - 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610 + 61, 61, 61, 61, 71, 71, 1654, 71, 71, 71, + 71, 123, 123, 1654, 1654, 1654, 123, 123, 125, 125, + 1654, 1654, 125, 1654, 125, 127, 1654, 1654, 1654, 1654, + 1654, 127, 130, 130, 1654, 1654, 1654, 130, 130, 132, + 1654, 1654, 1654, 1654, 1654, 132, 134, 134, 1654, 134, + 134, 134, 134, 72, 72, 1654, 72, 72, 72, 72, + 13, 1654, 1654, 1654, 1654, 1654, 1654, 1654, 1654, 1654, + 1654, 1654, 1654, 1654, 1654, 1654, 1654, 1654, 1654, 1654, + + 1654, 1654, 1654, 1654, 1654, 1654, 1654, 1654, 1654, 1654, + 1654, 1654, 1654, 1654, 1654, 1654, 1654, 1654, 1654, 1654 } ; -static yyconst flex_int16_t yy_chk[3241] = +static yyconst flex_int16_t yy_chk[3321] = { 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -1330,7 +1352,7 @@ static yyconst flex_int16_t yy_chk[3241] = 9, 151, 33, 6, 7, 7, 7, 7, 9, 7, 10, 10, 10, 44, 44, 7, 8, 8, 8, 8, 10, 8, 21, 33, 151, 21, 21, 8, 11, 11, - 11, 11, 11, 11, 1616, 20, 29, 20, 20, 11, + 11, 11, 11, 11, 1660, 20, 29, 20, 20, 11, 20, 29, 24, 21, 25, 20, 24, 28, 11, 12, 12, 12, 12, 12, 12, 74, 22, 22, 74, 25, @@ -1348,7 +1370,7 @@ static yyconst flex_int16_t yy_chk[3241] = 81, 83, 78, 68, 71, 79, 71, 71, 84, 71, 85, 80, 86, 84, 71, 71, 87, 88, 81, 82, 83, 89, 90, 91, 85, 89, 84, 90, 94, 97, - 92, 625, 86, 92, 87, 88, 92, 91, 93, 95, + 92, 631, 86, 92, 87, 88, 92, 91, 93, 95, 98, 104, 96, 93, 95, 96, 103, 94, 99, 96, 92, 100, 99, 97, 98, 102, 105, 106, 100, 103, 105, 104, 102, 107, 95, 108, 109, 111, 107, 112, @@ -1359,324 +1381,333 @@ static yyconst flex_int16_t yy_chk[3241] = 121, 126, 131, 131, 119, 140, 139, 128, 122, 128, 128, 133, 128, 133, 133, 134, 133, 134, 134, 136, 134, 139, 136, 137, 138, 141, 134, 142, 137, 138, - 143, 149, 144, 145, 130, 143, 143, 144, 145, 146, + 143, 149, 144, 145, 176, 143, 143, 144, 145, 146, 141, 146, 148, 148, 150, 154, 153, 142, 152, 150, - 149, 153, 152, 155, 157, 156, 159, 159, 158, 154, - 160, 146, 147, 158, 162, 161, 147, 156, 155, 162, - 156, 147, 157, 163, 163, 167, 147, 177, 166, 164, - - 160, 161, 147, 147, 164, 177, 165, 166, 168, 169, - 164, 165, 170, 168, 169, 171, 167, 170, 172, 174, - 171, 175, 176, 178, 180, 172, 179, 175, 181, 180, - 182, 183, 184, 185, 186, 185, 183, 187, 178, 174, - 188, 186, 187, 189, 176, 193, 179, 193, 182, 182, - 181, 191, 184, 190, 190, 189, 191, 192, 194, 188, - 195, 196, 192, 197, 193, 199, 198, 200, 197, 196, - 194, 198, 202, 201, 191, 203, 204, 204, 203, 195, - 199, 201, 208, 205, 209, 200, 205, 206, 206, 207, - 207, 210, 211, 212, 210, 213, 211, 202, 208, 209, - - 213, 214, 212, 215, 215, 216, 217, 218, 219, 220, - 216, 217, 221, 222, 222, 214, 223, 224, 226, 225, - 229, 224, 231, 226, 228, 221, 219, 218, 232, 227, - 220, 225, 227, 230, 223, 233, 228, 229, 235, 235, - 229, 232, 230, 231, 237, 238, 234, 233, 234, 234, - 236, 240, 240, 236, 239, 242, 242, 241, 243, 244, - 238, 237, 241, 239, 237, 245, 246, 247, 249, 250, - 245, 248, 247, 243, 248, 251, 248, 254, 252, 244, - 255, 256, 249, 252, 251, 250, 246, 253, 250, 257, - 250, 258, 253, 259, 258, 262, 256, 261, 254, 263, - - 255, 265, 264, 257, 262, 296, 296, 259, 266, 268, - 258, 260, 261, 260, 260, 272, 265, 267, 269, 270, - 263, 264, 267, 260, 273, 260, 260, 260, 266, 268, - 260, 271, 274, 269, 270, 272, 271, 274, 273, 275, - 276, 277, 277, 278, 277, 276, 279, 281, 280, 282, - 283, 282, 284, 285, 283, 275, 286, 287, 302, 302, - 290, 281, 279, 278, 280, 286, 287, 285, 288, 290, - 284, 289, 291, 288, 292, 289, 294, 291, 286, 293, - 292, 294, 295, 293, 297, 298, 300, 299, 301, 297, - 304, 308, 301, 129, 303, 295, 299, 309, 300, 303, - - 305, 305, 307, 307, 310, 309, 298, 311, 313, 314, - 315, 304, 308, 310, 317, 316, 318, 319, 320, 311, - 316, 318, 317, 321, 315, 325, 322, 323, 313, 314, - 322, 324, 326, 329, 328, 319, 324, 332, 330, 334, - 320, 325, 323, 330, 321, 333, 333, 331, 330, 329, - 331, 326, 327, 327, 328, 335, 335, 332, 327, 336, - 327, 337, 338, 339, 334, 340, 341, 342, 327, 343, - 344, 341, 347, 337, 336, 345, 342, 338, 327, 345, - 348, 346, 339, 349, 347, 344, 346, 340, 343, 350, - 350, 351, 352, 354, 353, 356, 349, 353, 348, 355, - - 358, 357, 355, 351, 352, 357, 357, 361, 354, 359, - 359, 356, 361, 358, 356, 360, 360, 362, 362, 363, - 363, 364, 365, 366, 371, 367, 364, 368, 366, 367, - 369, 370, 368, 372, 369, 373, 370, 374, 372, 377, - 373, 127, 375, 365, 371, 375, 376, 376, 378, 379, - 375, 374, 380, 378, 375, 380, 381, 377, 379, 382, - 381, 383, 384, 385, 382, 386, 387, 384, 392, 388, - 389, 387, 125, 389, 383, 388, 393, 386, 390, 385, - 391, 390, 381, 394, 395, 391, 392, 395, 394, 396, - 393, 397, 398, 399, 396, 397, 397, 398, 400, 401, - - 403, 400, 399, 401, 401, 402, 404, 402, 404, 405, - 406, 407, 408, 409, 410, 415, 400, 409, 411, 403, - 412, 413, 421, 410, 414, 416, 416, 405, 406, 414, - 407, 417, 408, 411, 415, 417, 413, 418, 412, 419, - 420, 421, 422, 423, 419, 425, 424, 428, 418, 426, - 423, 424, 425, 427, 426, 422, 420, 429, 427, 430, - 431, 432, 435, 444, 429, 428, 432, 433, 430, 434, - 433, 436, 436, 434, 431, 439, 438, 440, 441, 439, - 436, 438, 442, 435, 443, 447, 453, 450, 444, 440, - 442, 454, 460, 459, 455, 453, 441, 454, 455, 123, - - 447, 456, 443, 445, 450, 458, 456, 463, 445, 465, - 458, 459, 445, 461, 463, 445, 461, 460, 462, 462, - 464, 464, 445, 469, 465, 445, 457, 470, 457, 67, - 457, 466, 466, 457, 467, 468, 473, 467, 457, 471, - 468, 472, 472, 469, 457, 457, 470, 471, 474, 475, - 477, 473, 476, 476, 475, 477, 478, 479, 480, 481, - 479, 478, 482, 483, 480, 485, 474, 481, 484, 487, - 482, 488, 485, 484, 486, 486, 488, 487, 489, 491, - 492, 483, 491, 493, 493, 494, 497, 495, 498, 499, - 495, 500, 501, 498, 492, 502, 503, 489, 495, 504, - - 494, 505, 506, 504, 507, 503, 508, 497, 510, 511, - 499, 500, 509, 507, 501, 502, 515, 512, 509, 505, - 508, 506, 512, 511, 513, 516, 514, 517, 510, 513, - 514, 518, 517, 519, 519, 515, 520, 516, 522, 521, - 523, 520, 521, 524, 525, 526, 526, 528, 524, 527, - 527, 529, 518, 523, 530, 529, 522, 531, 532, 534, - 535, 536, 528, 534, 525, 537, 538, 553, 61, 539, - 540, 538, 532, 530, 541, 535, 546, 542, 541, 553, - 537, 531, 533, 539, 540, 533, 536, 533, 545, 543, - 544, 533, 542, 533, 543, 544, 547, 546, 533, 548, - - 558, 549, 545, 552, 543, 548, 549, 550, 550, 551, - 547, 554, 554, 550, 551, 550, 555, 552, 556, 558, - 557, 555, 559, 559, 560, 562, 556, 564, 560, 563, - 562, 565, 563, 550, 557, 566, 567, 569, 568, 571, - 566, 573, 576, 570, 565, 578, 564, 568, 570, 572, - 573, 577, 574, 579, 611, 567, 569, 574, 579, 571, - 578, 576, 580, 572, 582, 581, 611, 580, 590, 577, - 581, 583, 583, 582, 584, 588, 586, 589, 588, 584, - 586, 586, 587, 587, 591, 592, 587, 591, 590, 593, - 594, 595, 589, 596, 593, 594, 597, 595, 598, 598, - - 600, 599, 597, 602, 601, 603, 592, 601, 604, 596, - 599, 606, 605, 608, 607, 610, 606, 609, 600, 607, - 609, 602, 612, 614, 603, 605, 610, 612, 604, 613, - 616, 608, 615, 615, 613, 618, 617, 619, 614, 620, - 622, 619, 621, 623, 626, 627, 628, 618, 629, 616, - 617, 621, 56, 629, 637, 632, 630, 620, 626, 622, - 627, 630, 631, 633, 635, 634, 628, 631, 623, 632, - 634, 636, 639, 637, 638, 638, 640, 633, 641, 635, - 642, 642, 640, 643, 644, 645, 636, 647, 643, 646, - 639, 644, 646, 648, 650, 649, 651, 652, 641, 645, - - 649, 651, 648, 653, 655, 647, 654, 653, 656, 657, - 658, 659, 660, 650, 661, 654, 652, 663, 662, 655, - 661, 657, 664, 656, 667, 669, 745, 668, 745, 658, - 660, 664, 659, 662, 666, 668, 670, 663, 667, 666, - 672, 670, 671, 673, 674, 669, 671, 671, 683, 675, - 676, 676, 677, 674, 675, 678, 673, 677, 680, 672, - 678, 679, 679, 681, 682, 684, 683, 680, 681, 685, - 684, 686, 688, 687, 685, 691, 686, 687, 682, 691, - 689, 688, 692, 693, 694, 695, 688, 696, 688, 697, - 688, 689, 698, 699, 701, 696, 706, 692, 694, 700, - - 700, 693, 705, 697, 695, 702, 702, 705, 704, 701, - 698, 704, 707, 707, 708, 706, 709, 709, 699, 710, - 711, 711, 712, 713, 710, 714, 715, 717, 713, 716, - 716, 715, 718, 714, 708, 721, 719, 726, 718, 722, - 721, 723, 712, 719, 722, 717, 723, 724, 725, 727, - 728, 728, 724, 729, 730, 732, 726, 725, 731, 733, - 727, 736, 734, 731, 735, 735, 737, 738, 748, 739, - 750, 740, 748, 729, 746, 732, 740, 730, 742, 733, - 734, 736, 742, 742, 743, 738, 744, 749, 746, 750, - 747, 737, 739, 743, 744, 747, 751, 752, 749, 753, - - 754, 755, 756, 759, 753, 757, 756, 758, 761, 760, - 765, 762, 763, 769, 51, 752, 762, 759, 754, 763, - 764, 755, 751, 757, 760, 764, 758, 761, 765, 766, - 767, 768, 773, 773, 769, 770, 771, 768, 766, 772, - 770, 771, 775, 775, 772, 767, 776, 777, 778, 779, - 781, 776, 782, 779, 779, 780, 780, 783, 784, 785, - 787, 777, 788, 786, 791, 785, 789, 778, 786, 781, - 790, 782, 792, 793, 794, 794, 796, 784, 791, 787, - 796, 788, 795, 783, 797, 789, 798, 795, 793, 790, - 799, 800, 800, 792, 801, 798, 802, 797, 803, 801, - - 802, 804, 806, 808, 809, 809, 804, 810, 808, 819, - 799, 818, 810, 812, 812, 814, 814, 820, 803, 816, - 806, 822, 816, 817, 817, 821, 818, 823, 823, 824, - 821, 820, 825, 825, 819, 826, 822, 827, 828, 829, - 830, 832, 828, 834, 834, 830, 833, 835, 827, 824, - 836, 837, 838, 841, 838, 842, 839, 826, 829, 832, - 833, 839, 846, 840, 843, 843, 844, 835, 840, 842, - 836, 837, 844, 841, 847, 847, 848, 849, 850, 855, - 851, 850, 852, 853, 850, 851, 846, 852, 848, 854, - 856, 859, 853, 856, 854, 857, 850, 849, 858, 858, - - 857, 860, 861, 862, 855, 863, 864, 867, 862, 859, - 866, 864, 865, 865, 868, 866, 869, 870, 871, 872, - 860, 873, 861, 874, 875, 867, 873, 876, 863, 880, - 871, 870, 872, 879, 868, 878, 869, 881, 880, 885, - 879, 880, 885, 874, 875, 884, 884, 886, 876, 887, - 878, 888, 886, 881, 889, 890, 892, 891, 893, 893, - 888, 892, 894, 895, 890, 887, 891, 890, 896, 897, - 898, 899, 901, 889, 900, 900, 895, 908, 902, 904, - 905, 907, 894, 897, 904, 906, 896, 907, 901, 899, - 906, 910, 915, 914, 909, 898, 902, 908, 905, 909, - - 911, 914, 916, 911, 910, 915, 917, 918, 919, 911, - 920, 922, 918, 923, 924, 928, 925, 923, 926, 924, - 916, 925, 929, 926, 919, 917, 927, 931, 920, 930, - 930, 922, 932, 933, 934, 927, 935, 929, 928, 934, - 936, 931, 50, 937, 932, 933, 938, 938, 935, 937, - 939, 939, 941, 942, 942, 943, 945, 944, 936, 946, - 947, 941, 944, 948, 949, 945, 951, 943, 948, 950, - 954, 946, 956, 949, 953, 947, 952, 952, 955, 953, - 958, 951, 957, 957, 959, 959, 949, 960, 960, 950, - 961, 954, 956, 962, 964, 961, 962, 955, 965, 964, - - 958, 966, 967, 968, 968, 970, 966, 969, 969, 971, - 970, 972, 973, 974, 975, 977, 965, 976, 976, 978, - 978, 971, 979, 971, 983, 981, 986, 967, 971, 982, - 982, 972, 973, 974, 981, 975, 984, 985, 988, 983, - 989, 977, 979, 984, 987, 987, 985, 993, 991, 992, - 994, 998, 986, 991, 992, 996, 989, 994, 995, 995, - 996, 997, 999, 1000, 988, 1001, 1002, 993, 1005, 1003, - 997, 998, 1002, 1004, 1004, 1006, 1006, 999, 1003, 1005, - 1001, 1007, 1000, 1008, 1005, 1009, 1010, 1010, 1011, 1012, - 1009, 1013, 1011, 1011, 1012, 1014, 1015, 1008, 1016, 1017, - - 1018, 1015, 1007, 1017, 1019, 1021, 1020, 1014, 1022, 1023, - 1016, 1020, 1018, 1022, 45, 1025, 1031, 1013, 1025, 1026, - 1026, 1023, 1027, 1027, 1028, 1031, 1029, 1021, 1030, 1019, - 1028, 1029, 1034, 1036, 1030, 1037, 1037, 1038, 1039, 1042, - 1040, 1041, 1045, 1042, 1034, 1036, 1043, 1039, 1044, 1047, - 1047, 1043, 1048, 1044, 1049, 1050, 1045, 1038, 1040, 1055, - 1041, 1047, 1056, 1049, 1050, 1052, 1052, 1056, 1057, 1063, - 1048, 1058, 1058, 1055, 1059, 1062, 1059, 1060, 1060, 1064, - 1064, 1065, 1062, 1066, 1067, 1057, 1068, 1070, 1069, 1063, - 1074, 1073, 1072, 1073, 1079, 1066, 1067, 1072, 1068, 1077, - - 1075, 1080, 1065, 1069, 1078, 1078, 1080, 1070, 1074, 1075, - 1081, 1082, 1081, 1077, 1083, 1084, 1082, 1085, 1086, 1083, - 1079, 1087, 1089, 1086, 1090, 40, 1087, 1091, 1091, 1102, - 1092, 1093, 1094, 1084, 1094, 1095, 1096, 1085, 1090, 1101, - 1093, 1089, 1092, 1093, 1095, 1098, 1099, 1103, 1096, 1102, - 1098, 1099, 1101, 1105, 1107, 1108, 1109, 1110, 1112, 1105, - 1107, 1111, 1111, 1103, 1113, 1113, 1110, 1112, 1114, 1115, - 1108, 1116, 1117, 1115, 1118, 1118, 1116, 1129, 1119, 1122, - 1125, 1109, 1123, 1123, 1131, 1117, 1119, 1114, 1122, 1124, - 1124, 1126, 1125, 1127, 1127, 1130, 1126, 1132, 1133, 1134, - - 1130, 1138, 1129, 1133, 1135, 1136, 1136, 1131, 1140, 1137, - 1139, 1141, 1132, 1137, 1143, 1139, 1138, 1134, 1144, 1145, - 1135, 1146, 1147, 1149, 1149, 1150, 1151, 14, 1150, 1152, - 1141, 1153, 1153, 1146, 1140, 1154, 1154, 1144, 1143, 1151, - 1145, 1156, 1152, 1147, 1155, 1155, 1157, 1157, 1158, 1158, - 1159, 1160, 1160, 1162, 1156, 1161, 1161, 1164, 1165, 1165, - 1166, 1169, 1164, 1167, 1167, 1168, 1173, 1169, 1170, 1159, - 1177, 1170, 1162, 1171, 1171, 1172, 1172, 1174, 1174, 1166, - 1176, 1176, 1179, 1168, 1182, 1173, 1178, 1178, 1180, 1171, - 1181, 1179, 1183, 1184, 1185, 1181, 1177, 1186, 1187, 1185, - - 1180, 1188, 1189, 1190, 1193, 1183, 1200, 1187, 1190, 1182, - 1188, 1191, 1186, 1195, 1184, 1191, 1189, 1196, 1196, 1201, - 1197, 1200, 1195, 1204, 1193, 1197, 1198, 1198, 1204, 1203, - 1208, 1201, 1203, 1205, 1205, 1198, 1207, 1207, 1209, 1209, - 1210, 1211, 1212, 1215, 1208, 1213, 1213, 1215, 1216, 1217, - 1217, 1216, 1210, 1218, 1219, 1211, 1220, 1221, 1218, 1222, - 1220, 1212, 1223, 1224, 1222, 1226, 1227, 1223, 1228, 1224, - 1229, 1227, 1229, 1246, 1219, 1238, 1228, 1221, 1232, 1232, - 1238, 1226, 1235, 1235, 1239, 1239, 1241, 1241, 1243, 1244, - 1244, 1245, 1247, 1243, 1254, 1246, 1252, 1247, 1249, 1249, - - 1253, 1252, 1253, 1259, 1255, 1256, 1256, 1245, 1254, 1255, - 1257, 1257, 1258, 1258, 1260, 1259, 1261, 1262, 1263, 1264, - 1265, 1266, 1266, 1268, 1264, 1267, 1269, 1271, 1261, 1262, - 1263, 1270, 1260, 1272, 1267, 1273, 1274, 1275, 1276, 1276, - 1265, 1274, 1275, 1279, 1270, 1281, 1269, 1272, 1280, 1268, - 1282, 1271, 1283, 1273, 1284, 1285, 1286, 1286, 1288, 1288, - 1289, 1283, 1290, 1279, 1291, 1292, 1280, 1281, 1293, 1289, - 1282, 1294, 1297, 1285, 1284, 1295, 1295, 1290, 1298, 1299, - 1296, 1301, 1291, 1292, 1293, 1296, 1304, 1297, 1305, 1307, - 1294, 1305, 1306, 1306, 1308, 1310, 1310, 1298, 1311, 1299, - - 1301, 1312, 1312, 1318, 1307, 1319, 1304, 1313, 1313, 1317, - 1317, 1320, 1308, 1321, 1319, 1322, 1320, 1311, 1321, 1323, - 1325, 1326, 1328, 1327, 1342, 1318, 1326, 1323, 1327, 1329, - 1329, 1330, 1330, 1331, 1322, 1325, 1342, 1328, 1331, 1332, - 1332, 1333, 1333, 1335, 1338, 1336, 1337, 1339, 1335, 1336, - 1340, 1337, 1341, 1338, 1345, 1346, 1344, 1341, 1340, 1344, - 1347, 1349, 1339, 1351, 1346, 1348, 1348, 1345, 1352, 1347, - 1349, 1353, 1353, 1356, 1354, 1355, 1355, 1359, 1360, 1357, - 1365, 1356, 1351, 1354, 1357, 1371, 1359, 1352, 1362, 1362, - 1372, 1360, 1366, 1366, 1367, 1368, 1368, 1367, 1369, 1369, - - 1365, 1370, 1370, 1371, 1372, 1373, 1374, 1374, 1375, 1378, - 1373, 1381, 1375, 1382, 1383, 1378, 1384, 1385, 1382, 1386, - 1386, 1387, 1388, 1389, 1390, 1390, 1393, 1391, 1381, 1381, - 1388, 1385, 1383, 1392, 1394, 1395, 1384, 1396, 1392, 1407, - 1387, 1400, 1389, 1391, 1395, 1398, 1393, 1402, 1398, 1401, - 1401, 1402, 1403, 1405, 1406, 1412, 1412, 1396, 1411, 1400, - 1394, 1403, 1405, 1411, 1407, 1413, 1406, 1415, 1416, 1417, - 1418, 1419, 1415, 1420, 1413, 1418, 1421, 1416, 1420, 1422, - 1422, 1424, 1425, 1426, 1426, 1427, 1428, 1429, 1430, 1419, - 1417, 1432, 1433, 1431, 1427, 1438, 1430, 1425, 1431, 1439, - - 1424, 1421, 1428, 1434, 1434, 1435, 1429, 1437, 1437, 1442, - 1435, 1433, 1440, 1440, 1443, 1438, 1444, 1432, 1441, 1441, - 1446, 1439, 1445, 1445, 1447, 1448, 1443, 1454, 1442, 1447, - 1449, 1450, 1446, 1451, 1451, 1449, 1444, 1453, 1453, 1456, - 1457, 1458, 1458, 13, 1448, 1457, 1454, 1461, 1450, 1450, - 1459, 1459, 1461, 1456, 1460, 1460, 1462, 1462, 1464, 1464, - 1466, 1466, 1467, 1470, 1471, 1471, 1475, 1467, 1472, 1474, - 1476, 1470, 1477, 1472, 1474, 1478, 1479, 1482, 1483, 1483, - 1493, 1479, 1497, 1484, 1475, 1495, 1476, 1484, 1499, 1477, - 1477, 1488, 1488, 1498, 1490, 1493, 1478, 1482, 1490, 1492, - - 1492, 1496, 1496, 1495, 1502, 1500, 1501, 1497, 1498, 1499, - 1500, 1501, 1503, 1503, 1505, 1505, 1507, 1507, 1509, 1510, - 1510, 1512, 1513, 1502, 1514, 1515, 1512, 1516, 1521, 1514, - 1509, 1517, 1521, 1522, 1525, 1523, 1513, 1524, 1526, 1527, - 1528, 1529, 1529, 1533, 1515, 1516, 1531, 1517, 1523, 1530, - 1534, 1531, 1525, 1522, 1526, 1528, 1524, 1532, 1530, 1536, - 1538, 1535, 1532, 1539, 1527, 1534, 1535, 1540, 1533, 1541, - 1542, 1544, 1540, 1536, 1541, 1538, 1543, 1545, 1546, 1547, - 1548, 1549, 1539, 1550, 1542, 1552, 1553, 1553, 1550, 1543, - 1544, 1545, 1547, 1548, 1551, 1557, 1554, 1546, 1555, 1551, - - 1549, 1554, 1556, 1555, 1552, 1558, 1560, 1556, 1559, 1559, - 1558, 1561, 1562, 1560, 1557, 1563, 1564, 1567, 1568, 1565, - 1563, 1569, 1567, 1570, 1571, 1571, 1573, 1578, 1572, 1576, - 1561, 1562, 1565, 1572, 1576, 1564, 1570, 1568, 1574, 1574, - 1569, 1575, 1575, 1577, 1580, 1573, 1578, 1579, 1579, 1581, - 1577, 1580, 1582, 1583, 1584, 1586, 1581, 1587, 1583, 1588, - 1586, 1589, 1587, 1590, 1590, 1591, 1592, 1593, 1593, 1594, - 1594, 1582, 1595, 1584, 1596, 1596, 1597, 1597, 1588, 1595, - 1589, 1598, 1600, 1604, 1591, 1592, 1603, 1600, 1598, 1605, - 0, 1603, 1606, 1606, 1607, 1607, 0, 0, 0, 0, - - 0, 0, 1604, 0, 0, 0, 0, 0, 1605, 1611, - 1611, 1611, 1611, 1611, 1611, 1611, 1612, 1612, 1612, 1612, - 1612, 1612, 1612, 1613, 1613, 1613, 1613, 1613, 1613, 1613, - 1614, 1614, 1614, 1614, 1614, 1614, 1614, 1615, 1615, 1615, - 1615, 1615, 1615, 1615, 1617, 1617, 0, 1617, 1617, 1617, - 1617, 1618, 1618, 0, 0, 0, 1618, 1618, 1619, 1619, - 0, 0, 1619, 0, 1619, 1620, 0, 0, 0, 0, - 0, 1620, 1621, 1621, 0, 0, 0, 1621, 1621, 1622, - 0, 0, 0, 0, 0, 1622, 1623, 1623, 0, 1623, - 1623, 1623, 1623, 1624, 1624, 0, 1624, 1624, 1624, 1624, - - 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, - 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, - 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, - 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610 + 149, 153, 152, 155, 157, 156, 176, 160, 158, 154, + 130, 146, 147, 158, 159, 159, 147, 156, 155, 161, + 156, 147, 157, 162, 163, 163, 147, 160, 162, 167, + + 166, 164, 147, 147, 159, 161, 164, 174, 165, 166, + 168, 169, 164, 165, 170, 168, 169, 171, 172, 170, + 167, 178, 171, 175, 177, 172, 179, 174, 180, 175, + 181, 182, 177, 180, 183, 184, 178, 186, 185, 183, + 185, 188, 187, 191, 186, 189, 179, 187, 191, 182, + 182, 192, 181, 190, 190, 184, 192, 189, 194, 193, + 188, 193, 195, 196, 197, 199, 191, 198, 200, 197, + 194, 196, 198, 202, 201, 129, 203, 208, 193, 203, + 199, 195, 201, 204, 204, 205, 200, 209, 205, 206, + 206, 207, 207, 208, 210, 211, 212, 210, 202, 211, + + 214, 213, 209, 215, 215, 212, 213, 216, 217, 218, + 219, 220, 216, 217, 214, 222, 222, 221, 223, 224, + 225, 226, 227, 224, 230, 229, 231, 227, 219, 218, + 221, 228, 220, 226, 228, 231, 223, 229, 225, 232, + 233, 230, 234, 235, 230, 235, 235, 236, 236, 237, + 238, 239, 237, 233, 234, 240, 241, 241, 242, 244, + 232, 243, 243, 242, 240, 245, 239, 238, 246, 247, + 238, 248, 249, 246, 244, 249, 248, 249, 250, 251, + 255, 252, 253, 254, 256, 245, 265, 253, 254, 247, + 252, 257, 250, 258, 259, 251, 260, 259, 251, 263, + + 251, 255, 264, 262, 256, 265, 257, 258, 263, 266, + 260, 267, 269, 259, 261, 268, 261, 261, 262, 270, + 268, 271, 272, 264, 266, 273, 261, 272, 261, 261, + 261, 267, 269, 261, 270, 274, 271, 275, 276, 277, + 278, 278, 275, 278, 277, 273, 279, 280, 281, 274, + 283, 282, 283, 284, 276, 285, 286, 284, 287, 288, + 295, 295, 127, 280, 281, 282, 279, 287, 288, 289, + 286, 290, 291, 285, 289, 290, 297, 292, 293, 294, + 287, 291, 292, 294, 293, 296, 298, 298, 299, 297, + 296, 300, 301, 299, 302, 303, 304, 304, 305, 303, + + 306, 301, 310, 305, 307, 307, 302, 309, 309, 311, + 312, 315, 300, 313, 316, 317, 319, 311, 321, 312, + 322, 306, 318, 310, 319, 313, 323, 318, 320, 317, + 325, 315, 326, 320, 316, 324, 321, 326, 327, 324, + 328, 331, 322, 330, 334, 325, 332, 323, 336, 333, + 338, 332, 333, 342, 327, 339, 332, 331, 125, 328, + 329, 329, 340, 330, 334, 338, 329, 339, 329, 335, + 335, 337, 337, 336, 341, 342, 329, 340, 343, 344, + 345, 346, 348, 343, 351, 347, 329, 348, 344, 347, + 349, 350, 353, 341, 352, 352, 346, 351, 354, 345, + + 355, 356, 349, 355, 353, 357, 358, 359, 357, 350, + 354, 359, 359, 360, 361, 361, 356, 362, 362, 363, + 364, 364, 358, 367, 363, 358, 360, 365, 365, 366, + 368, 373, 369, 370, 366, 368, 369, 371, 370, 376, + 372, 371, 374, 375, 367, 372, 377, 374, 375, 377, + 379, 373, 380, 376, 377, 378, 378, 381, 377, 384, + 382, 383, 381, 384, 383, 385, 380, 386, 379, 382, + 385, 387, 388, 390, 389, 391, 387, 395, 390, 392, + 386, 391, 392, 396, 393, 384, 389, 393, 388, 394, + 398, 397, 406, 398, 394, 395, 397, 396, 399, 402, + + 400, 401, 408, 399, 400, 400, 401, 403, 402, 404, + 403, 406, 409, 404, 404, 405, 407, 405, 407, 410, + 408, 411, 412, 413, 414, 403, 412, 415, 416, 417, + 409, 418, 413, 420, 417, 419, 419, 420, 410, 414, + 421, 411, 422, 416, 423, 415, 424, 422, 425, 426, + 418, 421, 427, 429, 430, 428, 426, 427, 429, 430, + 423, 425, 428, 431, 432, 424, 433, 435, 434, 438, + 436, 432, 435, 436, 437, 433, 439, 439, 437, 441, + 443, 431, 434, 442, 441, 439, 444, 442, 445, 446, + 438, 447, 443, 450, 453, 457, 445, 456, 458, 463, + + 459, 457, 458, 462, 444, 459, 456, 446, 450, 461, + 464, 453, 470, 464, 461, 470, 447, 448, 465, 465, + 466, 462, 448, 468, 463, 473, 448, 466, 123, 448, + 467, 467, 469, 469, 472, 474, 448, 478, 468, 448, + 460, 460, 460, 471, 460, 473, 477, 460, 471, 475, + 476, 476, 460, 472, 474, 478, 67, 475, 460, 460, + 479, 477, 480, 480, 481, 479, 482, 483, 484, 481, + 483, 482, 485, 486, 484, 487, 489, 488, 490, 490, + 485, 486, 488, 489, 491, 492, 493, 496, 495, 498, + 492, 495, 491, 487, 497, 497, 499, 501, 502, 499, + + 503, 496, 504, 502, 498, 493, 505, 499, 507, 506, + 508, 511, 509, 510, 508, 514, 616, 507, 501, 512, + 511, 503, 504, 515, 519, 513, 522, 616, 505, 506, + 509, 513, 510, 512, 516, 514, 517, 515, 518, 516, + 520, 517, 518, 519, 521, 523, 523, 522, 524, 521, + 526, 525, 520, 524, 525, 528, 527, 529, 530, 530, + 528, 531, 531, 532, 534, 535, 533, 536, 526, 527, + 533, 538, 539, 540, 542, 538, 541, 529, 532, 542, + 543, 536, 544, 534, 545, 546, 61, 539, 545, 535, + 537, 541, 547, 537, 543, 537, 544, 547, 540, 537, + + 546, 537, 549, 548, 550, 551, 537, 547, 548, 552, + 555, 537, 553, 556, 56, 552, 549, 553, 556, 551, + 554, 554, 555, 557, 558, 550, 554, 560, 554, 559, + 559, 561, 560, 562, 563, 569, 558, 557, 565, 561, + 564, 564, 565, 567, 570, 571, 554, 562, 567, 568, + 571, 572, 568, 563, 569, 573, 574, 570, 575, 576, + 577, 578, 580, 579, 576, 574, 582, 580, 583, 584, + 572, 573, 579, 585, 586, 578, 587, 575, 585, 586, + 577, 587, 588, 596, 584, 582, 583, 589, 589, 590, + 594, 588, 592, 594, 590, 595, 592, 592, 593, 593, + + 598, 597, 593, 596, 597, 606, 599, 600, 601, 602, + 595, 599, 600, 603, 601, 604, 604, 605, 608, 603, + 609, 598, 610, 606, 607, 602, 605, 607, 611, 612, + 613, 614, 617, 618, 612, 613, 608, 615, 618, 609, + 615, 611, 610, 619, 617, 620, 621, 621, 619, 614, + 622, 623, 624, 625, 626, 628, 629, 625, 627, 632, + 620, 633, 634, 635, 624, 623, 636, 627, 635, 622, + 637, 636, 626, 632, 628, 637, 633, 638, 639, 640, + 641, 629, 634, 642, 640, 643, 644, 645, 645, 646, + 647, 638, 639, 648, 650, 641, 647, 649, 649, 650, + + 643, 651, 642, 652, 654, 644, 653, 646, 651, 653, + 655, 656, 657, 648, 659, 658, 656, 652, 661, 655, + 658, 660, 654, 662, 663, 660, 666, 661, 664, 665, + 667, 657, 668, 659, 669, 664, 671, 670, 662, 663, + 669, 665, 674, 675, 672, 666, 676, 674, 677, 51, + 668, 667, 670, 672, 676, 678, 671, 675, 679, 680, + 678, 682, 679, 679, 681, 681, 683, 684, 677, 685, + 685, 686, 684, 691, 682, 683, 686, 687, 680, 688, + 688, 689, 687, 690, 692, 701, 693, 691, 690, 694, + 689, 693, 695, 698, 694, 696, 697, 695, 702, 696, + + 701, 703, 692, 700, 698, 697, 704, 700, 706, 705, + 697, 707, 697, 708, 697, 703, 702, 705, 709, 709, + 710, 714, 706, 711, 711, 704, 714, 713, 715, 707, + 713, 716, 716, 717, 719, 710, 718, 718, 708, 719, + 720, 720, 721, 722, 723, 724, 726, 715, 722, 728, + 724, 727, 723, 717, 725, 725, 728, 727, 730, 734, + 731, 732, 721, 730, 726, 731, 732, 733, 734, 735, + 736, 738, 733, 737, 737, 739, 741, 740, 746, 742, + 745, 736, 740, 743, 744, 744, 747, 748, 735, 749, + 750, 738, 748, 758, 761, 750, 741, 758, 739, 742, + + 745, 743, 752, 746, 747, 753, 752, 752, 754, 755, + 756, 755, 749, 757, 753, 759, 754, 760, 757, 762, + 761, 764, 763, 765, 756, 766, 759, 763, 767, 766, + 768, 769, 770, 771, 772, 773, 760, 762, 776, 764, + 773, 780, 778, 765, 774, 769, 767, 770, 771, 768, + 775, 774, 777, 772, 779, 775, 776, 778, 781, 782, + 779, 777, 780, 781, 782, 783, 784, 784, 786, 786, + 783, 787, 788, 789, 790, 791, 793, 788, 794, 791, + 791, 792, 792, 795, 796, 797, 799, 789, 800, 787, + 803, 797, 801, 790, 798, 793, 802, 794, 804, 798, + + 811, 805, 808, 796, 803, 799, 808, 800, 809, 795, + 815, 801, 806, 806, 810, 802, 805, 807, 818, 804, + 811, 809, 807, 810, 812, 812, 813, 831, 814, 816, + 815, 813, 814, 820, 816, 836, 818, 822, 820, 821, + 821, 830, 822, 824, 824, 826, 826, 828, 829, 829, + 828, 832, 831, 833, 834, 836, 830, 838, 833, 835, + 835, 837, 837, 839, 840, 832, 841, 842, 840, 834, + 844, 847, 842, 845, 839, 846, 846, 848, 849, 838, + 850, 851, 850, 853, 852, 841, 851, 845, 844, 852, + 854, 847, 855, 856, 856, 854, 857, 848, 849, 859, + + 860, 860, 857, 853, 861, 862, 855, 864, 863, 866, + 865, 863, 864, 868, 863, 865, 861, 867, 866, 871, + 871, 869, 867, 859, 869, 862, 863, 870, 872, 873, + 874, 875, 870, 877, 876, 878, 875, 881, 868, 876, + 878, 879, 879, 882, 880, 883, 872, 884, 873, 880, + 874, 888, 885, 886, 887, 881, 877, 889, 890, 887, + 893, 884, 892, 882, 885, 883, 886, 892, 50, 894, + 896, 888, 895, 899, 899, 893, 894, 889, 902, 890, + 900, 895, 901, 900, 895, 904, 896, 901, 903, 905, + 906, 907, 908, 908, 902, 909, 907, 903, 905, 906, + + 910, 905, 911, 912, 904, 913, 914, 915, 915, 916, + 917, 919, 45, 910, 920, 909, 919, 912, 921, 923, + 911, 922, 925, 921, 914, 916, 924, 922, 917, 929, + 913, 924, 920, 926, 930, 925, 926, 929, 931, 923, + 932, 933, 926, 935, 934, 937, 933, 930, 938, 939, + 940, 943, 938, 941, 939, 940, 931, 942, 941, 932, + 934, 935, 944, 945, 945, 937, 942, 946, 947, 951, + 948, 949, 950, 952, 943, 954, 949, 944, 954, 952, + 947, 946, 948, 957, 950, 953, 953, 951, 955, 955, + 958, 958, 957, 959, 960, 961, 962, 963, 964, 960, + + 965, 966, 967, 964, 961, 959, 968, 968, 962, 965, + 969, 970, 963, 40, 971, 969, 972, 967, 973, 973, + 974, 966, 965, 975, 975, 976, 976, 977, 982, 978, + 984, 977, 970, 971, 978, 979, 972, 981, 979, 983, + 974, 989, 981, 990, 983, 988, 982, 985, 985, 986, + 986, 987, 991, 992, 993, 984, 987, 988, 988, 988, + 995, 989, 997, 990, 988, 994, 994, 992, 996, 996, + 999, 1002, 991, 1000, 1000, 993, 1001, 1004, 1002, 999, + 1003, 1006, 997, 1005, 1005, 1007, 995, 1009, 1010, 1003, + 1011, 1001, 1009, 1010, 1014, 1012, 1013, 1013, 1016, 1014, + + 1015, 1007, 1012, 1004, 1017, 1018, 1019, 1006, 1020, 1015, + 1011, 1021, 1022, 1022, 1020, 1023, 1024, 1024, 1016, 1017, + 1021, 1019, 1025, 1026, 1018, 1027, 1023, 1028, 1028, 1029, + 1027, 1023, 1031, 1029, 1029, 1030, 1032, 1026, 1034, 1033, + 1030, 1035, 1036, 1025, 1033, 1035, 1037, 1038, 1032, 1041, + 1034, 1039, 1038, 1043, 1036, 1040, 1043, 14, 1031, 1049, + 1040, 1041, 1044, 1044, 1045, 1045, 1046, 1053, 1049, 1047, + 1048, 1037, 1046, 1039, 1047, 1051, 1048, 1055, 1057, 1053, + 1051, 1056, 1056, 1058, 1059, 1060, 1061, 1064, 1062, 1055, + 1061, 1063, 1058, 1062, 1066, 1066, 1063, 1067, 1057, 1068, + + 1069, 1064, 1059, 1074, 1060, 1075, 1066, 1076, 1068, 1069, + 1071, 1071, 1076, 1077, 1082, 1067, 1078, 1078, 1074, 1075, + 1079, 1082, 1079, 1080, 1080, 1083, 1084, 1084, 1085, 1086, + 1077, 1087, 1088, 1090, 1089, 1091, 1092, 1085, 1095, 1096, + 1095, 1101, 1094, 1087, 1088, 1083, 1089, 1094, 1090, 1097, + 1086, 1099, 1100, 1100, 1102, 1091, 1092, 1096, 1097, 1102, + 1103, 1104, 1103, 1105, 1106, 1099, 1104, 1101, 1105, 1107, + 1108, 1109, 1111, 1112, 1114, 1108, 1109, 1113, 1113, 1117, + 1118, 1115, 1106, 1116, 1123, 1116, 1114, 1112, 1117, 1107, + 1115, 1111, 1118, 1115, 1120, 1121, 1124, 1123, 1125, 1120, + + 1121, 1127, 1129, 1130, 1131, 1136, 1132, 1127, 1129, 1133, + 1133, 1134, 1135, 1135, 1125, 1132, 1124, 1137, 1130, 1139, + 1134, 1137, 1138, 1141, 1136, 1140, 1140, 1138, 1144, 1131, + 1147, 1141, 1139, 1145, 1145, 1146, 1146, 1144, 1148, 1149, + 1150, 1150, 1147, 1152, 1149, 1153, 1154, 1155, 1156, 1157, + 1153, 1159, 1159, 1156, 1158, 1160, 1148, 1161, 1162, 1160, + 1163, 1164, 1155, 1162, 1166, 1167, 1168, 1157, 1152, 1154, + 1158, 1169, 1161, 1171, 1173, 1173, 1170, 1175, 1174, 1176, + 1164, 1174, 1177, 1178, 1178, 1168, 1163, 1166, 1170, 1167, + 1175, 1181, 1169, 1176, 1171, 1177, 1179, 1179, 1180, 1180, + + 1182, 1182, 1183, 1183, 1181, 1184, 1184, 1185, 1186, 1186, + 1187, 1187, 1188, 1190, 1191, 1191, 1192, 1195, 1190, 1193, + 1193, 1194, 1196, 1195, 1203, 1196, 1185, 1197, 1197, 1198, + 1198, 1188, 1199, 1200, 1200, 1192, 1202, 1202, 1205, 1194, + 1204, 1204, 1206, 1197, 1207, 1208, 1209, 1205, 1210, 1207, + 1203, 1199, 1211, 1212, 1206, 1213, 1214, 1211, 1215, 1209, + 1219, 1216, 1221, 1217, 1213, 1214, 1216, 1217, 1212, 1210, + 1208, 1221, 1215, 1222, 1222, 1226, 1223, 1224, 1224, 1227, + 1219, 1223, 1229, 1230, 1231, 1229, 1224, 1235, 1230, 1238, + 1226, 1227, 1232, 1232, 1234, 1234, 1236, 1236, 1237, 1239, + + 13, 1235, 1246, 1238, 1231, 1240, 1240, 1242, 1249, 1243, + 1237, 1242, 1243, 1244, 1244, 1245, 1247, 1248, 1239, 1250, + 1245, 1248, 1246, 1252, 1250, 1254, 1251, 1255, 1249, 1252, + 1247, 1251, 1255, 1256, 1257, 1258, 1276, 1258, 1256, 1261, + 1261, 1254, 1257, 1265, 1265, 1268, 1269, 1269, 1271, 1271, + 1268, 1273, 1274, 1274, 1275, 1277, 1273, 1284, 1276, 1282, + 1277, 1279, 1279, 1283, 1282, 1283, 1289, 1285, 1286, 1286, + 1275, 1284, 1285, 1287, 1287, 1288, 1288, 1290, 1289, 1291, + 1292, 1293, 1294, 1295, 1296, 1296, 1298, 1294, 1297, 1299, + 1301, 1291, 1292, 1293, 1300, 1290, 1302, 1297, 1303, 1304, + + 1305, 1306, 1306, 1295, 1304, 1305, 1310, 1300, 1312, 1299, + 1302, 1311, 1298, 1313, 1301, 1314, 1303, 1307, 1307, 1315, + 1316, 1317, 1317, 1320, 1314, 1322, 1310, 1319, 1319, 1311, + 1312, 1323, 1320, 1313, 1321, 1324, 1325, 1326, 1316, 1315, + 1322, 1321, 1327, 1327, 1329, 1328, 1330, 1331, 1332, 1323, + 1328, 1334, 1325, 1324, 1337, 1338, 1326, 1340, 1338, 1339, + 1339, 1330, 1329, 1341, 1343, 1343, 1331, 1344, 1332, 1351, + 1334, 1355, 1340, 1352, 1337, 1345, 1345, 1346, 1346, 1350, + 1350, 1341, 1352, 1353, 1354, 1356, 1344, 1358, 1353, 1354, + 1355, 1351, 1359, 1356, 1360, 1361, 1372, 1359, 0, 1360, + + 1362, 1362, 1358, 1363, 1363, 1372, 1364, 1365, 1365, 1369, + 1361, 1364, 1366, 1366, 1369, 1370, 1371, 1373, 1374, 1370, + 1375, 1371, 1376, 1379, 1378, 1375, 1374, 1378, 1381, 1380, + 1382, 1384, 1373, 1386, 1376, 1383, 1383, 1381, 1387, 1382, + 1384, 1379, 1380, 1388, 1389, 1389, 1390, 1391, 1391, 1392, + 1395, 1393, 1386, 1396, 1401, 1390, 1393, 1392, 1387, 1395, + 1398, 1398, 1388, 1402, 1402, 1403, 1396, 1407, 1403, 1404, + 1404, 1405, 1405, 1408, 1401, 1406, 1406, 1409, 1410, 1410, + 1411, 1417, 1409, 1414, 1411, 1407, 1418, 1408, 1419, 1414, + 1420, 1418, 1423, 1421, 1422, 1422, 1424, 1425, 1417, 1417, + + 1426, 1426, 1427, 1427, 1424, 1428, 1419, 1421, 1429, 1430, + 1420, 1423, 1431, 1429, 1432, 1433, 1425, 1434, 1438, 1436, + 1445, 1428, 1436, 1432, 1439, 1439, 1440, 1444, 1449, 1430, + 1440, 1441, 1443, 1449, 1455, 1433, 1438, 1434, 1431, 1444, + 1441, 1443, 1450, 1450, 1451, 1445, 1453, 1454, 1457, 1456, + 1459, 1453, 1458, 1451, 1456, 1455, 1454, 1458, 1460, 1460, + 1462, 1463, 1464, 1464, 1465, 1466, 1457, 1467, 1465, 1468, + 1469, 1471, 1470, 1473, 1466, 1459, 1463, 1470, 1469, 1462, + 1472, 1472, 1475, 1467, 1474, 1474, 1478, 1475, 1468, 1477, + 1477, 1479, 1473, 1480, 1480, 1481, 1481, 1471, 1482, 1483, + + 1484, 1485, 1485, 1486, 1487, 1488, 1478, 1489, 1490, 1487, + 1494, 1483, 1489, 1479, 0, 1486, 1497, 1482, 1491, 1491, + 1484, 1493, 1493, 1496, 1488, 1490, 1490, 1498, 1496, 1494, + 1497, 0, 1498, 1499, 1499, 1500, 1500, 1501, 1501, 1502, + 1504, 1504, 1506, 1506, 1502, 1508, 1508, 1509, 1512, 1513, + 1513, 1517, 1509, 1514, 1516, 1518, 1512, 1519, 1514, 1516, + 1520, 1521, 1524, 1525, 1525, 1536, 1521, 1526, 1526, 1517, + 1538, 1518, 1527, 1540, 1519, 1519, 1527, 1531, 1531, 1533, + 1536, 1520, 1524, 1533, 1535, 1535, 1539, 1539, 1538, 1541, + 1542, 1543, 1544, 1545, 1546, 1546, 1543, 1544, 1540, 1549, + + 1549, 1551, 1551, 1553, 1541, 1554, 1554, 1557, 1556, 1558, + 1559, 1542, 1545, 1556, 1558, 1553, 1560, 1561, 1565, 1566, + 1569, 1557, 1565, 1567, 1568, 1570, 1571, 1572, 1574, 1559, + 1573, 1573, 1577, 1561, 1560, 1575, 1567, 1574, 1569, 1566, + 1575, 1570, 1572, 1568, 1576, 1578, 1582, 1579, 1580, 1576, + 1583, 1571, 1579, 1584, 1585, 1587, 1586, 1577, 1584, 1585, + 1578, 1582, 1580, 1588, 1589, 1590, 1591, 1592, 1587, 1583, + 1586, 1593, 1594, 1596, 1597, 1597, 1595, 1594, 1589, 1591, + 1592, 1595, 1588, 1598, 1590, 1601, 1599, 0, 1598, 1600, + 1593, 1599, 1596, 1602, 1600, 1603, 1603, 1605, 1602, 1604, + + 1606, 1608, 1607, 1611, 1601, 1609, 1604, 1607, 1611, 1612, + 1613, 1614, 1615, 1615, 1617, 0, 1605, 1616, 1609, 1606, + 1608, 1622, 1616, 1626, 1614, 1618, 1618, 1620, 1612, 1613, + 1619, 1619, 1620, 1617, 1621, 1623, 1623, 1624, 1625, 1628, + 1622, 1621, 1626, 1627, 1624, 1625, 1630, 1631, 1627, 1632, + 1633, 1630, 1631, 1634, 1634, 1635, 1636, 1639, 1628, 1637, + 1637, 1638, 1638, 1648, 1639, 1640, 1640, 1649, 1632, 1633, + 1641, 1641, 1642, 1644, 1635, 1636, 1647, 0, 1644, 1642, + 0, 1647, 1648, 1650, 1650, 0, 1649, 1651, 1651, 1655, + 1655, 1655, 1655, 1655, 1655, 1655, 1656, 1656, 1656, 1656, + + 1656, 1656, 1656, 1657, 1657, 1657, 1657, 1657, 1657, 1657, + 1658, 1658, 1658, 1658, 1658, 1658, 1658, 1659, 1659, 1659, + 1659, 1659, 1659, 1659, 1661, 1661, 0, 1661, 1661, 1661, + 1661, 1662, 1662, 0, 0, 0, 1662, 1662, 1663, 1663, + 0, 0, 1663, 0, 1663, 1664, 0, 0, 0, 0, + 0, 1664, 1665, 1665, 0, 0, 0, 1665, 1665, 1666, + 0, 0, 0, 0, 0, 1666, 1667, 1667, 0, 1667, + 1667, 1667, 1667, 1668, 1668, 0, 1668, 1668, 1668, 1668, + 1654, 1654, 1654, 1654, 1654, 1654, 1654, 1654, 1654, 1654, + 1654, 1654, 1654, 1654, 1654, 1654, 1654, 1654, 1654, 1654, + + 1654, 1654, 1654, 1654, 1654, 1654, 1654, 1654, 1654, 1654, + 1654, 1654, 1654, 1654, 1654, 1654, 1654, 1654, 1654, 1654 } ; static yy_state_type yy_last_accepting_state; @@ -1877,7 +1908,7 @@ static void config_end_include(void) #define YY_NO_INPUT 1 #endif -#line 1879 "<stdout>" +#line 1910 "<stdout>" #define INITIAL 0 #define quotedstring 1 @@ -2064,7 +2095,7 @@ YY_DECL #line 197 "./util/configlexer.lex" -#line 2066 "<stdout>" +#line 2097 "<stdout>" if ( !(yy_init) ) { @@ -2123,13 +2154,13 @@ yy_match: while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 1611 ) + if ( yy_current_state >= 1655 ) yy_c = yy_meta[(unsigned int) yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; ++yy_cp; } - while ( yy_base[yy_current_state] != 3201 ); + while ( yy_base[yy_current_state] != 3281 ); yy_find_action: yy_act = yy_accept[yy_current_state]; @@ -2299,610 +2330,630 @@ YY_RULE_SETUP case 29: YY_RULE_SETUP #line 229 "./util/configlexer.lex" -{ YDVAR(1, VAR_CHROOT) } +{ YDVAR(1, VAR_IP_TRANSPARENT) } YY_BREAK case 30: YY_RULE_SETUP #line 230 "./util/configlexer.lex" -{ YDVAR(1, VAR_USERNAME) } +{ YDVAR(1, VAR_CHROOT) } YY_BREAK case 31: YY_RULE_SETUP #line 231 "./util/configlexer.lex" -{ YDVAR(1, VAR_DIRECTORY) } +{ YDVAR(1, VAR_USERNAME) } YY_BREAK case 32: YY_RULE_SETUP #line 232 "./util/configlexer.lex" -{ YDVAR(1, VAR_LOGFILE) } +{ YDVAR(1, VAR_DIRECTORY) } YY_BREAK case 33: YY_RULE_SETUP #line 233 "./util/configlexer.lex" -{ YDVAR(1, VAR_PIDFILE) } +{ YDVAR(1, VAR_LOGFILE) } YY_BREAK case 34: YY_RULE_SETUP #line 234 "./util/configlexer.lex" -{ YDVAR(1, VAR_ROOT_HINTS) } +{ YDVAR(1, VAR_PIDFILE) } YY_BREAK case 35: YY_RULE_SETUP #line 235 "./util/configlexer.lex" -{ YDVAR(1, VAR_EDNS_BUFFER_SIZE) } +{ YDVAR(1, VAR_ROOT_HINTS) } YY_BREAK case 36: YY_RULE_SETUP #line 236 "./util/configlexer.lex" -{ YDVAR(1, VAR_MSG_BUFFER_SIZE) } +{ YDVAR(1, VAR_EDNS_BUFFER_SIZE) } YY_BREAK case 37: YY_RULE_SETUP #line 237 "./util/configlexer.lex" -{ YDVAR(1, VAR_MSG_CACHE_SIZE) } +{ YDVAR(1, VAR_MSG_BUFFER_SIZE) } YY_BREAK case 38: YY_RULE_SETUP #line 238 "./util/configlexer.lex" -{ YDVAR(1, VAR_MSG_CACHE_SLABS) } +{ YDVAR(1, VAR_MSG_CACHE_SIZE) } YY_BREAK case 39: YY_RULE_SETUP #line 239 "./util/configlexer.lex" -{ YDVAR(1, VAR_RRSET_CACHE_SIZE) } +{ YDVAR(1, VAR_MSG_CACHE_SLABS) } YY_BREAK case 40: YY_RULE_SETUP #line 240 "./util/configlexer.lex" -{ YDVAR(1, VAR_RRSET_CACHE_SLABS) } +{ YDVAR(1, VAR_RRSET_CACHE_SIZE) } YY_BREAK case 41: YY_RULE_SETUP #line 241 "./util/configlexer.lex" -{ YDVAR(1, VAR_CACHE_MAX_TTL) } +{ YDVAR(1, VAR_RRSET_CACHE_SLABS) } YY_BREAK case 42: YY_RULE_SETUP #line 242 "./util/configlexer.lex" -{ YDVAR(1, VAR_CACHE_MIN_TTL) } +{ YDVAR(1, VAR_CACHE_MAX_TTL) } YY_BREAK case 43: YY_RULE_SETUP #line 243 "./util/configlexer.lex" -{ YDVAR(1, VAR_INFRA_HOST_TTL) } +{ YDVAR(1, VAR_CACHE_MIN_TTL) } YY_BREAK case 44: YY_RULE_SETUP #line 244 "./util/configlexer.lex" -{ YDVAR(1, VAR_INFRA_LAME_TTL) } +{ YDVAR(1, VAR_INFRA_HOST_TTL) } YY_BREAK case 45: YY_RULE_SETUP #line 245 "./util/configlexer.lex" -{ YDVAR(1, VAR_INFRA_CACHE_SLABS) } +{ YDVAR(1, VAR_INFRA_LAME_TTL) } YY_BREAK case 46: YY_RULE_SETUP #line 246 "./util/configlexer.lex" -{ YDVAR(1, VAR_INFRA_CACHE_NUMHOSTS) } +{ YDVAR(1, VAR_INFRA_CACHE_SLABS) } YY_BREAK case 47: YY_RULE_SETUP #line 247 "./util/configlexer.lex" -{ YDVAR(1, VAR_INFRA_CACHE_LAME_SIZE) } +{ YDVAR(1, VAR_INFRA_CACHE_NUMHOSTS) } YY_BREAK case 48: YY_RULE_SETUP #line 248 "./util/configlexer.lex" -{ YDVAR(1, VAR_NUM_QUERIES_PER_THREAD) } +{ YDVAR(1, VAR_INFRA_CACHE_LAME_SIZE) } YY_BREAK case 49: YY_RULE_SETUP #line 249 "./util/configlexer.lex" -{ YDVAR(1, VAR_JOSTLE_TIMEOUT) } +{ YDVAR(1, VAR_INFRA_CACHE_MIN_RTT) } YY_BREAK case 50: YY_RULE_SETUP #line 250 "./util/configlexer.lex" -{ YDVAR(1, VAR_DELAY_CLOSE) } +{ YDVAR(1, VAR_NUM_QUERIES_PER_THREAD) } YY_BREAK case 51: YY_RULE_SETUP #line 251 "./util/configlexer.lex" -{ YDVAR(1, VAR_TARGET_FETCH_POLICY) } +{ YDVAR(1, VAR_JOSTLE_TIMEOUT) } YY_BREAK case 52: YY_RULE_SETUP #line 252 "./util/configlexer.lex" -{ YDVAR(1, VAR_HARDEN_SHORT_BUFSIZE) } +{ YDVAR(1, VAR_DELAY_CLOSE) } YY_BREAK case 53: YY_RULE_SETUP #line 253 "./util/configlexer.lex" -{ YDVAR(1, VAR_HARDEN_LARGE_QUERIES) } +{ YDVAR(1, VAR_TARGET_FETCH_POLICY) } YY_BREAK case 54: YY_RULE_SETUP #line 254 "./util/configlexer.lex" -{ YDVAR(1, VAR_HARDEN_GLUE) } +{ YDVAR(1, VAR_HARDEN_SHORT_BUFSIZE) } YY_BREAK case 55: YY_RULE_SETUP #line 255 "./util/configlexer.lex" -{ YDVAR(1, VAR_HARDEN_DNSSEC_STRIPPED) } +{ YDVAR(1, VAR_HARDEN_LARGE_QUERIES) } YY_BREAK case 56: YY_RULE_SETUP #line 256 "./util/configlexer.lex" -{ YDVAR(1, VAR_HARDEN_BELOW_NXDOMAIN) } +{ YDVAR(1, VAR_HARDEN_GLUE) } YY_BREAK case 57: YY_RULE_SETUP #line 257 "./util/configlexer.lex" -{ YDVAR(1, VAR_HARDEN_REFERRAL_PATH) } +{ YDVAR(1, VAR_HARDEN_DNSSEC_STRIPPED) } YY_BREAK case 58: YY_RULE_SETUP #line 258 "./util/configlexer.lex" -{ YDVAR(1, VAR_USE_CAPS_FOR_ID) } +{ YDVAR(1, VAR_HARDEN_BELOW_NXDOMAIN) } YY_BREAK case 59: YY_RULE_SETUP #line 259 "./util/configlexer.lex" -{ YDVAR(1, VAR_UNWANTED_REPLY_THRESHOLD) } +{ YDVAR(1, VAR_HARDEN_REFERRAL_PATH) } YY_BREAK case 60: YY_RULE_SETUP #line 260 "./util/configlexer.lex" -{ YDVAR(1, VAR_PRIVATE_ADDRESS) } +{ YDVAR(1, VAR_HARDEN_ALGO_DOWNGRADE) } YY_BREAK case 61: YY_RULE_SETUP #line 261 "./util/configlexer.lex" -{ YDVAR(1, VAR_PRIVATE_DOMAIN) } +{ YDVAR(1, VAR_USE_CAPS_FOR_ID) } YY_BREAK case 62: YY_RULE_SETUP #line 262 "./util/configlexer.lex" -{ YDVAR(1, VAR_PREFETCH_KEY) } +{ YDVAR(1, VAR_UNWANTED_REPLY_THRESHOLD) } YY_BREAK case 63: YY_RULE_SETUP #line 263 "./util/configlexer.lex" -{ YDVAR(1, VAR_PREFETCH) } +{ YDVAR(1, VAR_PRIVATE_ADDRESS) } YY_BREAK case 64: YY_RULE_SETUP #line 264 "./util/configlexer.lex" -{ YDVAR(0, VAR_STUB_ZONE) } +{ YDVAR(1, VAR_PRIVATE_DOMAIN) } YY_BREAK case 65: YY_RULE_SETUP #line 265 "./util/configlexer.lex" -{ YDVAR(1, VAR_NAME) } +{ YDVAR(1, VAR_PREFETCH_KEY) } YY_BREAK case 66: YY_RULE_SETUP #line 266 "./util/configlexer.lex" -{ YDVAR(1, VAR_STUB_ADDR) } +{ YDVAR(1, VAR_PREFETCH) } YY_BREAK case 67: YY_RULE_SETUP #line 267 "./util/configlexer.lex" -{ YDVAR(1, VAR_STUB_HOST) } +{ YDVAR(0, VAR_STUB_ZONE) } YY_BREAK case 68: YY_RULE_SETUP #line 268 "./util/configlexer.lex" -{ YDVAR(1, VAR_STUB_PRIME) } +{ YDVAR(1, VAR_NAME) } YY_BREAK case 69: YY_RULE_SETUP #line 269 "./util/configlexer.lex" -{ YDVAR(1, VAR_STUB_FIRST) } +{ YDVAR(1, VAR_STUB_ADDR) } YY_BREAK case 70: YY_RULE_SETUP #line 270 "./util/configlexer.lex" -{ YDVAR(0, VAR_FORWARD_ZONE) } +{ YDVAR(1, VAR_STUB_HOST) } YY_BREAK case 71: YY_RULE_SETUP #line 271 "./util/configlexer.lex" -{ YDVAR(1, VAR_FORWARD_ADDR) } +{ YDVAR(1, VAR_STUB_PRIME) } YY_BREAK case 72: YY_RULE_SETUP #line 272 "./util/configlexer.lex" -{ YDVAR(1, VAR_FORWARD_HOST) } +{ YDVAR(1, VAR_STUB_FIRST) } YY_BREAK case 73: YY_RULE_SETUP #line 273 "./util/configlexer.lex" -{ YDVAR(1, VAR_FORWARD_FIRST) } +{ YDVAR(0, VAR_FORWARD_ZONE) } YY_BREAK case 74: YY_RULE_SETUP #line 274 "./util/configlexer.lex" -{ YDVAR(1, VAR_DO_NOT_QUERY_ADDRESS) } +{ YDVAR(1, VAR_FORWARD_ADDR) } YY_BREAK case 75: YY_RULE_SETUP #line 275 "./util/configlexer.lex" -{ YDVAR(1, VAR_DO_NOT_QUERY_LOCALHOST) } +{ YDVAR(1, VAR_FORWARD_HOST) } YY_BREAK case 76: YY_RULE_SETUP #line 276 "./util/configlexer.lex" -{ YDVAR(2, VAR_ACCESS_CONTROL) } +{ YDVAR(1, VAR_FORWARD_FIRST) } YY_BREAK case 77: YY_RULE_SETUP #line 277 "./util/configlexer.lex" -{ YDVAR(1, VAR_HIDE_IDENTITY) } +{ YDVAR(1, VAR_DO_NOT_QUERY_ADDRESS) } YY_BREAK case 78: YY_RULE_SETUP #line 278 "./util/configlexer.lex" -{ YDVAR(1, VAR_HIDE_VERSION) } +{ YDVAR(1, VAR_DO_NOT_QUERY_LOCALHOST) } YY_BREAK case 79: YY_RULE_SETUP #line 279 "./util/configlexer.lex" -{ YDVAR(1, VAR_IDENTITY) } +{ YDVAR(2, VAR_ACCESS_CONTROL) } YY_BREAK case 80: YY_RULE_SETUP #line 280 "./util/configlexer.lex" -{ YDVAR(1, VAR_VERSION) } +{ YDVAR(1, VAR_HIDE_IDENTITY) } YY_BREAK case 81: YY_RULE_SETUP #line 281 "./util/configlexer.lex" -{ YDVAR(1, VAR_MODULE_CONF) } +{ YDVAR(1, VAR_HIDE_VERSION) } YY_BREAK case 82: YY_RULE_SETUP #line 282 "./util/configlexer.lex" -{ YDVAR(1, VAR_DLV_ANCHOR) } +{ YDVAR(1, VAR_IDENTITY) } YY_BREAK case 83: YY_RULE_SETUP #line 283 "./util/configlexer.lex" -{ YDVAR(1, VAR_DLV_ANCHOR_FILE) } +{ YDVAR(1, VAR_VERSION) } YY_BREAK case 84: YY_RULE_SETUP #line 284 "./util/configlexer.lex" -{ YDVAR(1, VAR_TRUST_ANCHOR_FILE) } +{ YDVAR(1, VAR_MODULE_CONF) } YY_BREAK case 85: YY_RULE_SETUP #line 285 "./util/configlexer.lex" -{ YDVAR(1, VAR_AUTO_TRUST_ANCHOR_FILE) } +{ YDVAR(1, VAR_DLV_ANCHOR) } YY_BREAK case 86: YY_RULE_SETUP #line 286 "./util/configlexer.lex" -{ YDVAR(1, VAR_TRUSTED_KEYS_FILE) } +{ YDVAR(1, VAR_DLV_ANCHOR_FILE) } YY_BREAK case 87: YY_RULE_SETUP #line 287 "./util/configlexer.lex" -{ YDVAR(1, VAR_TRUST_ANCHOR) } +{ YDVAR(1, VAR_TRUST_ANCHOR_FILE) } YY_BREAK case 88: YY_RULE_SETUP #line 288 "./util/configlexer.lex" -{ YDVAR(1, VAR_VAL_OVERRIDE_DATE) } +{ YDVAR(1, VAR_AUTO_TRUST_ANCHOR_FILE) } YY_BREAK case 89: YY_RULE_SETUP #line 289 "./util/configlexer.lex" -{ YDVAR(1, VAR_VAL_SIG_SKEW_MIN) } +{ YDVAR(1, VAR_TRUSTED_KEYS_FILE) } YY_BREAK case 90: YY_RULE_SETUP #line 290 "./util/configlexer.lex" -{ YDVAR(1, VAR_VAL_SIG_SKEW_MAX) } +{ YDVAR(1, VAR_TRUST_ANCHOR) } YY_BREAK case 91: YY_RULE_SETUP #line 291 "./util/configlexer.lex" -{ YDVAR(1, VAR_BOGUS_TTL) } +{ YDVAR(1, VAR_VAL_OVERRIDE_DATE) } YY_BREAK case 92: YY_RULE_SETUP #line 292 "./util/configlexer.lex" -{ YDVAR(1, VAR_VAL_CLEAN_ADDITIONAL) } +{ YDVAR(1, VAR_VAL_SIG_SKEW_MIN) } YY_BREAK case 93: YY_RULE_SETUP #line 293 "./util/configlexer.lex" -{ YDVAR(1, VAR_VAL_PERMISSIVE_MODE) } +{ YDVAR(1, VAR_VAL_SIG_SKEW_MAX) } YY_BREAK case 94: YY_RULE_SETUP #line 294 "./util/configlexer.lex" -{ YDVAR(1, VAR_IGNORE_CD_FLAG) } +{ YDVAR(1, VAR_BOGUS_TTL) } YY_BREAK case 95: YY_RULE_SETUP #line 295 "./util/configlexer.lex" -{ YDVAR(1, VAR_VAL_LOG_LEVEL) } +{ YDVAR(1, VAR_VAL_CLEAN_ADDITIONAL) } YY_BREAK case 96: YY_RULE_SETUP #line 296 "./util/configlexer.lex" -{ YDVAR(1, VAR_KEY_CACHE_SIZE) } +{ YDVAR(1, VAR_VAL_PERMISSIVE_MODE) } YY_BREAK case 97: YY_RULE_SETUP #line 297 "./util/configlexer.lex" -{ YDVAR(1, VAR_KEY_CACHE_SLABS) } +{ YDVAR(1, VAR_IGNORE_CD_FLAG) } YY_BREAK case 98: YY_RULE_SETUP #line 298 "./util/configlexer.lex" -{ YDVAR(1, VAR_NEG_CACHE_SIZE) } +{ YDVAR(1, VAR_VAL_LOG_LEVEL) } YY_BREAK case 99: YY_RULE_SETUP #line 299 "./util/configlexer.lex" -{ - YDVAR(1, VAR_VAL_NSEC3_KEYSIZE_ITERATIONS) } +{ YDVAR(1, VAR_KEY_CACHE_SIZE) } YY_BREAK case 100: YY_RULE_SETUP -#line 301 "./util/configlexer.lex" -{ YDVAR(1, VAR_ADD_HOLDDOWN) } +#line 300 "./util/configlexer.lex" +{ YDVAR(1, VAR_KEY_CACHE_SLABS) } YY_BREAK case 101: YY_RULE_SETUP -#line 302 "./util/configlexer.lex" -{ YDVAR(1, VAR_DEL_HOLDDOWN) } +#line 301 "./util/configlexer.lex" +{ YDVAR(1, VAR_NEG_CACHE_SIZE) } YY_BREAK case 102: YY_RULE_SETUP -#line 303 "./util/configlexer.lex" -{ YDVAR(1, VAR_KEEP_MISSING) } +#line 302 "./util/configlexer.lex" +{ + YDVAR(1, VAR_VAL_NSEC3_KEYSIZE_ITERATIONS) } YY_BREAK case 103: YY_RULE_SETUP #line 304 "./util/configlexer.lex" -{ YDVAR(1, VAR_USE_SYSLOG) } +{ YDVAR(1, VAR_ADD_HOLDDOWN) } YY_BREAK case 104: YY_RULE_SETUP #line 305 "./util/configlexer.lex" -{ YDVAR(1, VAR_LOG_TIME_ASCII) } +{ YDVAR(1, VAR_DEL_HOLDDOWN) } YY_BREAK case 105: YY_RULE_SETUP #line 306 "./util/configlexer.lex" -{ YDVAR(1, VAR_LOG_QUERIES) } +{ YDVAR(1, VAR_KEEP_MISSING) } YY_BREAK case 106: YY_RULE_SETUP #line 307 "./util/configlexer.lex" -{ YDVAR(2, VAR_LOCAL_ZONE) } +{ YDVAR(1, VAR_USE_SYSLOG) } YY_BREAK case 107: YY_RULE_SETUP #line 308 "./util/configlexer.lex" -{ YDVAR(1, VAR_LOCAL_DATA) } +{ YDVAR(1, VAR_LOG_TIME_ASCII) } YY_BREAK case 108: YY_RULE_SETUP #line 309 "./util/configlexer.lex" -{ YDVAR(1, VAR_LOCAL_DATA_PTR) } +{ YDVAR(1, VAR_LOG_QUERIES) } YY_BREAK case 109: YY_RULE_SETUP #line 310 "./util/configlexer.lex" -{ YDVAR(1, VAR_UNBLOCK_LAN_ZONES) } +{ YDVAR(2, VAR_LOCAL_ZONE) } YY_BREAK case 110: YY_RULE_SETUP #line 311 "./util/configlexer.lex" -{ YDVAR(1, VAR_STATISTICS_INTERVAL) } +{ YDVAR(1, VAR_LOCAL_DATA) } YY_BREAK case 111: YY_RULE_SETUP #line 312 "./util/configlexer.lex" -{ YDVAR(1, VAR_STATISTICS_CUMULATIVE) } +{ YDVAR(1, VAR_LOCAL_DATA_PTR) } YY_BREAK case 112: YY_RULE_SETUP #line 313 "./util/configlexer.lex" -{ YDVAR(1, VAR_EXTENDED_STATISTICS) } +{ YDVAR(1, VAR_UNBLOCK_LAN_ZONES) } YY_BREAK case 113: YY_RULE_SETUP #line 314 "./util/configlexer.lex" -{ YDVAR(0, VAR_REMOTE_CONTROL) } +{ YDVAR(1, VAR_STATISTICS_INTERVAL) } YY_BREAK case 114: YY_RULE_SETUP #line 315 "./util/configlexer.lex" -{ YDVAR(1, VAR_CONTROL_ENABLE) } +{ YDVAR(1, VAR_STATISTICS_CUMULATIVE) } YY_BREAK case 115: YY_RULE_SETUP #line 316 "./util/configlexer.lex" -{ YDVAR(1, VAR_CONTROL_INTERFACE) } +{ YDVAR(1, VAR_EXTENDED_STATISTICS) } YY_BREAK case 116: YY_RULE_SETUP #line 317 "./util/configlexer.lex" -{ YDVAR(1, VAR_CONTROL_PORT) } +{ YDVAR(0, VAR_REMOTE_CONTROL) } YY_BREAK case 117: YY_RULE_SETUP #line 318 "./util/configlexer.lex" -{ YDVAR(1, VAR_SERVER_KEY_FILE) } +{ YDVAR(1, VAR_CONTROL_ENABLE) } YY_BREAK case 118: YY_RULE_SETUP #line 319 "./util/configlexer.lex" -{ YDVAR(1, VAR_SERVER_CERT_FILE) } +{ YDVAR(1, VAR_CONTROL_INTERFACE) } YY_BREAK case 119: YY_RULE_SETUP #line 320 "./util/configlexer.lex" -{ YDVAR(1, VAR_CONTROL_KEY_FILE) } +{ YDVAR(1, VAR_CONTROL_PORT) } YY_BREAK case 120: YY_RULE_SETUP #line 321 "./util/configlexer.lex" -{ YDVAR(1, VAR_CONTROL_CERT_FILE) } +{ YDVAR(1, VAR_CONTROL_USE_CERT) } YY_BREAK case 121: YY_RULE_SETUP #line 322 "./util/configlexer.lex" -{ YDVAR(1, VAR_PYTHON_SCRIPT) } +{ YDVAR(1, VAR_SERVER_KEY_FILE) } YY_BREAK case 122: YY_RULE_SETUP #line 323 "./util/configlexer.lex" -{ YDVAR(0, VAR_PYTHON) } +{ YDVAR(1, VAR_SERVER_CERT_FILE) } YY_BREAK case 123: YY_RULE_SETUP #line 324 "./util/configlexer.lex" -{ YDVAR(1, VAR_DOMAIN_INSECURE) } +{ YDVAR(1, VAR_CONTROL_KEY_FILE) } YY_BREAK case 124: YY_RULE_SETUP #line 325 "./util/configlexer.lex" -{ YDVAR(1, VAR_MINIMAL_RESPONSES) } +{ YDVAR(1, VAR_CONTROL_CERT_FILE) } YY_BREAK case 125: YY_RULE_SETUP #line 326 "./util/configlexer.lex" -{ YDVAR(1, VAR_RRSET_ROUNDROBIN) } +{ YDVAR(1, VAR_PYTHON_SCRIPT) } YY_BREAK case 126: YY_RULE_SETUP #line 327 "./util/configlexer.lex" -{ YDVAR(1, VAR_MAX_UDP_SIZE) } +{ YDVAR(0, VAR_PYTHON) } YY_BREAK case 127: YY_RULE_SETUP #line 328 "./util/configlexer.lex" -{ YDVAR(1, VAR_DNS64_PREFIX) } +{ YDVAR(1, VAR_DOMAIN_INSECURE) } YY_BREAK case 128: YY_RULE_SETUP #line 329 "./util/configlexer.lex" -{ YDVAR(1, VAR_DNS64_SYNTHALL) } +{ YDVAR(1, VAR_MINIMAL_RESPONSES) } YY_BREAK case 129: YY_RULE_SETUP #line 330 "./util/configlexer.lex" -{ YDVAR(0, VAR_DNSTAP) } +{ YDVAR(1, VAR_RRSET_ROUNDROBIN) } YY_BREAK case 130: YY_RULE_SETUP #line 331 "./util/configlexer.lex" -{ YDVAR(1, VAR_DNSTAP_ENABLE) } +{ YDVAR(1, VAR_MAX_UDP_SIZE) } YY_BREAK case 131: YY_RULE_SETUP #line 332 "./util/configlexer.lex" -{ YDVAR(1, VAR_DNSTAP_SOCKET_PATH) } +{ YDVAR(1, VAR_DNS64_PREFIX) } YY_BREAK case 132: YY_RULE_SETUP #line 333 "./util/configlexer.lex" -{ YDVAR(1, VAR_DNSTAP_SEND_IDENTITY) } +{ YDVAR(1, VAR_DNS64_SYNTHALL) } YY_BREAK case 133: YY_RULE_SETUP #line 334 "./util/configlexer.lex" -{ YDVAR(1, VAR_DNSTAP_SEND_VERSION) } +{ YDVAR(0, VAR_DNSTAP) } YY_BREAK case 134: YY_RULE_SETUP #line 335 "./util/configlexer.lex" -{ YDVAR(1, VAR_DNSTAP_IDENTITY) } +{ YDVAR(1, VAR_DNSTAP_ENABLE) } YY_BREAK case 135: YY_RULE_SETUP #line 336 "./util/configlexer.lex" -{ YDVAR(1, VAR_DNSTAP_VERSION) } +{ YDVAR(1, VAR_DNSTAP_SOCKET_PATH) } YY_BREAK case 136: YY_RULE_SETUP #line 337 "./util/configlexer.lex" -{ - YDVAR(1, VAR_DNSTAP_LOG_RESOLVER_QUERY_MESSAGES) } +{ YDVAR(1, VAR_DNSTAP_SEND_IDENTITY) } YY_BREAK case 137: YY_RULE_SETUP +#line 338 "./util/configlexer.lex" +{ YDVAR(1, VAR_DNSTAP_SEND_VERSION) } + YY_BREAK +case 138: +YY_RULE_SETUP #line 339 "./util/configlexer.lex" +{ YDVAR(1, VAR_DNSTAP_IDENTITY) } + YY_BREAK +case 139: +YY_RULE_SETUP +#line 340 "./util/configlexer.lex" +{ YDVAR(1, VAR_DNSTAP_VERSION) } + YY_BREAK +case 140: +YY_RULE_SETUP +#line 341 "./util/configlexer.lex" +{ + YDVAR(1, VAR_DNSTAP_LOG_RESOLVER_QUERY_MESSAGES) } + YY_BREAK +case 141: +YY_RULE_SETUP +#line 343 "./util/configlexer.lex" { YDVAR(1, VAR_DNSTAP_LOG_RESOLVER_RESPONSE_MESSAGES) } YY_BREAK -case 138: +case 142: YY_RULE_SETUP -#line 341 "./util/configlexer.lex" +#line 345 "./util/configlexer.lex" { YDVAR(1, VAR_DNSTAP_LOG_CLIENT_QUERY_MESSAGES) } YY_BREAK -case 139: +case 143: YY_RULE_SETUP -#line 343 "./util/configlexer.lex" +#line 347 "./util/configlexer.lex" { YDVAR(1, VAR_DNSTAP_LOG_CLIENT_RESPONSE_MESSAGES) } YY_BREAK -case 140: +case 144: YY_RULE_SETUP -#line 345 "./util/configlexer.lex" +#line 349 "./util/configlexer.lex" { YDVAR(1, VAR_DNSTAP_LOG_FORWARDER_QUERY_MESSAGES) } YY_BREAK -case 141: +case 145: YY_RULE_SETUP -#line 347 "./util/configlexer.lex" +#line 351 "./util/configlexer.lex" { YDVAR(1, VAR_DNSTAP_LOG_FORWARDER_RESPONSE_MESSAGES) } YY_BREAK -case 142: -/* rule 142 can match eol */ +case 146: +/* rule 146 can match eol */ YY_RULE_SETUP -#line 349 "./util/configlexer.lex" +#line 353 "./util/configlexer.lex" { LEXOUT(("NL\n")); cfg_parser->line++; } YY_BREAK /* Quoted strings. Strip leading and ending quotes */ -case 143: +case 147: YY_RULE_SETUP -#line 352 "./util/configlexer.lex" +#line 356 "./util/configlexer.lex" { BEGIN(quotedstring); LEXOUT(("QS ")); } YY_BREAK case YY_STATE_EOF(quotedstring): -#line 353 "./util/configlexer.lex" +#line 357 "./util/configlexer.lex" { yyerror("EOF inside quoted string"); if(--num_args == 0) { BEGIN(INITIAL); } else { BEGIN(val); } } YY_BREAK -case 144: +case 148: YY_RULE_SETUP -#line 358 "./util/configlexer.lex" +#line 362 "./util/configlexer.lex" { LEXOUT(("STR(%s) ", yytext)); yymore(); } YY_BREAK -case 145: -/* rule 145 can match eol */ +case 149: +/* rule 149 can match eol */ YY_RULE_SETUP -#line 359 "./util/configlexer.lex" +#line 363 "./util/configlexer.lex" { yyerror("newline inside quoted string, no end \""); cfg_parser->line++; BEGIN(INITIAL); } YY_BREAK -case 146: +case 150: YY_RULE_SETUP -#line 361 "./util/configlexer.lex" +#line 365 "./util/configlexer.lex" { LEXOUT(("QE ")); if(--num_args == 0) { BEGIN(INITIAL); } @@ -2915,34 +2966,34 @@ YY_RULE_SETUP } YY_BREAK /* Single Quoted strings. Strip leading and ending quotes */ -case 147: +case 151: YY_RULE_SETUP -#line 373 "./util/configlexer.lex" +#line 377 "./util/configlexer.lex" { BEGIN(singlequotedstr); LEXOUT(("SQS ")); } YY_BREAK case YY_STATE_EOF(singlequotedstr): -#line 374 "./util/configlexer.lex" +#line 378 "./util/configlexer.lex" { yyerror("EOF inside quoted string"); if(--num_args == 0) { BEGIN(INITIAL); } else { BEGIN(val); } } YY_BREAK -case 148: +case 152: YY_RULE_SETUP -#line 379 "./util/configlexer.lex" +#line 383 "./util/configlexer.lex" { LEXOUT(("STR(%s) ", yytext)); yymore(); } YY_BREAK -case 149: -/* rule 149 can match eol */ +case 153: +/* rule 153 can match eol */ YY_RULE_SETUP -#line 380 "./util/configlexer.lex" +#line 384 "./util/configlexer.lex" { yyerror("newline inside quoted string, no end '"); cfg_parser->line++; BEGIN(INITIAL); } YY_BREAK -case 150: +case 154: YY_RULE_SETUP -#line 382 "./util/configlexer.lex" +#line 386 "./util/configlexer.lex" { LEXOUT(("SQE ")); if(--num_args == 0) { BEGIN(INITIAL); } @@ -2955,38 +3006,38 @@ YY_RULE_SETUP } YY_BREAK /* include: directive */ -case 151: +case 155: YY_RULE_SETUP -#line 394 "./util/configlexer.lex" +#line 398 "./util/configlexer.lex" { LEXOUT(("v(%s) ", yytext)); inc_prev = YYSTATE; BEGIN(include); } YY_BREAK case YY_STATE_EOF(include): -#line 396 "./util/configlexer.lex" +#line 400 "./util/configlexer.lex" { yyerror("EOF inside include directive"); BEGIN(inc_prev); } YY_BREAK -case 152: +case 156: YY_RULE_SETUP -#line 400 "./util/configlexer.lex" +#line 404 "./util/configlexer.lex" { LEXOUT(("ISP ")); /* ignore */ } YY_BREAK -case 153: -/* rule 153 can match eol */ +case 157: +/* rule 157 can match eol */ YY_RULE_SETUP -#line 401 "./util/configlexer.lex" +#line 405 "./util/configlexer.lex" { LEXOUT(("NL\n")); cfg_parser->line++;} YY_BREAK -case 154: +case 158: YY_RULE_SETUP -#line 402 "./util/configlexer.lex" +#line 406 "./util/configlexer.lex" { LEXOUT(("IQS ")); BEGIN(include_quoted); } YY_BREAK -case 155: +case 159: YY_RULE_SETUP -#line 403 "./util/configlexer.lex" +#line 407 "./util/configlexer.lex" { LEXOUT(("Iunquotedstr(%s) ", yytext)); config_start_include_glob(yytext); @@ -2994,27 +3045,27 @@ YY_RULE_SETUP } YY_BREAK case YY_STATE_EOF(include_quoted): -#line 408 "./util/configlexer.lex" +#line 412 "./util/configlexer.lex" { yyerror("EOF inside quoted string"); BEGIN(inc_prev); } YY_BREAK -case 156: +case 160: YY_RULE_SETUP -#line 412 "./util/configlexer.lex" +#line 416 "./util/configlexer.lex" { LEXOUT(("ISTR(%s) ", yytext)); yymore(); } YY_BREAK -case 157: -/* rule 157 can match eol */ +case 161: +/* rule 161 can match eol */ YY_RULE_SETUP -#line 413 "./util/configlexer.lex" +#line 417 "./util/configlexer.lex" { yyerror("newline before \" in include name"); cfg_parser->line++; BEGIN(inc_prev); } YY_BREAK -case 158: +case 162: YY_RULE_SETUP -#line 415 "./util/configlexer.lex" +#line 419 "./util/configlexer.lex" { LEXOUT(("IQE ")); yytext[yyleng - 1] = '\0'; @@ -3024,7 +3075,7 @@ YY_RULE_SETUP YY_BREAK case YY_STATE_EOF(INITIAL): case YY_STATE_EOF(val): -#line 421 "./util/configlexer.lex" +#line 425 "./util/configlexer.lex" { LEXOUT(("LEXEOF ")); yy_set_bol(1); /* Set beginning of line, so "^" rules match. */ @@ -3036,33 +3087,33 @@ case YY_STATE_EOF(val): } } YY_BREAK -case 159: +case 163: YY_RULE_SETUP -#line 432 "./util/configlexer.lex" +#line 436 "./util/configlexer.lex" { LEXOUT(("unquotedstr(%s) ", yytext)); if(--num_args == 0) { BEGIN(INITIAL); } yylval.str = strdup(yytext); return STRING_ARG; } YY_BREAK -case 160: +case 164: YY_RULE_SETUP -#line 436 "./util/configlexer.lex" +#line 440 "./util/configlexer.lex" { ub_c_error_msg("unknown keyword '%s'", yytext); } YY_BREAK -case 161: +case 165: YY_RULE_SETUP -#line 440 "./util/configlexer.lex" +#line 444 "./util/configlexer.lex" { ub_c_error_msg("stray '%s'", yytext); } YY_BREAK -case 162: +case 166: YY_RULE_SETUP -#line 444 "./util/configlexer.lex" +#line 448 "./util/configlexer.lex" ECHO; YY_BREAK -#line 3064 "<stdout>" +#line 3115 "<stdout>" case YY_END_OF_BUFFER: { @@ -3352,7 +3403,7 @@ static int yy_get_next_buffer (void) while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 1611 ) + if ( yy_current_state >= 1655 ) yy_c = yy_meta[(unsigned int) yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; @@ -3380,11 +3431,11 @@ static int yy_get_next_buffer (void) while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 1611 ) + if ( yy_current_state >= 1655 ) yy_c = yy_meta[(unsigned int) yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; - yy_is_jam = (yy_current_state == 1610); + yy_is_jam = (yy_current_state == 1654); return yy_is_jam ? 0 : yy_current_state; } @@ -4017,7 +4068,7 @@ void yyfree (void * ptr ) #define YYTABLES_NAME "yytables" -#line 444 "./util/configlexer.lex" +#line 448 "./util/configlexer.lex" diff --git a/external/unbound/util/configlexer.lex b/external/unbound/util/configlexer.lex index 7ee7b9bd9..5b1389dfb 100644 --- a/external/unbound/util/configlexer.lex +++ b/external/unbound/util/configlexer.lex @@ -226,6 +226,7 @@ interface-automatic{COLON} { YDVAR(1, VAR_INTERFACE_AUTOMATIC) } so-rcvbuf{COLON} { YDVAR(1, VAR_SO_RCVBUF) } so-sndbuf{COLON} { YDVAR(1, VAR_SO_SNDBUF) } so-reuseport{COLON} { YDVAR(1, VAR_SO_REUSEPORT) } +ip-transparent{COLON} { YDVAR(1, VAR_IP_TRANSPARENT) } chroot{COLON} { YDVAR(1, VAR_CHROOT) } username{COLON} { YDVAR(1, VAR_USERNAME) } directory{COLON} { YDVAR(1, VAR_DIRECTORY) } @@ -245,6 +246,7 @@ infra-lame-ttl{COLON} { YDVAR(1, VAR_INFRA_LAME_TTL) } infra-cache-slabs{COLON} { YDVAR(1, VAR_INFRA_CACHE_SLABS) } infra-cache-numhosts{COLON} { YDVAR(1, VAR_INFRA_CACHE_NUMHOSTS) } infra-cache-lame-size{COLON} { YDVAR(1, VAR_INFRA_CACHE_LAME_SIZE) } +infra-cache-min-rtt{COLON} { YDVAR(1, VAR_INFRA_CACHE_MIN_RTT) } num-queries-per-thread{COLON} { YDVAR(1, VAR_NUM_QUERIES_PER_THREAD) } jostle-timeout{COLON} { YDVAR(1, VAR_JOSTLE_TIMEOUT) } delay-close{COLON} { YDVAR(1, VAR_DELAY_CLOSE) } @@ -255,6 +257,7 @@ harden-glue{COLON} { YDVAR(1, VAR_HARDEN_GLUE) } harden-dnssec-stripped{COLON} { YDVAR(1, VAR_HARDEN_DNSSEC_STRIPPED) } harden-below-nxdomain{COLON} { YDVAR(1, VAR_HARDEN_BELOW_NXDOMAIN) } harden-referral-path{COLON} { YDVAR(1, VAR_HARDEN_REFERRAL_PATH) } +harden-algo-downgrade{COLON} { YDVAR(1, VAR_HARDEN_ALGO_DOWNGRADE) } use-caps-for-id{COLON} { YDVAR(1, VAR_USE_CAPS_FOR_ID) } unwanted-reply-threshold{COLON} { YDVAR(1, VAR_UNWANTED_REPLY_THRESHOLD) } private-address{COLON} { YDVAR(1, VAR_PRIVATE_ADDRESS) } @@ -315,6 +318,7 @@ remote-control{COLON} { YDVAR(0, VAR_REMOTE_CONTROL) } control-enable{COLON} { YDVAR(1, VAR_CONTROL_ENABLE) } control-interface{COLON} { YDVAR(1, VAR_CONTROL_INTERFACE) } control-port{COLON} { YDVAR(1, VAR_CONTROL_PORT) } +control-use-cert{COLON} { YDVAR(1, VAR_CONTROL_USE_CERT) } server-key-file{COLON} { YDVAR(1, VAR_SERVER_KEY_FILE) } server-cert-file{COLON} { YDVAR(1, VAR_SERVER_CERT_FILE) } control-key-file{COLON} { YDVAR(1, VAR_CONTROL_KEY_FILE) } diff --git a/external/unbound/util/configparser.c b/external/unbound/util/configparser.c index 6285e3f6f..745a9ba79 100644 --- a/external/unbound/util/configparser.c +++ b/external/unbound/util/configparser.c @@ -220,59 +220,63 @@ extern int yydebug; VAR_SERVER_CERT_FILE = 348, VAR_CONTROL_KEY_FILE = 349, VAR_CONTROL_CERT_FILE = 350, - VAR_EXTENDED_STATISTICS = 351, - VAR_LOCAL_DATA_PTR = 352, - VAR_JOSTLE_TIMEOUT = 353, - VAR_STUB_PRIME = 354, - VAR_UNWANTED_REPLY_THRESHOLD = 355, - VAR_LOG_TIME_ASCII = 356, - VAR_DOMAIN_INSECURE = 357, - VAR_PYTHON = 358, - VAR_PYTHON_SCRIPT = 359, - VAR_VAL_SIG_SKEW_MIN = 360, - VAR_VAL_SIG_SKEW_MAX = 361, - VAR_CACHE_MIN_TTL = 362, - VAR_VAL_LOG_LEVEL = 363, - VAR_AUTO_TRUST_ANCHOR_FILE = 364, - VAR_KEEP_MISSING = 365, - VAR_ADD_HOLDDOWN = 366, - VAR_DEL_HOLDDOWN = 367, - VAR_SO_RCVBUF = 368, - VAR_EDNS_BUFFER_SIZE = 369, - VAR_PREFETCH = 370, - VAR_PREFETCH_KEY = 371, - VAR_SO_SNDBUF = 372, - VAR_SO_REUSEPORT = 373, - VAR_HARDEN_BELOW_NXDOMAIN = 374, - VAR_IGNORE_CD_FLAG = 375, - VAR_LOG_QUERIES = 376, - VAR_TCP_UPSTREAM = 377, - VAR_SSL_UPSTREAM = 378, - VAR_SSL_SERVICE_KEY = 379, - VAR_SSL_SERVICE_PEM = 380, - VAR_SSL_PORT = 381, - VAR_FORWARD_FIRST = 382, - VAR_STUB_FIRST = 383, - VAR_MINIMAL_RESPONSES = 384, - VAR_RRSET_ROUNDROBIN = 385, - VAR_MAX_UDP_SIZE = 386, - VAR_DELAY_CLOSE = 387, - VAR_UNBLOCK_LAN_ZONES = 388, - VAR_DNS64_PREFIX = 389, - VAR_DNS64_SYNTHALL = 390, - VAR_DNSTAP = 391, - VAR_DNSTAP_ENABLE = 392, - VAR_DNSTAP_SOCKET_PATH = 393, - VAR_DNSTAP_SEND_IDENTITY = 394, - VAR_DNSTAP_SEND_VERSION = 395, - VAR_DNSTAP_IDENTITY = 396, - VAR_DNSTAP_VERSION = 397, - VAR_DNSTAP_LOG_RESOLVER_QUERY_MESSAGES = 398, - VAR_DNSTAP_LOG_RESOLVER_RESPONSE_MESSAGES = 399, - VAR_DNSTAP_LOG_CLIENT_QUERY_MESSAGES = 400, - VAR_DNSTAP_LOG_CLIENT_RESPONSE_MESSAGES = 401, - VAR_DNSTAP_LOG_FORWARDER_QUERY_MESSAGES = 402, - VAR_DNSTAP_LOG_FORWARDER_RESPONSE_MESSAGES = 403 + VAR_CONTROL_USE_CERT = 351, + VAR_EXTENDED_STATISTICS = 352, + VAR_LOCAL_DATA_PTR = 353, + VAR_JOSTLE_TIMEOUT = 354, + VAR_STUB_PRIME = 355, + VAR_UNWANTED_REPLY_THRESHOLD = 356, + VAR_LOG_TIME_ASCII = 357, + VAR_DOMAIN_INSECURE = 358, + VAR_PYTHON = 359, + VAR_PYTHON_SCRIPT = 360, + VAR_VAL_SIG_SKEW_MIN = 361, + VAR_VAL_SIG_SKEW_MAX = 362, + VAR_CACHE_MIN_TTL = 363, + VAR_VAL_LOG_LEVEL = 364, + VAR_AUTO_TRUST_ANCHOR_FILE = 365, + VAR_KEEP_MISSING = 366, + VAR_ADD_HOLDDOWN = 367, + VAR_DEL_HOLDDOWN = 368, + VAR_SO_RCVBUF = 369, + VAR_EDNS_BUFFER_SIZE = 370, + VAR_PREFETCH = 371, + VAR_PREFETCH_KEY = 372, + VAR_SO_SNDBUF = 373, + VAR_SO_REUSEPORT = 374, + VAR_HARDEN_BELOW_NXDOMAIN = 375, + VAR_IGNORE_CD_FLAG = 376, + VAR_LOG_QUERIES = 377, + VAR_TCP_UPSTREAM = 378, + VAR_SSL_UPSTREAM = 379, + VAR_SSL_SERVICE_KEY = 380, + VAR_SSL_SERVICE_PEM = 381, + VAR_SSL_PORT = 382, + VAR_FORWARD_FIRST = 383, + VAR_STUB_FIRST = 384, + VAR_MINIMAL_RESPONSES = 385, + VAR_RRSET_ROUNDROBIN = 386, + VAR_MAX_UDP_SIZE = 387, + VAR_DELAY_CLOSE = 388, + VAR_UNBLOCK_LAN_ZONES = 389, + VAR_INFRA_CACHE_MIN_RTT = 390, + VAR_DNS64_PREFIX = 391, + VAR_DNS64_SYNTHALL = 392, + VAR_DNSTAP = 393, + VAR_DNSTAP_ENABLE = 394, + VAR_DNSTAP_SOCKET_PATH = 395, + VAR_DNSTAP_SEND_IDENTITY = 396, + VAR_DNSTAP_SEND_VERSION = 397, + VAR_DNSTAP_IDENTITY = 398, + VAR_DNSTAP_VERSION = 399, + VAR_DNSTAP_LOG_RESOLVER_QUERY_MESSAGES = 400, + VAR_DNSTAP_LOG_RESOLVER_RESPONSE_MESSAGES = 401, + VAR_DNSTAP_LOG_CLIENT_QUERY_MESSAGES = 402, + VAR_DNSTAP_LOG_CLIENT_RESPONSE_MESSAGES = 403, + VAR_DNSTAP_LOG_FORWARDER_QUERY_MESSAGES = 404, + VAR_DNSTAP_LOG_FORWARDER_RESPONSE_MESSAGES = 405, + VAR_HARDEN_ALGO_DOWNGRADE = 406, + VAR_IP_TRANSPARENT = 407 }; #endif /* Tokens. */ @@ -369,59 +373,63 @@ extern int yydebug; #define VAR_SERVER_CERT_FILE 348 #define VAR_CONTROL_KEY_FILE 349 #define VAR_CONTROL_CERT_FILE 350 -#define VAR_EXTENDED_STATISTICS 351 -#define VAR_LOCAL_DATA_PTR 352 -#define VAR_JOSTLE_TIMEOUT 353 -#define VAR_STUB_PRIME 354 -#define VAR_UNWANTED_REPLY_THRESHOLD 355 -#define VAR_LOG_TIME_ASCII 356 -#define VAR_DOMAIN_INSECURE 357 -#define VAR_PYTHON 358 -#define VAR_PYTHON_SCRIPT 359 -#define VAR_VAL_SIG_SKEW_MIN 360 -#define VAR_VAL_SIG_SKEW_MAX 361 -#define VAR_CACHE_MIN_TTL 362 -#define VAR_VAL_LOG_LEVEL 363 -#define VAR_AUTO_TRUST_ANCHOR_FILE 364 -#define VAR_KEEP_MISSING 365 -#define VAR_ADD_HOLDDOWN 366 -#define VAR_DEL_HOLDDOWN 367 -#define VAR_SO_RCVBUF 368 -#define VAR_EDNS_BUFFER_SIZE 369 -#define VAR_PREFETCH 370 -#define VAR_PREFETCH_KEY 371 -#define VAR_SO_SNDBUF 372 -#define VAR_SO_REUSEPORT 373 -#define VAR_HARDEN_BELOW_NXDOMAIN 374 -#define VAR_IGNORE_CD_FLAG 375 -#define VAR_LOG_QUERIES 376 -#define VAR_TCP_UPSTREAM 377 -#define VAR_SSL_UPSTREAM 378 -#define VAR_SSL_SERVICE_KEY 379 -#define VAR_SSL_SERVICE_PEM 380 -#define VAR_SSL_PORT 381 -#define VAR_FORWARD_FIRST 382 -#define VAR_STUB_FIRST 383 -#define VAR_MINIMAL_RESPONSES 384 -#define VAR_RRSET_ROUNDROBIN 385 -#define VAR_MAX_UDP_SIZE 386 -#define VAR_DELAY_CLOSE 387 -#define VAR_UNBLOCK_LAN_ZONES 388 -#define VAR_DNS64_PREFIX 389 -#define VAR_DNS64_SYNTHALL 390 -#define VAR_DNSTAP 391 -#define VAR_DNSTAP_ENABLE 392 -#define VAR_DNSTAP_SOCKET_PATH 393 -#define VAR_DNSTAP_SEND_IDENTITY 394 -#define VAR_DNSTAP_SEND_VERSION 395 -#define VAR_DNSTAP_IDENTITY 396 -#define VAR_DNSTAP_VERSION 397 -#define VAR_DNSTAP_LOG_RESOLVER_QUERY_MESSAGES 398 -#define VAR_DNSTAP_LOG_RESOLVER_RESPONSE_MESSAGES 399 -#define VAR_DNSTAP_LOG_CLIENT_QUERY_MESSAGES 400 -#define VAR_DNSTAP_LOG_CLIENT_RESPONSE_MESSAGES 401 -#define VAR_DNSTAP_LOG_FORWARDER_QUERY_MESSAGES 402 -#define VAR_DNSTAP_LOG_FORWARDER_RESPONSE_MESSAGES 403 +#define VAR_CONTROL_USE_CERT 351 +#define VAR_EXTENDED_STATISTICS 352 +#define VAR_LOCAL_DATA_PTR 353 +#define VAR_JOSTLE_TIMEOUT 354 +#define VAR_STUB_PRIME 355 +#define VAR_UNWANTED_REPLY_THRESHOLD 356 +#define VAR_LOG_TIME_ASCII 357 +#define VAR_DOMAIN_INSECURE 358 +#define VAR_PYTHON 359 +#define VAR_PYTHON_SCRIPT 360 +#define VAR_VAL_SIG_SKEW_MIN 361 +#define VAR_VAL_SIG_SKEW_MAX 362 +#define VAR_CACHE_MIN_TTL 363 +#define VAR_VAL_LOG_LEVEL 364 +#define VAR_AUTO_TRUST_ANCHOR_FILE 365 +#define VAR_KEEP_MISSING 366 +#define VAR_ADD_HOLDDOWN 367 +#define VAR_DEL_HOLDDOWN 368 +#define VAR_SO_RCVBUF 369 +#define VAR_EDNS_BUFFER_SIZE 370 +#define VAR_PREFETCH 371 +#define VAR_PREFETCH_KEY 372 +#define VAR_SO_SNDBUF 373 +#define VAR_SO_REUSEPORT 374 +#define VAR_HARDEN_BELOW_NXDOMAIN 375 +#define VAR_IGNORE_CD_FLAG 376 +#define VAR_LOG_QUERIES 377 +#define VAR_TCP_UPSTREAM 378 +#define VAR_SSL_UPSTREAM 379 +#define VAR_SSL_SERVICE_KEY 380 +#define VAR_SSL_SERVICE_PEM 381 +#define VAR_SSL_PORT 382 +#define VAR_FORWARD_FIRST 383 +#define VAR_STUB_FIRST 384 +#define VAR_MINIMAL_RESPONSES 385 +#define VAR_RRSET_ROUNDROBIN 386 +#define VAR_MAX_UDP_SIZE 387 +#define VAR_DELAY_CLOSE 388 +#define VAR_UNBLOCK_LAN_ZONES 389 +#define VAR_INFRA_CACHE_MIN_RTT 390 +#define VAR_DNS64_PREFIX 391 +#define VAR_DNS64_SYNTHALL 392 +#define VAR_DNSTAP 393 +#define VAR_DNSTAP_ENABLE 394 +#define VAR_DNSTAP_SOCKET_PATH 395 +#define VAR_DNSTAP_SEND_IDENTITY 396 +#define VAR_DNSTAP_SEND_VERSION 397 +#define VAR_DNSTAP_IDENTITY 398 +#define VAR_DNSTAP_VERSION 399 +#define VAR_DNSTAP_LOG_RESOLVER_QUERY_MESSAGES 400 +#define VAR_DNSTAP_LOG_RESOLVER_RESPONSE_MESSAGES 401 +#define VAR_DNSTAP_LOG_CLIENT_QUERY_MESSAGES 402 +#define VAR_DNSTAP_LOG_CLIENT_RESPONSE_MESSAGES 403 +#define VAR_DNSTAP_LOG_FORWARDER_QUERY_MESSAGES 404 +#define VAR_DNSTAP_LOG_FORWARDER_RESPONSE_MESSAGES 405 +#define VAR_HARDEN_ALGO_DOWNGRADE 406 +#define VAR_IP_TRANSPARENT 407 @@ -435,7 +443,7 @@ typedef union YYSTYPE /* Line 387 of yacc.c */ -#line 439 "util/configparser.c" +#line 447 "util/configparser.c" } YYSTYPE; # define YYSTYPE_IS_TRIVIAL 1 # define yystype YYSTYPE /* obsolescent; will be withdrawn */ @@ -463,7 +471,7 @@ int yyparse (); /* Copy the second part of user declarations. */ /* Line 390 of yacc.c */ -#line 467 "util/configparser.c" +#line 475 "util/configparser.c" #ifdef short # undef short @@ -683,20 +691,20 @@ union yyalloc /* YYFINAL -- State number of the termination state. */ #define YYFINAL 2 /* YYLAST -- Last index in YYTABLE. */ -#define YYLAST 276 +#define YYLAST 283 /* YYNTOKENS -- Number of terminals. */ -#define YYNTOKENS 149 +#define YYNTOKENS 153 /* YYNNTS -- Number of nonterminals. */ -#define YYNNTS 154 +#define YYNNTS 158 /* YYNRULES -- Number of rules. */ -#define YYNRULES 293 +#define YYNRULES 301 /* YYNRULES -- Number of states. */ -#define YYNSTATES 429 +#define YYNSTATES 441 /* YYTRANSLATE(YYLEX) -- Bison symbol number corresponding to YYLEX. */ #define YYUNDEFTOK 2 -#define YYMAXUTOK 403 +#define YYMAXUTOK 407 #define YYTRANSLATE(YYX) \ ((unsigned int) (YYX) <= YYMAXUTOK ? yytranslate[YYX] : YYUNDEFTOK) @@ -744,7 +752,7 @@ static const yytype_uint8 yytranslate[] = 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, - 145, 146, 147, 148 + 145, 146, 147, 148, 149, 150, 151, 152 }; #if YYDEBUG @@ -763,138 +771,142 @@ static const yytype_uint16 yyprhs[] = 165, 167, 169, 171, 173, 175, 177, 179, 181, 183, 185, 187, 189, 191, 193, 195, 197, 199, 201, 203, 205, 207, 209, 211, 213, 215, 217, 219, 221, 223, - 225, 227, 229, 231, 233, 235, 237, 239, 241, 244, - 245, 247, 249, 251, 253, 255, 257, 260, 261, 263, - 265, 267, 269, 272, 275, 278, 281, 284, 287, 290, - 293, 296, 299, 302, 305, 308, 311, 314, 317, 320, - 323, 326, 329, 332, 335, 338, 341, 344, 347, 350, - 353, 356, 359, 362, 365, 368, 371, 374, 377, 380, - 383, 386, 389, 392, 395, 398, 401, 404, 407, 410, - 413, 416, 419, 422, 425, 428, 431, 434, 437, 440, - 443, 446, 449, 452, 455, 458, 461, 464, 467, 470, - 473, 476, 479, 482, 485, 488, 491, 494, 497, 500, - 504, 507, 510, 513, 516, 519, 522, 525, 528, 531, - 534, 537, 540, 543, 546, 549, 552, 555, 558, 562, - 565, 568, 571, 574, 577, 580, 583, 586, 589, 592, - 595, 598, 601, 604, 607, 610, 612, 615, 616, 618, - 620, 622, 624, 626, 628, 630, 633, 636, 639, 642, - 645, 648, 651, 653, 656, 657, 659, 661, 663, 665, - 667, 669, 671, 673, 675, 677, 679, 681, 684, 687, - 690, 693, 696, 699, 702, 705, 708, 711, 714, 717, - 719, 722, 723, 725 + 225, 227, 229, 231, 233, 235, 237, 239, 241, 243, + 245, 247, 250, 251, 253, 255, 257, 259, 261, 263, + 266, 267, 269, 271, 273, 275, 278, 281, 284, 287, + 290, 293, 296, 299, 302, 305, 308, 311, 314, 317, + 320, 323, 326, 329, 332, 335, 338, 341, 344, 347, + 350, 353, 356, 359, 362, 365, 368, 371, 374, 377, + 380, 383, 386, 389, 392, 395, 398, 401, 404, 407, + 410, 413, 416, 419, 422, 425, 428, 431, 434, 437, + 440, 443, 446, 449, 452, 455, 458, 461, 464, 467, + 470, 473, 476, 479, 482, 485, 488, 491, 494, 497, + 500, 503, 506, 509, 512, 515, 519, 522, 525, 528, + 531, 534, 537, 540, 543, 546, 549, 552, 555, 558, + 561, 564, 567, 570, 573, 577, 580, 583, 586, 589, + 592, 595, 598, 601, 604, 607, 610, 613, 616, 619, + 622, 625, 627, 630, 631, 633, 635, 637, 639, 641, + 643, 645, 647, 650, 653, 656, 659, 662, 665, 668, + 671, 673, 676, 677, 679, 681, 683, 685, 687, 689, + 691, 693, 695, 697, 699, 701, 704, 707, 710, 713, + 716, 719, 722, 725, 728, 731, 734, 737, 739, 742, + 743, 745 }; /* YYRHS -- A `-1'-separated list of the rules' RHS. */ static const yytype_int16 yyrhs[] = { - 150, 0, -1, -1, 150, 151, -1, 152, 153, -1, - 155, 156, -1, 158, 159, -1, 299, 300, -1, 274, - 275, -1, 284, 285, -1, 11, -1, 153, 154, -1, - -1, 161, -1, 162, -1, 166, -1, 169, -1, 175, - -1, 176, -1, 177, -1, 178, -1, 167, -1, 188, - -1, 189, -1, 190, -1, 191, -1, 192, -1, 210, - -1, 211, -1, 212, -1, 216, -1, 217, -1, 172, - -1, 218, -1, 219, -1, 222, -1, 220, -1, 221, - -1, 223, -1, 224, -1, 225, -1, 236, -1, 201, - -1, 202, -1, 203, -1, 204, -1, 226, -1, 239, - -1, 197, -1, 199, -1, 240, -1, 245, -1, 246, - -1, 247, -1, 173, -1, 209, -1, 254, -1, 255, - -1, 198, -1, 250, -1, 185, -1, 168, -1, 193, - -1, 237, -1, 243, -1, 227, -1, 238, -1, 257, - -1, 258, -1, 174, -1, 163, -1, 184, -1, 230, - -1, 164, -1, 170, -1, 171, -1, 194, -1, 195, - -1, 256, -1, 229, -1, 231, -1, 232, -1, 165, - -1, 259, -1, 213, -1, 235, -1, 186, -1, 200, - -1, 241, -1, 242, -1, 244, -1, 249, -1, 196, - -1, 251, -1, 252, -1, 253, -1, 205, -1, 208, - -1, 233, -1, 234, -1, 206, -1, 228, -1, 248, - -1, 187, -1, 179, -1, 180, -1, 181, -1, 182, - -1, 183, -1, 260, -1, 261, -1, 262, -1, 207, - -1, 214, -1, 215, -1, 263, -1, 264, -1, 38, - -1, 156, 157, -1, -1, 265, -1, 266, -1, 267, - -1, 269, -1, 268, -1, 44, -1, 159, 160, -1, - -1, 270, -1, 271, -1, 272, -1, 273, -1, 13, - 10, -1, 12, 10, -1, 76, 10, -1, 79, 10, - -1, 96, 10, -1, 14, 10, -1, 16, 10, -1, - 67, 10, -1, 15, 10, -1, 80, 10, -1, 81, - 10, -1, 31, 10, -1, 60, 10, -1, 75, 10, - -1, 17, 10, -1, 18, 10, -1, 19, 10, -1, - 20, 10, -1, 122, 10, -1, 123, 10, -1, 124, - 10, -1, 125, 10, -1, 126, 10, -1, 77, 10, - -1, 66, 10, -1, 101, 10, -1, 121, 10, -1, - 21, 10, -1, 22, 10, -1, 23, 10, -1, 24, - 10, -1, 25, 10, -1, 68, 10, -1, 82, 10, - -1, 83, 10, -1, 109, 10, -1, 54, 10, -1, - 64, 10, -1, 55, 10, -1, 102, 10, -1, 48, - 10, -1, 49, 10, -1, 50, 10, -1, 51, 10, - -1, 113, 10, -1, 117, 10, -1, 118, 10, -1, - 114, 10, -1, 61, 10, -1, 26, 10, -1, 27, - 10, -1, 28, 10, -1, 98, 10, -1, 132, 10, - -1, 133, 10, -1, 29, 10, -1, 30, 10, -1, - 32, 10, -1, 33, 10, -1, 35, 10, -1, 36, - 10, -1, 34, 10, -1, 41, 10, -1, 42, 10, - -1, 43, 10, -1, 52, 10, -1, 71, 10, -1, - 119, 10, -1, 85, 10, -1, 78, 10, -1, 86, - 10, -1, 87, 10, -1, 115, 10, -1, 116, 10, - -1, 100, 10, -1, 47, 10, -1, 69, 10, -1, - 72, 10, 10, -1, 53, 10, -1, 56, 10, -1, - 105, 10, -1, 106, 10, -1, 70, 10, -1, 107, - 10, -1, 57, 10, -1, 58, 10, -1, 59, 10, - -1, 120, 10, -1, 108, 10, -1, 65, 10, -1, - 111, 10, -1, 112, 10, -1, 110, 10, -1, 62, - 10, -1, 63, 10, -1, 84, 10, -1, 73, 10, - 10, -1, 74, 10, -1, 97, 10, -1, 129, 10, - -1, 130, 10, -1, 131, 10, -1, 134, 10, -1, - 135, 10, -1, 37, 10, -1, 39, 10, -1, 40, - 10, -1, 128, 10, -1, 99, 10, -1, 37, 10, - -1, 45, 10, -1, 46, 10, -1, 127, 10, -1, - 88, -1, 275, 276, -1, -1, 277, -1, 279, -1, - 278, -1, 280, -1, 281, -1, 282, -1, 283, -1, - 89, 10, -1, 91, 10, -1, 90, 10, -1, 92, + 154, 0, -1, -1, 154, 155, -1, 156, 157, -1, + 159, 160, -1, 162, 163, -1, 307, 308, -1, 281, + 282, -1, 292, 293, -1, 11, -1, 157, 158, -1, + -1, 165, -1, 166, -1, 170, -1, 173, -1, 179, + -1, 180, -1, 181, -1, 182, -1, 171, -1, 192, + -1, 193, -1, 194, -1, 195, -1, 196, -1, 215, + -1, 216, -1, 217, -1, 221, -1, 222, -1, 176, + -1, 223, -1, 224, -1, 227, -1, 225, -1, 226, + -1, 229, -1, 230, -1, 231, -1, 243, -1, 205, + -1, 206, -1, 207, -1, 208, -1, 232, -1, 246, + -1, 201, -1, 203, -1, 247, -1, 252, -1, 253, + -1, 254, -1, 177, -1, 214, -1, 261, -1, 262, + -1, 202, -1, 257, -1, 189, -1, 172, -1, 197, + -1, 244, -1, 250, -1, 233, -1, 245, -1, 264, + -1, 265, -1, 178, -1, 167, -1, 188, -1, 237, + -1, 168, -1, 174, -1, 175, -1, 198, -1, 199, + -1, 263, -1, 235, -1, 238, -1, 239, -1, 169, + -1, 266, -1, 218, -1, 242, -1, 190, -1, 204, + -1, 248, -1, 249, -1, 251, -1, 256, -1, 200, + -1, 258, -1, 259, -1, 260, -1, 209, -1, 213, + -1, 240, -1, 241, -1, 210, -1, 234, -1, 255, + -1, 191, -1, 183, -1, 184, -1, 185, -1, 186, + -1, 187, -1, 267, -1, 268, -1, 269, -1, 211, + -1, 219, -1, 220, -1, 270, -1, 271, -1, 228, + -1, 236, -1, 212, -1, 38, -1, 160, 161, -1, + -1, 272, -1, 273, -1, 274, -1, 276, -1, 275, + -1, 44, -1, 163, 164, -1, -1, 277, -1, 278, + -1, 279, -1, 280, -1, 13, 10, -1, 12, 10, + -1, 76, 10, -1, 79, 10, -1, 97, 10, -1, + 14, 10, -1, 16, 10, -1, 67, 10, -1, 15, + 10, -1, 80, 10, -1, 81, 10, -1, 31, 10, + -1, 60, 10, -1, 75, 10, -1, 17, 10, -1, + 18, 10, -1, 19, 10, -1, 20, 10, -1, 123, + 10, -1, 124, 10, -1, 125, 10, -1, 126, 10, + -1, 127, 10, -1, 77, 10, -1, 66, 10, -1, + 102, 10, -1, 122, 10, -1, 21, 10, -1, 22, + 10, -1, 23, 10, -1, 24, 10, -1, 25, 10, + -1, 68, 10, -1, 82, 10, -1, 83, 10, -1, + 110, 10, -1, 54, 10, -1, 64, 10, -1, 55, + 10, -1, 103, 10, -1, 48, 10, -1, 49, 10, + -1, 50, 10, -1, 51, 10, -1, 114, 10, -1, + 118, 10, -1, 119, 10, -1, 152, 10, -1, 115, + 10, -1, 61, 10, -1, 26, 10, -1, 27, 10, + -1, 28, 10, -1, 99, 10, -1, 133, 10, -1, + 134, 10, -1, 29, 10, -1, 30, 10, -1, 32, + 10, -1, 33, 10, -1, 35, 10, -1, 36, 10, + -1, 34, 10, -1, 135, 10, -1, 41, 10, -1, + 42, 10, -1, 43, 10, -1, 52, 10, -1, 71, + 10, -1, 120, 10, -1, 85, 10, -1, 151, 10, + -1, 78, 10, -1, 86, 10, -1, 87, 10, -1, + 116, 10, -1, 117, 10, -1, 101, 10, -1, 47, + 10, -1, 69, 10, -1, 72, 10, 10, -1, 53, + 10, -1, 56, 10, -1, 106, 10, -1, 107, 10, + -1, 70, 10, -1, 108, 10, -1, 57, 10, -1, + 58, 10, -1, 59, 10, -1, 121, 10, -1, 109, + 10, -1, 65, 10, -1, 112, 10, -1, 113, 10, + -1, 111, 10, -1, 62, 10, -1, 63, 10, -1, + 84, 10, -1, 73, 10, 10, -1, 74, 10, -1, + 98, 10, -1, 130, 10, -1, 131, 10, -1, 132, + 10, -1, 136, 10, -1, 137, 10, -1, 37, 10, + -1, 39, 10, -1, 40, 10, -1, 129, 10, -1, + 100, 10, -1, 37, 10, -1, 45, 10, -1, 46, + 10, -1, 128, 10, -1, 88, -1, 282, 283, -1, + -1, 284, -1, 286, -1, 285, -1, 288, -1, 289, + -1, 290, -1, 291, -1, 287, -1, 89, 10, -1, + 91, 10, -1, 90, 10, -1, 96, 10, -1, 92, 10, -1, 93, 10, -1, 94, 10, -1, 95, 10, - -1, 136, -1, 285, 286, -1, -1, 287, -1, 288, - -1, 289, -1, 290, -1, 291, -1, 292, -1, 293, - -1, 294, -1, 295, -1, 296, -1, 297, -1, 298, - -1, 137, 10, -1, 138, 10, -1, 139, 10, -1, - 140, 10, -1, 141, 10, -1, 142, 10, -1, 143, - 10, -1, 144, 10, -1, 145, 10, -1, 146, 10, - -1, 147, 10, -1, 148, 10, -1, 103, -1, 300, - 301, -1, -1, 302, -1, 104, 10, -1 + -1, 138, -1, 293, 294, -1, -1, 295, -1, 296, + -1, 297, -1, 298, -1, 299, -1, 300, -1, 301, + -1, 302, -1, 303, -1, 304, -1, 305, -1, 306, + -1, 139, 10, -1, 140, 10, -1, 141, 10, -1, + 142, 10, -1, 143, 10, -1, 144, 10, -1, 145, + 10, -1, 146, 10, -1, 147, 10, -1, 148, 10, + -1, 149, 10, -1, 150, 10, -1, 104, -1, 308, + 309, -1, -1, 310, -1, 105, 10, -1 }; /* YYRLINE[YYN] -- source line where rule number YYN was defined. */ static const yytype_uint16 yyrline[] = { - 0, 121, 121, 121, 122, 122, 123, 123, 124, 124, - 128, 133, 134, 135, 135, 135, 136, 136, 137, 137, - 137, 138, 138, 138, 139, 139, 139, 140, 140, 141, - 141, 142, 142, 143, 143, 144, 144, 145, 145, 146, - 146, 147, 147, 148, 148, 148, 149, 149, 149, 150, - 150, 150, 151, 151, 152, 152, 153, 153, 154, 154, - 155, 155, 155, 156, 156, 157, 157, 158, 158, 158, - 159, 159, 160, 160, 161, 161, 162, 162, 162, 163, - 163, 164, 164, 165, 165, 166, 166, 167, 167, 168, - 168, 168, 169, 169, 170, 170, 170, 171, 171, 171, - 172, 172, 172, 173, 173, 173, 174, 174, 174, 175, - 175, 175, 176, 176, 176, 177, 177, 179, 191, 192, - 193, 193, 193, 193, 193, 195, 207, 208, 209, 209, - 209, 209, 211, 220, 229, 240, 249, 258, 267, 280, - 295, 304, 313, 322, 331, 340, 349, 358, 367, 376, - 385, 394, 403, 410, 417, 426, 435, 449, 458, 467, - 474, 481, 488, 496, 503, 510, 517, 524, 532, 540, - 548, 555, 562, 571, 580, 587, 594, 602, 610, 620, - 633, 644, 652, 665, 674, 683, 692, 702, 710, 723, - 732, 740, 749, 757, 770, 777, 787, 797, 807, 817, - 827, 837, 847, 854, 861, 870, 879, 888, 895, 905, - 922, 929, 947, 960, 973, 982, 991, 1000, 1010, 1020, - 1029, 1038, 1045, 1054, 1063, 1072, 1080, 1093, 1101, 1123, - 1130, 1145, 1155, 1165, 1172, 1179, 1188, 1198, 1205, 1212, - 1221, 1231, 1241, 1248, 1255, 1264, 1269, 1270, 1271, 1271, - 1271, 1272, 1272, 1272, 1273, 1275, 1285, 1294, 1301, 1308, - 1315, 1322, 1329, 1334, 1335, 1336, 1336, 1337, 1337, 1338, - 1338, 1339, 1340, 1341, 1342, 1343, 1344, 1346, 1354, 1361, - 1369, 1377, 1384, 1391, 1400, 1409, 1418, 1427, 1436, 1445, - 1450, 1451, 1452, 1454 + 0, 124, 124, 124, 125, 125, 126, 126, 127, 127, + 131, 136, 137, 138, 138, 138, 139, 139, 140, 140, + 140, 141, 141, 141, 142, 142, 142, 143, 143, 144, + 144, 145, 145, 146, 146, 147, 147, 148, 148, 149, + 149, 150, 150, 151, 151, 151, 152, 152, 152, 153, + 153, 153, 154, 154, 155, 155, 156, 156, 157, 157, + 158, 158, 158, 159, 159, 160, 160, 161, 161, 161, + 162, 162, 163, 163, 164, 164, 165, 165, 165, 166, + 166, 167, 167, 168, 168, 169, 169, 170, 170, 171, + 171, 171, 172, 172, 173, 173, 173, 174, 174, 174, + 175, 175, 175, 176, 176, 176, 177, 177, 177, 178, + 178, 178, 179, 179, 179, 180, 180, 181, 181, 182, + 184, 196, 197, 198, 198, 198, 198, 198, 200, 212, + 213, 214, 214, 214, 214, 216, 225, 234, 245, 254, + 263, 272, 285, 300, 309, 318, 327, 336, 345, 354, + 363, 372, 381, 390, 399, 408, 415, 422, 431, 440, + 454, 463, 472, 479, 486, 493, 501, 508, 515, 522, + 529, 537, 545, 553, 560, 567, 576, 585, 592, 599, + 607, 615, 625, 635, 648, 659, 667, 680, 689, 698, + 707, 717, 725, 738, 747, 755, 764, 772, 785, 794, + 801, 811, 821, 831, 841, 851, 861, 871, 881, 888, + 895, 904, 913, 922, 929, 939, 956, 963, 981, 994, + 1007, 1016, 1025, 1034, 1044, 1054, 1063, 1072, 1079, 1088, + 1097, 1106, 1114, 1127, 1135, 1158, 1165, 1180, 1190, 1200, + 1207, 1214, 1223, 1233, 1240, 1247, 1256, 1266, 1276, 1283, + 1290, 1299, 1304, 1305, 1306, 1306, 1306, 1307, 1307, 1307, + 1308, 1308, 1310, 1320, 1329, 1336, 1346, 1353, 1360, 1367, + 1374, 1379, 1380, 1381, 1381, 1382, 1382, 1383, 1383, 1384, + 1385, 1386, 1387, 1388, 1389, 1391, 1399, 1406, 1414, 1422, + 1429, 1436, 1445, 1454, 1463, 1472, 1481, 1490, 1495, 1496, + 1497, 1499 }; #endif @@ -933,32 +945,34 @@ static const char *const yytname[] = "VAR_PRIVATE_DOMAIN", "VAR_REMOTE_CONTROL", "VAR_CONTROL_ENABLE", "VAR_CONTROL_INTERFACE", "VAR_CONTROL_PORT", "VAR_SERVER_KEY_FILE", "VAR_SERVER_CERT_FILE", "VAR_CONTROL_KEY_FILE", "VAR_CONTROL_CERT_FILE", - "VAR_EXTENDED_STATISTICS", "VAR_LOCAL_DATA_PTR", "VAR_JOSTLE_TIMEOUT", - "VAR_STUB_PRIME", "VAR_UNWANTED_REPLY_THRESHOLD", "VAR_LOG_TIME_ASCII", - "VAR_DOMAIN_INSECURE", "VAR_PYTHON", "VAR_PYTHON_SCRIPT", - "VAR_VAL_SIG_SKEW_MIN", "VAR_VAL_SIG_SKEW_MAX", "VAR_CACHE_MIN_TTL", - "VAR_VAL_LOG_LEVEL", "VAR_AUTO_TRUST_ANCHOR_FILE", "VAR_KEEP_MISSING", - "VAR_ADD_HOLDDOWN", "VAR_DEL_HOLDDOWN", "VAR_SO_RCVBUF", - "VAR_EDNS_BUFFER_SIZE", "VAR_PREFETCH", "VAR_PREFETCH_KEY", - "VAR_SO_SNDBUF", "VAR_SO_REUSEPORT", "VAR_HARDEN_BELOW_NXDOMAIN", - "VAR_IGNORE_CD_FLAG", "VAR_LOG_QUERIES", "VAR_TCP_UPSTREAM", - "VAR_SSL_UPSTREAM", "VAR_SSL_SERVICE_KEY", "VAR_SSL_SERVICE_PEM", - "VAR_SSL_PORT", "VAR_FORWARD_FIRST", "VAR_STUB_FIRST", - "VAR_MINIMAL_RESPONSES", "VAR_RRSET_ROUNDROBIN", "VAR_MAX_UDP_SIZE", - "VAR_DELAY_CLOSE", "VAR_UNBLOCK_LAN_ZONES", "VAR_DNS64_PREFIX", - "VAR_DNS64_SYNTHALL", "VAR_DNSTAP", "VAR_DNSTAP_ENABLE", - "VAR_DNSTAP_SOCKET_PATH", "VAR_DNSTAP_SEND_IDENTITY", - "VAR_DNSTAP_SEND_VERSION", "VAR_DNSTAP_IDENTITY", "VAR_DNSTAP_VERSION", + "VAR_CONTROL_USE_CERT", "VAR_EXTENDED_STATISTICS", "VAR_LOCAL_DATA_PTR", + "VAR_JOSTLE_TIMEOUT", "VAR_STUB_PRIME", "VAR_UNWANTED_REPLY_THRESHOLD", + "VAR_LOG_TIME_ASCII", "VAR_DOMAIN_INSECURE", "VAR_PYTHON", + "VAR_PYTHON_SCRIPT", "VAR_VAL_SIG_SKEW_MIN", "VAR_VAL_SIG_SKEW_MAX", + "VAR_CACHE_MIN_TTL", "VAR_VAL_LOG_LEVEL", "VAR_AUTO_TRUST_ANCHOR_FILE", + "VAR_KEEP_MISSING", "VAR_ADD_HOLDDOWN", "VAR_DEL_HOLDDOWN", + "VAR_SO_RCVBUF", "VAR_EDNS_BUFFER_SIZE", "VAR_PREFETCH", + "VAR_PREFETCH_KEY", "VAR_SO_SNDBUF", "VAR_SO_REUSEPORT", + "VAR_HARDEN_BELOW_NXDOMAIN", "VAR_IGNORE_CD_FLAG", "VAR_LOG_QUERIES", + "VAR_TCP_UPSTREAM", "VAR_SSL_UPSTREAM", "VAR_SSL_SERVICE_KEY", + "VAR_SSL_SERVICE_PEM", "VAR_SSL_PORT", "VAR_FORWARD_FIRST", + "VAR_STUB_FIRST", "VAR_MINIMAL_RESPONSES", "VAR_RRSET_ROUNDROBIN", + "VAR_MAX_UDP_SIZE", "VAR_DELAY_CLOSE", "VAR_UNBLOCK_LAN_ZONES", + "VAR_INFRA_CACHE_MIN_RTT", "VAR_DNS64_PREFIX", "VAR_DNS64_SYNTHALL", + "VAR_DNSTAP", "VAR_DNSTAP_ENABLE", "VAR_DNSTAP_SOCKET_PATH", + "VAR_DNSTAP_SEND_IDENTITY", "VAR_DNSTAP_SEND_VERSION", + "VAR_DNSTAP_IDENTITY", "VAR_DNSTAP_VERSION", "VAR_DNSTAP_LOG_RESOLVER_QUERY_MESSAGES", "VAR_DNSTAP_LOG_RESOLVER_RESPONSE_MESSAGES", "VAR_DNSTAP_LOG_CLIENT_QUERY_MESSAGES", "VAR_DNSTAP_LOG_CLIENT_RESPONSE_MESSAGES", "VAR_DNSTAP_LOG_FORWARDER_QUERY_MESSAGES", - "VAR_DNSTAP_LOG_FORWARDER_RESPONSE_MESSAGES", "$accept", "toplevelvars", - "toplevelvar", "serverstart", "contents_server", "content_server", - "stubstart", "contents_stub", "content_stub", "forwardstart", - "contents_forward", "content_forward", "server_num_threads", - "server_verbosity", "server_statistics_interval", + "VAR_DNSTAP_LOG_FORWARDER_RESPONSE_MESSAGES", + "VAR_HARDEN_ALGO_DOWNGRADE", "VAR_IP_TRANSPARENT", "$accept", + "toplevelvars", "toplevelvar", "serverstart", "contents_server", + "content_server", "stubstart", "contents_stub", "content_stub", + "forwardstart", "contents_forward", "content_forward", + "server_num_threads", "server_verbosity", "server_statistics_interval", "server_statistics_cumulative", "server_extended_statistics", "server_port", "server_interface", "server_outgoing_interface", "server_outgoing_range", "server_outgoing_port_permit", @@ -975,24 +989,26 @@ static const char *const yytname[] = "server_trust_anchor", "server_domain_insecure", "server_hide_identity", "server_hide_version", "server_identity", "server_version", "server_so_rcvbuf", "server_so_sndbuf", "server_so_reuseport", - "server_edns_buffer_size", "server_msg_buffer_size", - "server_msg_cache_size", "server_msg_cache_slabs", - "server_num_queries_per_thread", "server_jostle_timeout", - "server_delay_close", "server_unblock_lan_zones", - "server_rrset_cache_size", "server_rrset_cache_slabs", - "server_infra_host_ttl", "server_infra_lame_ttl", - "server_infra_cache_numhosts", "server_infra_cache_lame_size", - "server_infra_cache_slabs", "server_target_fetch_policy", + "server_ip_transparent", "server_edns_buffer_size", + "server_msg_buffer_size", "server_msg_cache_size", + "server_msg_cache_slabs", "server_num_queries_per_thread", + "server_jostle_timeout", "server_delay_close", + "server_unblock_lan_zones", "server_rrset_cache_size", + "server_rrset_cache_slabs", "server_infra_host_ttl", + "server_infra_lame_ttl", "server_infra_cache_numhosts", + "server_infra_cache_lame_size", "server_infra_cache_slabs", + "server_infra_cache_min_rtt", "server_target_fetch_policy", "server_harden_short_bufsize", "server_harden_large_queries", "server_harden_glue", "server_harden_dnssec_stripped", "server_harden_below_nxdomain", "server_harden_referral_path", - "server_use_caps_for_id", "server_private_address", - "server_private_domain", "server_prefetch", "server_prefetch_key", - "server_unwanted_reply_threshold", "server_do_not_query_address", - "server_do_not_query_localhost", "server_access_control", - "server_module_conf", "server_val_override_date", - "server_val_sig_skew_min", "server_val_sig_skew_max", - "server_cache_max_ttl", "server_cache_min_ttl", "server_bogus_ttl", + "server_harden_algo_downgrade", "server_use_caps_for_id", + "server_private_address", "server_private_domain", "server_prefetch", + "server_prefetch_key", "server_unwanted_reply_threshold", + "server_do_not_query_address", "server_do_not_query_localhost", + "server_access_control", "server_module_conf", + "server_val_override_date", "server_val_sig_skew_min", + "server_val_sig_skew_max", "server_cache_max_ttl", + "server_cache_min_ttl", "server_bogus_ttl", "server_val_clean_additional", "server_val_permissive_mode", "server_ignore_cd_flag", "server_val_log_level", "server_val_nsec3_keysize_iterations", "server_add_holddown", @@ -1004,10 +1020,11 @@ static const char *const yytname[] = "stub_first", "stub_prime", "forward_name", "forward_host", "forward_addr", "forward_first", "rcstart", "contents_rc", "content_rc", "rc_control_enable", "rc_control_port", "rc_control_interface", - "rc_server_key_file", "rc_server_cert_file", "rc_control_key_file", - "rc_control_cert_file", "dtstart", "contents_dt", "content_dt", - "dt_dnstap_enable", "dt_dnstap_socket_path", "dt_dnstap_send_identity", - "dt_dnstap_send_version", "dt_dnstap_identity", "dt_dnstap_version", + "rc_control_use_cert", "rc_server_key_file", "rc_server_cert_file", + "rc_control_key_file", "rc_control_cert_file", "dtstart", "contents_dt", + "content_dt", "dt_dnstap_enable", "dt_dnstap_socket_path", + "dt_dnstap_send_identity", "dt_dnstap_send_version", + "dt_dnstap_identity", "dt_dnstap_version", "dt_dnstap_log_resolver_query_messages", "dt_dnstap_log_resolver_response_messages", "dt_dnstap_log_client_query_messages", @@ -1037,43 +1054,45 @@ static const yytype_uint16 yytoknum[] = 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, - 395, 396, 397, 398, 399, 400, 401, 402, 403 + 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, + 405, 406, 407 }; # endif /* YYR1[YYN] -- Symbol number of symbol that rule YYN derives. */ static const yytype_uint16 yyr1[] = { - 0, 149, 150, 150, 151, 151, 151, 151, 151, 151, - 152, 153, 153, 154, 154, 154, 154, 154, 154, 154, - 154, 154, 154, 154, 154, 154, 154, 154, 154, 154, - 154, 154, 154, 154, 154, 154, 154, 154, 154, 154, - 154, 154, 154, 154, 154, 154, 154, 154, 154, 154, - 154, 154, 154, 154, 154, 154, 154, 154, 154, 154, - 154, 154, 154, 154, 154, 154, 154, 154, 154, 154, - 154, 154, 154, 154, 154, 154, 154, 154, 154, 154, - 154, 154, 154, 154, 154, 154, 154, 154, 154, 154, - 154, 154, 154, 154, 154, 154, 154, 154, 154, 154, - 154, 154, 154, 154, 154, 154, 154, 154, 154, 154, - 154, 154, 154, 154, 154, 154, 154, 155, 156, 156, - 157, 157, 157, 157, 157, 158, 159, 159, 160, 160, - 160, 160, 161, 162, 163, 164, 165, 166, 167, 168, - 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, - 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, - 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, - 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, - 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, - 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, - 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, - 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, - 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, - 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, - 269, 270, 271, 272, 273, 274, 275, 275, 276, 276, - 276, 276, 276, 276, 276, 277, 278, 279, 280, 281, - 282, 283, 284, 285, 285, 286, 286, 286, 286, 286, - 286, 286, 286, 286, 286, 286, 286, 287, 288, 289, - 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, - 300, 300, 301, 302 + 0, 153, 154, 154, 155, 155, 155, 155, 155, 155, + 156, 157, 157, 158, 158, 158, 158, 158, 158, 158, + 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, + 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, + 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, + 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, + 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, + 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, + 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, + 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, + 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, + 158, 158, 158, 158, 158, 158, 158, 158, 158, 158, + 159, 160, 160, 161, 161, 161, 161, 161, 162, 163, + 163, 164, 164, 164, 164, 165, 166, 167, 168, 169, + 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, + 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, + 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, + 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, + 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, + 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, + 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, + 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, + 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, + 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, + 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, + 280, 281, 282, 282, 283, 283, 283, 283, 283, 283, + 283, 283, 284, 285, 286, 287, 288, 289, 290, 291, + 292, 293, 293, 294, 294, 294, 294, 294, 294, 294, + 294, 294, 294, 294, 294, 295, 296, 297, 298, 299, + 300, 301, 302, 303, 304, 305, 306, 307, 308, 308, + 309, 310 }; /* YYR2[YYN] -- Number of symbols composing right hand side of rule YYN. */ @@ -1090,25 +1109,26 @@ static const yytype_uint8 yyr2[] = 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 2, 0, - 1, 1, 1, 1, 1, 1, 2, 0, 1, 1, - 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 2, 0, 1, 1, 1, 1, 1, 1, 2, + 0, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, + 2, 2, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 1, 2, 0, 1, 1, + 2, 2, 2, 2, 3, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 1, 2, 0, 1, 1, 1, 1, 1, 1, + 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, + 1, 2, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, - 2, 2, 1, 2, 0, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, - 2, 0, 1, 2 + 2, 2, 2, 2, 2, 2, 2, 1, 2, 0, + 1, 2 }; /* YYDEFACT[STATE-NAME] -- Default reduction number in state STATE-NUM. @@ -1116,8 +1136,8 @@ static const yytype_uint8 yyr2[] = means the default is an error. */ static const yytype_uint16 yydefact[] = { - 2, 0, 1, 10, 117, 125, 245, 289, 262, 3, - 12, 119, 127, 247, 264, 291, 4, 5, 6, 8, + 2, 0, 1, 10, 120, 128, 251, 297, 270, 3, + 12, 122, 130, 253, 272, 299, 4, 5, 6, 8, 9, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -1128,129 +1148,133 @@ static const yytype_uint16 yydefact[] = 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 11, 13, 14, 70, - 73, 82, 15, 21, 61, 16, 74, 75, 32, 54, - 69, 17, 18, 19, 20, 104, 105, 106, 107, 108, - 71, 60, 86, 103, 22, 23, 24, 25, 26, 62, - 76, 77, 92, 48, 58, 49, 87, 42, 43, 44, - 45, 96, 100, 112, 97, 55, 27, 28, 29, 84, - 113, 114, 30, 31, 33, 34, 36, 37, 35, 38, - 39, 40, 46, 65, 101, 79, 72, 80, 81, 98, - 99, 85, 41, 63, 66, 47, 50, 88, 89, 64, - 90, 51, 52, 53, 102, 91, 59, 93, 94, 95, - 56, 57, 78, 67, 68, 83, 109, 110, 111, 115, - 116, 0, 0, 0, 0, 0, 118, 120, 121, 122, - 124, 123, 0, 0, 0, 0, 126, 128, 129, 130, - 131, 0, 0, 0, 0, 0, 0, 0, 246, 248, - 250, 249, 251, 252, 253, 254, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 263, 265, - 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, - 276, 0, 290, 292, 133, 132, 137, 140, 138, 146, - 147, 148, 149, 159, 160, 161, 162, 163, 181, 182, - 183, 187, 188, 143, 189, 190, 193, 191, 192, 194, - 195, 196, 207, 172, 173, 174, 175, 197, 210, 168, - 170, 211, 216, 217, 218, 144, 180, 225, 226, 169, - 221, 156, 139, 164, 208, 214, 198, 0, 0, 229, - 145, 134, 155, 201, 135, 141, 142, 165, 166, 227, - 200, 202, 203, 136, 230, 184, 206, 157, 171, 212, - 213, 215, 220, 167, 224, 222, 223, 176, 179, 204, - 205, 177, 178, 199, 219, 158, 150, 151, 152, 153, - 154, 231, 232, 233, 185, 186, 234, 235, 236, 237, - 238, 240, 239, 241, 242, 243, 244, 255, 257, 256, - 258, 259, 260, 261, 277, 278, 279, 280, 281, 282, - 283, 284, 285, 286, 287, 288, 293, 209, 228 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, + 13, 14, 70, 73, 82, 15, 21, 61, 16, 74, + 75, 32, 54, 69, 17, 18, 19, 20, 104, 105, + 106, 107, 108, 71, 60, 86, 103, 22, 23, 24, + 25, 26, 62, 76, 77, 92, 48, 58, 49, 87, + 42, 43, 44, 45, 96, 100, 112, 119, 97, 55, + 27, 28, 29, 84, 113, 114, 30, 31, 33, 34, + 36, 37, 35, 117, 38, 39, 40, 46, 65, 101, + 79, 118, 72, 80, 81, 98, 99, 85, 41, 63, + 66, 47, 50, 88, 89, 64, 90, 51, 52, 53, + 102, 91, 59, 93, 94, 95, 56, 57, 78, 67, + 68, 83, 109, 110, 111, 115, 116, 0, 0, 0, + 0, 0, 121, 123, 124, 125, 127, 126, 0, 0, + 0, 0, 129, 131, 132, 133, 134, 0, 0, 0, + 0, 0, 0, 0, 0, 252, 254, 256, 255, 261, + 257, 258, 259, 260, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 271, 273, 274, 275, + 276, 277, 278, 279, 280, 281, 282, 283, 284, 0, + 298, 300, 136, 135, 140, 143, 141, 149, 150, 151, + 152, 162, 163, 164, 165, 166, 185, 186, 187, 191, + 192, 146, 193, 194, 197, 195, 196, 199, 200, 201, + 213, 175, 176, 177, 178, 202, 216, 171, 173, 217, + 222, 223, 224, 147, 184, 231, 232, 172, 227, 159, + 142, 167, 214, 220, 203, 0, 0, 235, 148, 137, + 158, 207, 138, 144, 145, 168, 169, 233, 205, 208, + 209, 139, 236, 188, 212, 160, 174, 218, 219, 221, + 226, 170, 230, 228, 229, 179, 183, 210, 211, 180, + 181, 204, 225, 161, 153, 154, 155, 156, 157, 237, + 238, 239, 189, 190, 198, 240, 241, 206, 182, 242, + 243, 244, 246, 245, 247, 248, 249, 250, 262, 264, + 263, 266, 267, 268, 269, 265, 285, 286, 287, 288, + 289, 290, 291, 292, 293, 294, 295, 296, 301, 215, + 234 }; /* YYDEFGOTO[NTERM-NUM]. */ static const yytype_int16 yydefgoto[] = { - -1, 1, 9, 10, 16, 126, 11, 17, 236, 12, - 18, 246, 127, 128, 129, 130, 131, 132, 133, 134, - 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, - 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, - 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, - 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, - 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, - 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, - 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, - 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, - 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, - 225, 226, 227, 228, 229, 230, 237, 238, 239, 240, - 241, 247, 248, 249, 250, 13, 19, 258, 259, 260, - 261, 262, 263, 264, 265, 14, 20, 278, 279, 280, - 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, - 15, 21, 292, 293 + -1, 1, 9, 10, 16, 129, 11, 17, 242, 12, + 18, 252, 130, 131, 132, 133, 134, 135, 136, 137, + 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, + 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, + 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, + 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, + 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, + 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, + 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, + 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, + 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, + 228, 229, 230, 231, 232, 233, 234, 235, 236, 243, + 244, 245, 246, 247, 253, 254, 255, 256, 13, 19, + 265, 266, 267, 268, 269, 270, 271, 272, 273, 14, + 20, 286, 287, 288, 289, 290, 291, 292, 293, 294, + 295, 296, 297, 298, 15, 21, 300, 301 }; /* YYPACT[STATE-NUM] -- Index in YYTABLE of the portion describing STATE-NUM. */ -#define YYPACT_NINF -123 +#define YYPACT_NINF -81 static const yytype_int16 yypact[] = { - -123, 0, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, 92, -36, -32, -62, - -122, -102, -4, -3, -2, -1, 2, 24, 25, 26, - 27, 29, 30, 31, 32, 33, 35, 36, 37, 38, - 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, - 49, 50, 51, 52, 54, 55, 56, 57, 58, 59, - 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, - 70, 71, 72, 73, 74, 75, 76, 77, 79, 80, - 81, 83, 84, 86, 87, 88, 89, 90, 91, 119, - 120, 121, 122, 127, 128, 170, 171, 172, 173, 174, - 175, 176, 177, 181, 185, 186, 209, 210, 218, 219, - 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, - 230, 231, 232, 233, 234, 235, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, 236, 237, 238, 239, 240, -123, -123, -123, -123, - -123, -123, 241, 242, 243, 244, -123, -123, -123, -123, - -123, 245, 246, 247, 248, 249, 250, 251, -123, -123, - -123, -123, -123, -123, -123, -123, 252, 253, 254, 255, - 256, 257, 258, 259, 260, 261, 262, 263, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, 264, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, 265, 266, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123 + -81, 116, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -12, 41, 47, 39, + 3, -80, 16, 17, 18, 22, 23, 24, 66, 67, + 69, 72, 73, 78, 107, 126, 127, 128, 145, 146, + 147, 148, 149, 151, 152, 153, 154, 155, 156, 157, + 158, 159, 161, 162, 163, 164, 166, 167, 168, 169, + 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, + 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, + 190, 191, 192, 193, 195, 196, 197, 198, 199, 200, + 201, 202, 203, 204, 205, 206, 207, 208, 209, 211, + 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, + 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, + 232, 233, 234, 235, 236, 237, 238, 239, 240, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, 241, 242, 243, + 245, 246, -81, -81, -81, -81, -81, -81, 247, 248, + 249, 250, -81, -81, -81, -81, -81, 251, 252, 253, + 254, 255, 256, 257, 258, -81, -81, -81, -81, -81, + -81, -81, -81, -81, 259, 260, 261, 262, 263, 264, + 265, 266, 267, 268, 269, 270, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, 271, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, 272, 273, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81 }; /* YYPGOTO[NTERM-NUM]. */ static const yytype_int8 yypgoto[] = { - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123, -123, -123, -123, -123, -123, -123, - -123, -123, -123, -123 + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81, -81, -81, + -81, -81, -81, -81, -81, -81, -81, -81 }; /* YYTABLE[YYPACT[STATE-NUM]]. What to do in state STATE-NUM. If @@ -1259,108 +1283,111 @@ static const yytype_int8 yypgoto[] = #define YYTABLE_NINF -1 static const yytype_uint16 yytable[] = { - 2, 231, 291, 232, 233, 242, 294, 295, 296, 297, - 0, 3, 298, 243, 244, 266, 267, 268, 269, 270, - 271, 272, 273, 274, 275, 276, 277, 251, 252, 253, - 254, 255, 256, 257, 299, 300, 301, 302, 4, 303, - 304, 305, 306, 307, 5, 308, 309, 310, 311, 312, - 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, - 323, 324, 325, 234, 326, 327, 328, 329, 330, 331, - 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, - 342, 343, 344, 345, 346, 347, 348, 349, 6, 350, - 351, 352, 235, 353, 354, 245, 355, 356, 357, 358, - 359, 360, 0, 7, 22, 23, 24, 25, 26, 27, - 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, - 38, 39, 40, 41, 42, 43, 44, 45, 46, 361, - 362, 363, 364, 47, 48, 49, 8, 365, 366, 50, - 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, - 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, - 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, - 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, - 367, 368, 369, 370, 371, 372, 373, 374, 91, 92, - 93, 375, 94, 95, 96, 376, 377, 97, 98, 99, - 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, - 110, 111, 112, 113, 114, 115, 116, 117, 118, 378, - 379, 119, 120, 121, 122, 123, 124, 125, 380, 381, - 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, - 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, - 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, - 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, - 422, 423, 424, 425, 426, 427, 428 + 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, + 42, 43, 44, 45, 46, 299, 302, 303, 304, 47, + 48, 49, 305, 306, 307, 50, 51, 52, 53, 54, + 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, + 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, + 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, + 85, 86, 87, 88, 89, 90, 308, 309, 237, 310, + 238, 239, 311, 312, 248, 91, 92, 93, 313, 94, + 95, 96, 249, 250, 97, 98, 99, 100, 101, 102, + 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + 113, 114, 115, 116, 117, 118, 2, 314, 119, 120, + 121, 122, 123, 124, 125, 126, 0, 3, 257, 258, + 259, 260, 261, 262, 263, 264, 315, 316, 317, 127, + 128, 240, 274, 275, 276, 277, 278, 279, 280, 281, + 282, 283, 284, 285, 4, 318, 319, 320, 321, 322, + 5, 323, 324, 325, 326, 327, 328, 329, 330, 331, + 241, 332, 333, 334, 335, 251, 336, 337, 338, 339, + 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, + 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, + 360, 361, 362, 363, 6, 364, 365, 366, 367, 368, + 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, + 7, 379, 380, 381, 382, 383, 384, 385, 386, 387, + 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, + 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, + 408, 409, 410, 411, 8, 412, 413, 414, 415, 416, + 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, + 427, 428, 429, 430, 431, 432, 433, 434, 435, 436, + 437, 438, 439, 440 }; #define yypact_value_is_default(Yystate) \ - (!!((Yystate) == (-123))) + (!!((Yystate) == (-81))) #define yytable_value_is_error(Yytable_value) \ YYID (0) static const yytype_int16 yycheck[] = { - 0, 37, 104, 39, 40, 37, 10, 10, 10, 10, - -1, 11, 10, 45, 46, 137, 138, 139, 140, 141, - 142, 143, 144, 145, 146, 147, 148, 89, 90, 91, - 92, 93, 94, 95, 10, 10, 10, 10, 38, 10, - 10, 10, 10, 10, 44, 10, 10, 10, 10, 10, + 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 105, 10, 10, 10, 41, + 42, 43, 10, 10, 10, 47, 48, 49, 50, 51, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, + 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, + 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, + 82, 83, 84, 85, 86, 87, 10, 10, 37, 10, + 39, 40, 10, 10, 37, 97, 98, 99, 10, 101, + 102, 103, 45, 46, 106, 107, 108, 109, 110, 111, + 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, + 122, 123, 124, 125, 126, 127, 0, 10, 130, 131, + 132, 133, 134, 135, 136, 137, -1, 11, 89, 90, + 91, 92, 93, 94, 95, 96, 10, 10, 10, 151, + 152, 100, 139, 140, 141, 142, 143, 144, 145, 146, + 147, 148, 149, 150, 38, 10, 10, 10, 10, 10, + 44, 10, 10, 10, 10, 10, 10, 10, 10, 10, + 129, 10, 10, 10, 10, 128, 10, 10, 10, 10, + 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, - 10, 10, 10, 99, 10, 10, 10, 10, 10, 10, + 10, 10, 10, 10, 88, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, - 10, 10, 10, 10, 10, 10, 10, 10, 88, 10, - 10, 10, 128, 10, 10, 127, 10, 10, 10, 10, - 10, 10, -1, 103, 12, 13, 14, 15, 16, 17, - 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, - 28, 29, 30, 31, 32, 33, 34, 35, 36, 10, - 10, 10, 10, 41, 42, 43, 136, 10, 10, 47, - 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, - 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, - 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, - 10, 10, 10, 10, 10, 10, 10, 10, 96, 97, - 98, 10, 100, 101, 102, 10, 10, 105, 106, 107, - 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, - 118, 119, 120, 121, 122, 123, 124, 125, 126, 10, - 10, 129, 130, 131, 132, 133, 134, 135, 10, 10, + 104, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, + 10, 10, 10, 10, 138, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, - 10, 10, 10, 10, 10, 10, 10 + 10, 10, 10, 10 }; /* YYSTOS[STATE-NUM] -- The (internal number of the) accessing symbol of state STATE-NUM. */ static const yytype_uint16 yystos[] = { - 0, 150, 0, 11, 38, 44, 88, 103, 136, 151, - 152, 155, 158, 274, 284, 299, 153, 156, 159, 275, - 285, 300, 12, 13, 14, 15, 16, 17, 18, 19, + 0, 154, 0, 11, 38, 44, 88, 104, 138, 155, + 156, 159, 162, 281, 292, 307, 157, 160, 163, 282, + 293, 308, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 41, 42, 43, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, - 87, 96, 97, 98, 100, 101, 102, 105, 106, 107, - 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, - 118, 119, 120, 121, 122, 123, 124, 125, 126, 129, - 130, 131, 132, 133, 134, 135, 154, 161, 162, 163, - 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, - 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, - 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, - 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, - 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, - 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, - 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, - 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, - 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, - 264, 37, 39, 40, 99, 128, 157, 265, 266, 267, - 268, 269, 37, 45, 46, 127, 160, 270, 271, 272, - 273, 89, 90, 91, 92, 93, 94, 95, 276, 277, - 278, 279, 280, 281, 282, 283, 137, 138, 139, 140, - 141, 142, 143, 144, 145, 146, 147, 148, 286, 287, - 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, - 298, 104, 301, 302, 10, 10, 10, 10, 10, 10, + 87, 97, 98, 99, 101, 102, 103, 106, 107, 108, + 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, + 119, 120, 121, 122, 123, 124, 125, 126, 127, 130, + 131, 132, 133, 134, 135, 136, 137, 151, 152, 158, + 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, + 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, + 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, + 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, + 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, + 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, + 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, + 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, + 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, + 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, + 265, 266, 267, 268, 269, 270, 271, 37, 39, 40, + 100, 129, 161, 272, 273, 274, 275, 276, 37, 45, + 46, 128, 164, 277, 278, 279, 280, 89, 90, 91, + 92, 93, 94, 95, 96, 283, 284, 285, 286, 287, + 288, 289, 290, 291, 139, 140, 141, 142, 143, 144, + 145, 146, 147, 148, 149, 150, 294, 295, 296, 297, + 298, 299, 300, 301, 302, 303, 304, 305, 306, 105, + 309, 310, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, @@ -1373,7 +1400,8 @@ static const yytype_uint16 yystos[] = 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, - 10, 10, 10, 10, 10, 10, 10, 10, 10 + 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, + 10 }; #define yyerrok (yyerrstatus = 0) @@ -2175,15 +2203,15 @@ yyreduce: { case 10: /* Line 1792 of yacc.c */ -#line 129 "./util/configparser.y" +#line 132 "./util/configparser.y" { OUTYY(("\nP(server:)\n")); } break; - case 117: + case 120: /* Line 1792 of yacc.c */ -#line 180 "./util/configparser.y" +#line 185 "./util/configparser.y" { struct config_stub* s; OUTYY(("\nP(stub_zone:)\n")); @@ -2196,9 +2224,9 @@ yyreduce: } break; - case 125: + case 128: /* Line 1792 of yacc.c */ -#line 196 "./util/configparser.y" +#line 201 "./util/configparser.y" { struct config_stub* s; OUTYY(("\nP(forward_zone:)\n")); @@ -2211,9 +2239,9 @@ yyreduce: } break; - case 132: + case 135: /* Line 1792 of yacc.c */ -#line 212 "./util/configparser.y" +#line 217 "./util/configparser.y" { OUTYY(("P(server_num_threads:%s)\n", (yyvsp[(2) - (2)].str))); if(atoi((yyvsp[(2) - (2)].str)) == 0 && strcmp((yyvsp[(2) - (2)].str), "0") != 0) @@ -2223,9 +2251,9 @@ yyreduce: } break; - case 133: + case 136: /* Line 1792 of yacc.c */ -#line 221 "./util/configparser.y" +#line 226 "./util/configparser.y" { OUTYY(("P(server_verbosity:%s)\n", (yyvsp[(2) - (2)].str))); if(atoi((yyvsp[(2) - (2)].str)) == 0 && strcmp((yyvsp[(2) - (2)].str), "0") != 0) @@ -2235,9 +2263,9 @@ yyreduce: } break; - case 134: + case 137: /* Line 1792 of yacc.c */ -#line 230 "./util/configparser.y" +#line 235 "./util/configparser.y" { OUTYY(("P(server_statistics_interval:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "") == 0 || strcmp((yyvsp[(2) - (2)].str), "0") == 0) @@ -2249,9 +2277,9 @@ yyreduce: } break; - case 135: + case 138: /* Line 1792 of yacc.c */ -#line 241 "./util/configparser.y" +#line 246 "./util/configparser.y" { OUTYY(("P(server_statistics_cumulative:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -2261,9 +2289,9 @@ yyreduce: } break; - case 136: + case 139: /* Line 1792 of yacc.c */ -#line 250 "./util/configparser.y" +#line 255 "./util/configparser.y" { OUTYY(("P(server_extended_statistics:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -2273,9 +2301,9 @@ yyreduce: } break; - case 137: + case 140: /* Line 1792 of yacc.c */ -#line 259 "./util/configparser.y" +#line 264 "./util/configparser.y" { OUTYY(("P(server_port:%s)\n", (yyvsp[(2) - (2)].str))); if(atoi((yyvsp[(2) - (2)].str)) == 0) @@ -2285,9 +2313,9 @@ yyreduce: } break; - case 138: + case 141: /* Line 1792 of yacc.c */ -#line 268 "./util/configparser.y" +#line 273 "./util/configparser.y" { OUTYY(("P(server_interface:%s)\n", (yyvsp[(2) - (2)].str))); if(cfg_parser->cfg->num_ifs == 0) @@ -2301,9 +2329,9 @@ yyreduce: } break; - case 139: + case 142: /* Line 1792 of yacc.c */ -#line 281 "./util/configparser.y" +#line 286 "./util/configparser.y" { OUTYY(("P(server_outgoing_interface:%s)\n", (yyvsp[(2) - (2)].str))); if(cfg_parser->cfg->num_out_ifs == 0) @@ -2319,9 +2347,9 @@ yyreduce: } break; - case 140: + case 143: /* Line 1792 of yacc.c */ -#line 296 "./util/configparser.y" +#line 301 "./util/configparser.y" { OUTYY(("P(server_outgoing_range:%s)\n", (yyvsp[(2) - (2)].str))); if(atoi((yyvsp[(2) - (2)].str)) == 0) @@ -2331,9 +2359,9 @@ yyreduce: } break; - case 141: + case 144: /* Line 1792 of yacc.c */ -#line 305 "./util/configparser.y" +#line 310 "./util/configparser.y" { OUTYY(("P(server_outgoing_port_permit:%s)\n", (yyvsp[(2) - (2)].str))); if(!cfg_mark_ports((yyvsp[(2) - (2)].str), 1, @@ -2343,9 +2371,9 @@ yyreduce: } break; - case 142: + case 145: /* Line 1792 of yacc.c */ -#line 314 "./util/configparser.y" +#line 319 "./util/configparser.y" { OUTYY(("P(server_outgoing_port_avoid:%s)\n", (yyvsp[(2) - (2)].str))); if(!cfg_mark_ports((yyvsp[(2) - (2)].str), 0, @@ -2355,9 +2383,9 @@ yyreduce: } break; - case 143: + case 146: /* Line 1792 of yacc.c */ -#line 323 "./util/configparser.y" +#line 328 "./util/configparser.y" { OUTYY(("P(server_outgoing_num_tcp:%s)\n", (yyvsp[(2) - (2)].str))); if(atoi((yyvsp[(2) - (2)].str)) == 0 && strcmp((yyvsp[(2) - (2)].str), "0") != 0) @@ -2367,9 +2395,9 @@ yyreduce: } break; - case 144: + case 147: /* Line 1792 of yacc.c */ -#line 332 "./util/configparser.y" +#line 337 "./util/configparser.y" { OUTYY(("P(server_incoming_num_tcp:%s)\n", (yyvsp[(2) - (2)].str))); if(atoi((yyvsp[(2) - (2)].str)) == 0 && strcmp((yyvsp[(2) - (2)].str), "0") != 0) @@ -2379,9 +2407,9 @@ yyreduce: } break; - case 145: + case 148: /* Line 1792 of yacc.c */ -#line 341 "./util/configparser.y" +#line 346 "./util/configparser.y" { OUTYY(("P(server_interface_automatic:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -2391,9 +2419,9 @@ yyreduce: } break; - case 146: + case 149: /* Line 1792 of yacc.c */ -#line 350 "./util/configparser.y" +#line 355 "./util/configparser.y" { OUTYY(("P(server_do_ip4:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -2403,9 +2431,9 @@ yyreduce: } break; - case 147: + case 150: /* Line 1792 of yacc.c */ -#line 359 "./util/configparser.y" +#line 364 "./util/configparser.y" { OUTYY(("P(server_do_ip6:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -2415,9 +2443,9 @@ yyreduce: } break; - case 148: + case 151: /* Line 1792 of yacc.c */ -#line 368 "./util/configparser.y" +#line 373 "./util/configparser.y" { OUTYY(("P(server_do_udp:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -2427,9 +2455,9 @@ yyreduce: } break; - case 149: + case 152: /* Line 1792 of yacc.c */ -#line 377 "./util/configparser.y" +#line 382 "./util/configparser.y" { OUTYY(("P(server_do_tcp:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -2439,9 +2467,9 @@ yyreduce: } break; - case 150: + case 153: /* Line 1792 of yacc.c */ -#line 386 "./util/configparser.y" +#line 391 "./util/configparser.y" { OUTYY(("P(server_tcp_upstream:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -2451,9 +2479,9 @@ yyreduce: } break; - case 151: + case 154: /* Line 1792 of yacc.c */ -#line 395 "./util/configparser.y" +#line 400 "./util/configparser.y" { OUTYY(("P(server_ssl_upstream:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -2463,9 +2491,9 @@ yyreduce: } break; - case 152: + case 155: /* Line 1792 of yacc.c */ -#line 404 "./util/configparser.y" +#line 409 "./util/configparser.y" { OUTYY(("P(server_ssl_service_key:%s)\n", (yyvsp[(2) - (2)].str))); free(cfg_parser->cfg->ssl_service_key); @@ -2473,9 +2501,9 @@ yyreduce: } break; - case 153: + case 156: /* Line 1792 of yacc.c */ -#line 411 "./util/configparser.y" +#line 416 "./util/configparser.y" { OUTYY(("P(server_ssl_service_pem:%s)\n", (yyvsp[(2) - (2)].str))); free(cfg_parser->cfg->ssl_service_pem); @@ -2483,9 +2511,9 @@ yyreduce: } break; - case 154: + case 157: /* Line 1792 of yacc.c */ -#line 418 "./util/configparser.y" +#line 423 "./util/configparser.y" { OUTYY(("P(server_ssl_port:%s)\n", (yyvsp[(2) - (2)].str))); if(atoi((yyvsp[(2) - (2)].str)) == 0) @@ -2495,9 +2523,9 @@ yyreduce: } break; - case 155: + case 158: /* Line 1792 of yacc.c */ -#line 427 "./util/configparser.y" +#line 432 "./util/configparser.y" { OUTYY(("P(server_do_daemonize:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -2507,9 +2535,9 @@ yyreduce: } break; - case 156: + case 159: /* Line 1792 of yacc.c */ -#line 436 "./util/configparser.y" +#line 441 "./util/configparser.y" { OUTYY(("P(server_use_syslog:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -2524,9 +2552,9 @@ yyreduce: } break; - case 157: + case 160: /* Line 1792 of yacc.c */ -#line 450 "./util/configparser.y" +#line 455 "./util/configparser.y" { OUTYY(("P(server_log_time_ascii:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -2536,9 +2564,9 @@ yyreduce: } break; - case 158: + case 161: /* Line 1792 of yacc.c */ -#line 459 "./util/configparser.y" +#line 464 "./util/configparser.y" { OUTYY(("P(server_log_queries:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -2548,9 +2576,9 @@ yyreduce: } break; - case 159: + case 162: /* Line 1792 of yacc.c */ -#line 468 "./util/configparser.y" +#line 473 "./util/configparser.y" { OUTYY(("P(server_chroot:%s)\n", (yyvsp[(2) - (2)].str))); free(cfg_parser->cfg->chrootdir); @@ -2558,9 +2586,9 @@ yyreduce: } break; - case 160: + case 163: /* Line 1792 of yacc.c */ -#line 475 "./util/configparser.y" +#line 480 "./util/configparser.y" { OUTYY(("P(server_username:%s)\n", (yyvsp[(2) - (2)].str))); free(cfg_parser->cfg->username); @@ -2568,9 +2596,9 @@ yyreduce: } break; - case 161: + case 164: /* Line 1792 of yacc.c */ -#line 482 "./util/configparser.y" +#line 487 "./util/configparser.y" { OUTYY(("P(server_directory:%s)\n", (yyvsp[(2) - (2)].str))); free(cfg_parser->cfg->directory); @@ -2578,9 +2606,9 @@ yyreduce: } break; - case 162: + case 165: /* Line 1792 of yacc.c */ -#line 489 "./util/configparser.y" +#line 494 "./util/configparser.y" { OUTYY(("P(server_logfile:%s)\n", (yyvsp[(2) - (2)].str))); free(cfg_parser->cfg->logfile); @@ -2589,9 +2617,9 @@ yyreduce: } break; - case 163: + case 166: /* Line 1792 of yacc.c */ -#line 497 "./util/configparser.y" +#line 502 "./util/configparser.y" { OUTYY(("P(server_pidfile:%s)\n", (yyvsp[(2) - (2)].str))); free(cfg_parser->cfg->pidfile); @@ -2599,9 +2627,9 @@ yyreduce: } break; - case 164: + case 167: /* Line 1792 of yacc.c */ -#line 504 "./util/configparser.y" +#line 509 "./util/configparser.y" { OUTYY(("P(server_root_hints:%s)\n", (yyvsp[(2) - (2)].str))); if(!cfg_strlist_insert(&cfg_parser->cfg->root_hints, (yyvsp[(2) - (2)].str))) @@ -2609,9 +2637,9 @@ yyreduce: } break; - case 165: + case 168: /* Line 1792 of yacc.c */ -#line 511 "./util/configparser.y" +#line 516 "./util/configparser.y" { OUTYY(("P(server_dlv_anchor_file:%s)\n", (yyvsp[(2) - (2)].str))); free(cfg_parser->cfg->dlv_anchor_file); @@ -2619,9 +2647,9 @@ yyreduce: } break; - case 166: + case 169: /* Line 1792 of yacc.c */ -#line 518 "./util/configparser.y" +#line 523 "./util/configparser.y" { OUTYY(("P(server_dlv_anchor:%s)\n", (yyvsp[(2) - (2)].str))); if(!cfg_strlist_insert(&cfg_parser->cfg->dlv_anchor_list, (yyvsp[(2) - (2)].str))) @@ -2629,9 +2657,9 @@ yyreduce: } break; - case 167: + case 170: /* Line 1792 of yacc.c */ -#line 525 "./util/configparser.y" +#line 530 "./util/configparser.y" { OUTYY(("P(server_auto_trust_anchor_file:%s)\n", (yyvsp[(2) - (2)].str))); if(!cfg_strlist_insert(&cfg_parser->cfg-> @@ -2640,9 +2668,9 @@ yyreduce: } break; - case 168: + case 171: /* Line 1792 of yacc.c */ -#line 533 "./util/configparser.y" +#line 538 "./util/configparser.y" { OUTYY(("P(server_trust_anchor_file:%s)\n", (yyvsp[(2) - (2)].str))); if(!cfg_strlist_insert(&cfg_parser->cfg-> @@ -2651,9 +2679,9 @@ yyreduce: } break; - case 169: + case 172: /* Line 1792 of yacc.c */ -#line 541 "./util/configparser.y" +#line 546 "./util/configparser.y" { OUTYY(("P(server_trusted_keys_file:%s)\n", (yyvsp[(2) - (2)].str))); if(!cfg_strlist_insert(&cfg_parser->cfg-> @@ -2662,9 +2690,9 @@ yyreduce: } break; - case 170: + case 173: /* Line 1792 of yacc.c */ -#line 549 "./util/configparser.y" +#line 554 "./util/configparser.y" { OUTYY(("P(server_trust_anchor:%s)\n", (yyvsp[(2) - (2)].str))); if(!cfg_strlist_insert(&cfg_parser->cfg->trust_anchor_list, (yyvsp[(2) - (2)].str))) @@ -2672,9 +2700,9 @@ yyreduce: } break; - case 171: + case 174: /* Line 1792 of yacc.c */ -#line 556 "./util/configparser.y" +#line 561 "./util/configparser.y" { OUTYY(("P(server_domain_insecure:%s)\n", (yyvsp[(2) - (2)].str))); if(!cfg_strlist_insert(&cfg_parser->cfg->domain_insecure, (yyvsp[(2) - (2)].str))) @@ -2682,9 +2710,9 @@ yyreduce: } break; - case 172: + case 175: /* Line 1792 of yacc.c */ -#line 563 "./util/configparser.y" +#line 568 "./util/configparser.y" { OUTYY(("P(server_hide_identity:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -2694,9 +2722,9 @@ yyreduce: } break; - case 173: + case 176: /* Line 1792 of yacc.c */ -#line 572 "./util/configparser.y" +#line 577 "./util/configparser.y" { OUTYY(("P(server_hide_version:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -2706,9 +2734,9 @@ yyreduce: } break; - case 174: + case 177: /* Line 1792 of yacc.c */ -#line 581 "./util/configparser.y" +#line 586 "./util/configparser.y" { OUTYY(("P(server_identity:%s)\n", (yyvsp[(2) - (2)].str))); free(cfg_parser->cfg->identity); @@ -2716,9 +2744,9 @@ yyreduce: } break; - case 175: + case 178: /* Line 1792 of yacc.c */ -#line 588 "./util/configparser.y" +#line 593 "./util/configparser.y" { OUTYY(("P(server_version:%s)\n", (yyvsp[(2) - (2)].str))); free(cfg_parser->cfg->version); @@ -2726,9 +2754,9 @@ yyreduce: } break; - case 176: + case 179: /* Line 1792 of yacc.c */ -#line 595 "./util/configparser.y" +#line 600 "./util/configparser.y" { OUTYY(("P(server_so_rcvbuf:%s)\n", (yyvsp[(2) - (2)].str))); if(!cfg_parse_memsize((yyvsp[(2) - (2)].str), &cfg_parser->cfg->so_rcvbuf)) @@ -2737,9 +2765,9 @@ yyreduce: } break; - case 177: + case 180: /* Line 1792 of yacc.c */ -#line 603 "./util/configparser.y" +#line 608 "./util/configparser.y" { OUTYY(("P(server_so_sndbuf:%s)\n", (yyvsp[(2) - (2)].str))); if(!cfg_parse_memsize((yyvsp[(2) - (2)].str), &cfg_parser->cfg->so_sndbuf)) @@ -2748,9 +2776,9 @@ yyreduce: } break; - case 178: + case 181: /* Line 1792 of yacc.c */ -#line 611 "./util/configparser.y" +#line 616 "./util/configparser.y" { OUTYY(("P(server_so_reuseport:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -2761,9 +2789,22 @@ yyreduce: } break; - case 179: + case 182: /* Line 1792 of yacc.c */ -#line 621 "./util/configparser.y" +#line 626 "./util/configparser.y" + { + OUTYY(("P(server_ip_transparent:%s)\n", (yyvsp[(2) - (2)].str))); + if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) + yyerror("expected yes or no."); + else cfg_parser->cfg->ip_transparent = + (strcmp((yyvsp[(2) - (2)].str), "yes")==0); + free((yyvsp[(2) - (2)].str)); + } + break; + + case 183: +/* Line 1792 of yacc.c */ +#line 636 "./util/configparser.y" { OUTYY(("P(server_edns_buffer_size:%s)\n", (yyvsp[(2) - (2)].str))); if(atoi((yyvsp[(2) - (2)].str)) == 0) @@ -2777,9 +2818,9 @@ yyreduce: } break; - case 180: + case 184: /* Line 1792 of yacc.c */ -#line 634 "./util/configparser.y" +#line 649 "./util/configparser.y" { OUTYY(("P(server_msg_buffer_size:%s)\n", (yyvsp[(2) - (2)].str))); if(atoi((yyvsp[(2) - (2)].str)) == 0) @@ -2791,9 +2832,9 @@ yyreduce: } break; - case 181: + case 185: /* Line 1792 of yacc.c */ -#line 645 "./util/configparser.y" +#line 660 "./util/configparser.y" { OUTYY(("P(server_msg_cache_size:%s)\n", (yyvsp[(2) - (2)].str))); if(!cfg_parse_memsize((yyvsp[(2) - (2)].str), &cfg_parser->cfg->msg_cache_size)) @@ -2802,9 +2843,9 @@ yyreduce: } break; - case 182: + case 186: /* Line 1792 of yacc.c */ -#line 653 "./util/configparser.y" +#line 668 "./util/configparser.y" { OUTYY(("P(server_msg_cache_slabs:%s)\n", (yyvsp[(2) - (2)].str))); if(atoi((yyvsp[(2) - (2)].str)) == 0) @@ -2818,9 +2859,9 @@ yyreduce: } break; - case 183: + case 187: /* Line 1792 of yacc.c */ -#line 666 "./util/configparser.y" +#line 681 "./util/configparser.y" { OUTYY(("P(server_num_queries_per_thread:%s)\n", (yyvsp[(2) - (2)].str))); if(atoi((yyvsp[(2) - (2)].str)) == 0) @@ -2830,9 +2871,9 @@ yyreduce: } break; - case 184: + case 188: /* Line 1792 of yacc.c */ -#line 675 "./util/configparser.y" +#line 690 "./util/configparser.y" { OUTYY(("P(server_jostle_timeout:%s)\n", (yyvsp[(2) - (2)].str))); if(atoi((yyvsp[(2) - (2)].str)) == 0 && strcmp((yyvsp[(2) - (2)].str), "0") != 0) @@ -2842,9 +2883,9 @@ yyreduce: } break; - case 185: + case 189: /* Line 1792 of yacc.c */ -#line 684 "./util/configparser.y" +#line 699 "./util/configparser.y" { OUTYY(("P(server_delay_close:%s)\n", (yyvsp[(2) - (2)].str))); if(atoi((yyvsp[(2) - (2)].str)) == 0 && strcmp((yyvsp[(2) - (2)].str), "0") != 0) @@ -2854,9 +2895,9 @@ yyreduce: } break; - case 186: + case 190: /* Line 1792 of yacc.c */ -#line 693 "./util/configparser.y" +#line 708 "./util/configparser.y" { OUTYY(("P(server_unblock_lan_zones:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -2867,9 +2908,9 @@ yyreduce: } break; - case 187: + case 191: /* Line 1792 of yacc.c */ -#line 703 "./util/configparser.y" +#line 718 "./util/configparser.y" { OUTYY(("P(server_rrset_cache_size:%s)\n", (yyvsp[(2) - (2)].str))); if(!cfg_parse_memsize((yyvsp[(2) - (2)].str), &cfg_parser->cfg->rrset_cache_size)) @@ -2878,9 +2919,9 @@ yyreduce: } break; - case 188: + case 192: /* Line 1792 of yacc.c */ -#line 711 "./util/configparser.y" +#line 726 "./util/configparser.y" { OUTYY(("P(server_rrset_cache_slabs:%s)\n", (yyvsp[(2) - (2)].str))); if(atoi((yyvsp[(2) - (2)].str)) == 0) @@ -2894,9 +2935,9 @@ yyreduce: } break; - case 189: + case 193: /* Line 1792 of yacc.c */ -#line 724 "./util/configparser.y" +#line 739 "./util/configparser.y" { OUTYY(("P(server_infra_host_ttl:%s)\n", (yyvsp[(2) - (2)].str))); if(atoi((yyvsp[(2) - (2)].str)) == 0 && strcmp((yyvsp[(2) - (2)].str), "0") != 0) @@ -2906,9 +2947,9 @@ yyreduce: } break; - case 190: + case 194: /* Line 1792 of yacc.c */ -#line 733 "./util/configparser.y" +#line 748 "./util/configparser.y" { OUTYY(("P(server_infra_lame_ttl:%s)\n", (yyvsp[(2) - (2)].str))); verbose(VERB_DETAIL, "ignored infra-lame-ttl: %s (option " @@ -2917,9 +2958,9 @@ yyreduce: } break; - case 191: + case 195: /* Line 1792 of yacc.c */ -#line 741 "./util/configparser.y" +#line 756 "./util/configparser.y" { OUTYY(("P(server_infra_cache_numhosts:%s)\n", (yyvsp[(2) - (2)].str))); if(atoi((yyvsp[(2) - (2)].str)) == 0) @@ -2929,9 +2970,9 @@ yyreduce: } break; - case 192: + case 196: /* Line 1792 of yacc.c */ -#line 750 "./util/configparser.y" +#line 765 "./util/configparser.y" { OUTYY(("P(server_infra_cache_lame_size:%s)\n", (yyvsp[(2) - (2)].str))); verbose(VERB_DETAIL, "ignored infra-cache-lame-size: %s " @@ -2940,9 +2981,9 @@ yyreduce: } break; - case 193: + case 197: /* Line 1792 of yacc.c */ -#line 758 "./util/configparser.y" +#line 773 "./util/configparser.y" { OUTYY(("P(server_infra_cache_slabs:%s)\n", (yyvsp[(2) - (2)].str))); if(atoi((yyvsp[(2) - (2)].str)) == 0) @@ -2956,9 +2997,21 @@ yyreduce: } break; - case 194: + case 198: +/* Line 1792 of yacc.c */ +#line 786 "./util/configparser.y" + { + OUTYY(("P(server_infra_cache_min_rtt:%s)\n", (yyvsp[(2) - (2)].str))); + if(atoi((yyvsp[(2) - (2)].str)) == 0 && strcmp((yyvsp[(2) - (2)].str), "0") != 0) + yyerror("number expected"); + else cfg_parser->cfg->infra_cache_min_rtt = atoi((yyvsp[(2) - (2)].str)); + free((yyvsp[(2) - (2)].str)); + } + break; + + case 199: /* Line 1792 of yacc.c */ -#line 771 "./util/configparser.y" +#line 795 "./util/configparser.y" { OUTYY(("P(server_target_fetch_policy:%s)\n", (yyvsp[(2) - (2)].str))); free(cfg_parser->cfg->target_fetch_policy); @@ -2966,9 +3019,9 @@ yyreduce: } break; - case 195: + case 200: /* Line 1792 of yacc.c */ -#line 778 "./util/configparser.y" +#line 802 "./util/configparser.y" { OUTYY(("P(server_harden_short_bufsize:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -2979,9 +3032,9 @@ yyreduce: } break; - case 196: + case 201: /* Line 1792 of yacc.c */ -#line 788 "./util/configparser.y" +#line 812 "./util/configparser.y" { OUTYY(("P(server_harden_large_queries:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -2992,9 +3045,9 @@ yyreduce: } break; - case 197: + case 202: /* Line 1792 of yacc.c */ -#line 798 "./util/configparser.y" +#line 822 "./util/configparser.y" { OUTYY(("P(server_harden_glue:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -3005,9 +3058,9 @@ yyreduce: } break; - case 198: + case 203: /* Line 1792 of yacc.c */ -#line 808 "./util/configparser.y" +#line 832 "./util/configparser.y" { OUTYY(("P(server_harden_dnssec_stripped:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -3018,9 +3071,9 @@ yyreduce: } break; - case 199: + case 204: /* Line 1792 of yacc.c */ -#line 818 "./util/configparser.y" +#line 842 "./util/configparser.y" { OUTYY(("P(server_harden_below_nxdomain:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -3031,9 +3084,9 @@ yyreduce: } break; - case 200: + case 205: /* Line 1792 of yacc.c */ -#line 828 "./util/configparser.y" +#line 852 "./util/configparser.y" { OUTYY(("P(server_harden_referral_path:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -3044,9 +3097,22 @@ yyreduce: } break; - case 201: + case 206: /* Line 1792 of yacc.c */ -#line 838 "./util/configparser.y" +#line 862 "./util/configparser.y" + { + OUTYY(("P(server_harden_algo_downgrade:%s)\n", (yyvsp[(2) - (2)].str))); + if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) + yyerror("expected yes or no."); + else cfg_parser->cfg->harden_algo_downgrade = + (strcmp((yyvsp[(2) - (2)].str), "yes")==0); + free((yyvsp[(2) - (2)].str)); + } + break; + + case 207: +/* Line 1792 of yacc.c */ +#line 872 "./util/configparser.y" { OUTYY(("P(server_use_caps_for_id:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -3057,9 +3123,9 @@ yyreduce: } break; - case 202: + case 208: /* Line 1792 of yacc.c */ -#line 848 "./util/configparser.y" +#line 882 "./util/configparser.y" { OUTYY(("P(server_private_address:%s)\n", (yyvsp[(2) - (2)].str))); if(!cfg_strlist_insert(&cfg_parser->cfg->private_address, (yyvsp[(2) - (2)].str))) @@ -3067,9 +3133,9 @@ yyreduce: } break; - case 203: + case 209: /* Line 1792 of yacc.c */ -#line 855 "./util/configparser.y" +#line 889 "./util/configparser.y" { OUTYY(("P(server_private_domain:%s)\n", (yyvsp[(2) - (2)].str))); if(!cfg_strlist_insert(&cfg_parser->cfg->private_domain, (yyvsp[(2) - (2)].str))) @@ -3077,9 +3143,9 @@ yyreduce: } break; - case 204: + case 210: /* Line 1792 of yacc.c */ -#line 862 "./util/configparser.y" +#line 896 "./util/configparser.y" { OUTYY(("P(server_prefetch:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -3089,9 +3155,9 @@ yyreduce: } break; - case 205: + case 211: /* Line 1792 of yacc.c */ -#line 871 "./util/configparser.y" +#line 905 "./util/configparser.y" { OUTYY(("P(server_prefetch_key:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -3101,9 +3167,9 @@ yyreduce: } break; - case 206: + case 212: /* Line 1792 of yacc.c */ -#line 880 "./util/configparser.y" +#line 914 "./util/configparser.y" { OUTYY(("P(server_unwanted_reply_threshold:%s)\n", (yyvsp[(2) - (2)].str))); if(atoi((yyvsp[(2) - (2)].str)) == 0 && strcmp((yyvsp[(2) - (2)].str), "0") != 0) @@ -3113,9 +3179,9 @@ yyreduce: } break; - case 207: + case 213: /* Line 1792 of yacc.c */ -#line 889 "./util/configparser.y" +#line 923 "./util/configparser.y" { OUTYY(("P(server_do_not_query_address:%s)\n", (yyvsp[(2) - (2)].str))); if(!cfg_strlist_insert(&cfg_parser->cfg->donotqueryaddrs, (yyvsp[(2) - (2)].str))) @@ -3123,9 +3189,9 @@ yyreduce: } break; - case 208: + case 214: /* Line 1792 of yacc.c */ -#line 896 "./util/configparser.y" +#line 930 "./util/configparser.y" { OUTYY(("P(server_do_not_query_localhost:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -3136,9 +3202,9 @@ yyreduce: } break; - case 209: + case 215: /* Line 1792 of yacc.c */ -#line 906 "./util/configparser.y" +#line 940 "./util/configparser.y" { OUTYY(("P(server_access_control:%s %s)\n", (yyvsp[(2) - (3)].str), (yyvsp[(3) - (3)].str))); if(strcmp((yyvsp[(3) - (3)].str), "deny")!=0 && strcmp((yyvsp[(3) - (3)].str), "refuse")!=0 && @@ -3156,9 +3222,9 @@ yyreduce: } break; - case 210: + case 216: /* Line 1792 of yacc.c */ -#line 923 "./util/configparser.y" +#line 957 "./util/configparser.y" { OUTYY(("P(server_module_conf:%s)\n", (yyvsp[(2) - (2)].str))); free(cfg_parser->cfg->module_conf); @@ -3166,9 +3232,9 @@ yyreduce: } break; - case 211: + case 217: /* Line 1792 of yacc.c */ -#line 930 "./util/configparser.y" +#line 964 "./util/configparser.y" { OUTYY(("P(server_val_override_date:%s)\n", (yyvsp[(2) - (2)].str))); if(strlen((yyvsp[(2) - (2)].str)) == 0 || strcmp((yyvsp[(2) - (2)].str), "0") == 0) { @@ -3187,9 +3253,9 @@ yyreduce: } break; - case 212: + case 218: /* Line 1792 of yacc.c */ -#line 948 "./util/configparser.y" +#line 982 "./util/configparser.y" { OUTYY(("P(server_val_sig_skew_min:%s)\n", (yyvsp[(2) - (2)].str))); if(strlen((yyvsp[(2) - (2)].str)) == 0 || strcmp((yyvsp[(2) - (2)].str), "0") == 0) { @@ -3203,9 +3269,9 @@ yyreduce: } break; - case 213: + case 219: /* Line 1792 of yacc.c */ -#line 961 "./util/configparser.y" +#line 995 "./util/configparser.y" { OUTYY(("P(server_val_sig_skew_max:%s)\n", (yyvsp[(2) - (2)].str))); if(strlen((yyvsp[(2) - (2)].str)) == 0 || strcmp((yyvsp[(2) - (2)].str), "0") == 0) { @@ -3219,9 +3285,9 @@ yyreduce: } break; - case 214: + case 220: /* Line 1792 of yacc.c */ -#line 974 "./util/configparser.y" +#line 1008 "./util/configparser.y" { OUTYY(("P(server_cache_max_ttl:%s)\n", (yyvsp[(2) - (2)].str))); if(atoi((yyvsp[(2) - (2)].str)) == 0 && strcmp((yyvsp[(2) - (2)].str), "0") != 0) @@ -3231,9 +3297,9 @@ yyreduce: } break; - case 215: + case 221: /* Line 1792 of yacc.c */ -#line 983 "./util/configparser.y" +#line 1017 "./util/configparser.y" { OUTYY(("P(server_cache_min_ttl:%s)\n", (yyvsp[(2) - (2)].str))); if(atoi((yyvsp[(2) - (2)].str)) == 0 && strcmp((yyvsp[(2) - (2)].str), "0") != 0) @@ -3243,9 +3309,9 @@ yyreduce: } break; - case 216: + case 222: /* Line 1792 of yacc.c */ -#line 992 "./util/configparser.y" +#line 1026 "./util/configparser.y" { OUTYY(("P(server_bogus_ttl:%s)\n", (yyvsp[(2) - (2)].str))); if(atoi((yyvsp[(2) - (2)].str)) == 0 && strcmp((yyvsp[(2) - (2)].str), "0") != 0) @@ -3255,9 +3321,9 @@ yyreduce: } break; - case 217: + case 223: /* Line 1792 of yacc.c */ -#line 1001 "./util/configparser.y" +#line 1035 "./util/configparser.y" { OUTYY(("P(server_val_clean_additional:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -3268,9 +3334,9 @@ yyreduce: } break; - case 218: + case 224: /* Line 1792 of yacc.c */ -#line 1011 "./util/configparser.y" +#line 1045 "./util/configparser.y" { OUTYY(("P(server_val_permissive_mode:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -3281,9 +3347,9 @@ yyreduce: } break; - case 219: + case 225: /* Line 1792 of yacc.c */ -#line 1021 "./util/configparser.y" +#line 1055 "./util/configparser.y" { OUTYY(("P(server_ignore_cd_flag:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -3293,9 +3359,9 @@ yyreduce: } break; - case 220: + case 226: /* Line 1792 of yacc.c */ -#line 1030 "./util/configparser.y" +#line 1064 "./util/configparser.y" { OUTYY(("P(server_val_log_level:%s)\n", (yyvsp[(2) - (2)].str))); if(atoi((yyvsp[(2) - (2)].str)) == 0 && strcmp((yyvsp[(2) - (2)].str), "0") != 0) @@ -3305,9 +3371,9 @@ yyreduce: } break; - case 221: + case 227: /* Line 1792 of yacc.c */ -#line 1039 "./util/configparser.y" +#line 1073 "./util/configparser.y" { OUTYY(("P(server_val_nsec3_keysize_iterations:%s)\n", (yyvsp[(2) - (2)].str))); free(cfg_parser->cfg->val_nsec3_key_iterations); @@ -3315,9 +3381,9 @@ yyreduce: } break; - case 222: + case 228: /* Line 1792 of yacc.c */ -#line 1046 "./util/configparser.y" +#line 1080 "./util/configparser.y" { OUTYY(("P(server_add_holddown:%s)\n", (yyvsp[(2) - (2)].str))); if(atoi((yyvsp[(2) - (2)].str)) == 0 && strcmp((yyvsp[(2) - (2)].str), "0") != 0) @@ -3327,9 +3393,9 @@ yyreduce: } break; - case 223: + case 229: /* Line 1792 of yacc.c */ -#line 1055 "./util/configparser.y" +#line 1089 "./util/configparser.y" { OUTYY(("P(server_del_holddown:%s)\n", (yyvsp[(2) - (2)].str))); if(atoi((yyvsp[(2) - (2)].str)) == 0 && strcmp((yyvsp[(2) - (2)].str), "0") != 0) @@ -3339,9 +3405,9 @@ yyreduce: } break; - case 224: + case 230: /* Line 1792 of yacc.c */ -#line 1064 "./util/configparser.y" +#line 1098 "./util/configparser.y" { OUTYY(("P(server_keep_missing:%s)\n", (yyvsp[(2) - (2)].str))); if(atoi((yyvsp[(2) - (2)].str)) == 0 && strcmp((yyvsp[(2) - (2)].str), "0") != 0) @@ -3351,9 +3417,9 @@ yyreduce: } break; - case 225: + case 231: /* Line 1792 of yacc.c */ -#line 1073 "./util/configparser.y" +#line 1107 "./util/configparser.y" { OUTYY(("P(server_key_cache_size:%s)\n", (yyvsp[(2) - (2)].str))); if(!cfg_parse_memsize((yyvsp[(2) - (2)].str), &cfg_parser->cfg->key_cache_size)) @@ -3362,9 +3428,9 @@ yyreduce: } break; - case 226: + case 232: /* Line 1792 of yacc.c */ -#line 1081 "./util/configparser.y" +#line 1115 "./util/configparser.y" { OUTYY(("P(server_key_cache_slabs:%s)\n", (yyvsp[(2) - (2)].str))); if(atoi((yyvsp[(2) - (2)].str)) == 0) @@ -3378,9 +3444,9 @@ yyreduce: } break; - case 227: + case 233: /* Line 1792 of yacc.c */ -#line 1094 "./util/configparser.y" +#line 1128 "./util/configparser.y" { OUTYY(("P(server_neg_cache_size:%s)\n", (yyvsp[(2) - (2)].str))); if(!cfg_parse_memsize((yyvsp[(2) - (2)].str), &cfg_parser->cfg->neg_cache_size)) @@ -3389,18 +3455,19 @@ yyreduce: } break; - case 228: + case 234: /* Line 1792 of yacc.c */ -#line 1102 "./util/configparser.y" +#line 1136 "./util/configparser.y" { OUTYY(("P(server_local_zone:%s %s)\n", (yyvsp[(2) - (3)].str), (yyvsp[(3) - (3)].str))); if(strcmp((yyvsp[(3) - (3)].str), "static")!=0 && strcmp((yyvsp[(3) - (3)].str), "deny")!=0 && strcmp((yyvsp[(3) - (3)].str), "refuse")!=0 && strcmp((yyvsp[(3) - (3)].str), "redirect")!=0 && strcmp((yyvsp[(3) - (3)].str), "transparent")!=0 && strcmp((yyvsp[(3) - (3)].str), "nodefault")!=0 - && strcmp((yyvsp[(3) - (3)].str), "typetransparent")!=0) + && strcmp((yyvsp[(3) - (3)].str), "typetransparent")!=0 && + strcmp((yyvsp[(3) - (3)].str), "inform")!=0) yyerror("local-zone type: expected static, deny, " "refuse, redirect, transparent, " - "typetransparent or nodefault"); + "typetransparent, inform or nodefault"); else if(strcmp((yyvsp[(3) - (3)].str), "nodefault")==0) { if(!cfg_strlist_insert(&cfg_parser->cfg-> local_zones_nodefault, (yyvsp[(2) - (3)].str))) @@ -3414,9 +3481,9 @@ yyreduce: } break; - case 229: + case 235: /* Line 1792 of yacc.c */ -#line 1124 "./util/configparser.y" +#line 1159 "./util/configparser.y" { OUTYY(("P(server_local_data:%s)\n", (yyvsp[(2) - (2)].str))); if(!cfg_strlist_insert(&cfg_parser->cfg->local_data, (yyvsp[(2) - (2)].str))) @@ -3424,9 +3491,9 @@ yyreduce: } break; - case 230: + case 236: /* Line 1792 of yacc.c */ -#line 1131 "./util/configparser.y" +#line 1166 "./util/configparser.y" { char* ptr; OUTYY(("P(server_local_data_ptr:%s)\n", (yyvsp[(2) - (2)].str))); @@ -3442,9 +3509,9 @@ yyreduce: } break; - case 231: + case 237: /* Line 1792 of yacc.c */ -#line 1146 "./util/configparser.y" +#line 1181 "./util/configparser.y" { OUTYY(("P(server_minimal_responses:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -3455,9 +3522,9 @@ yyreduce: } break; - case 232: + case 238: /* Line 1792 of yacc.c */ -#line 1156 "./util/configparser.y" +#line 1191 "./util/configparser.y" { OUTYY(("P(server_rrset_roundrobin:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -3468,9 +3535,9 @@ yyreduce: } break; - case 233: + case 239: /* Line 1792 of yacc.c */ -#line 1166 "./util/configparser.y" +#line 1201 "./util/configparser.y" { OUTYY(("P(server_max_udp_size:%s)\n", (yyvsp[(2) - (2)].str))); cfg_parser->cfg->max_udp_size = atoi((yyvsp[(2) - (2)].str)); @@ -3478,9 +3545,9 @@ yyreduce: } break; - case 234: + case 240: /* Line 1792 of yacc.c */ -#line 1173 "./util/configparser.y" +#line 1208 "./util/configparser.y" { OUTYY(("P(dns64_prefix:%s)\n", (yyvsp[(2) - (2)].str))); free(cfg_parser->cfg->dns64_prefix); @@ -3488,9 +3555,9 @@ yyreduce: } break; - case 235: + case 241: /* Line 1792 of yacc.c */ -#line 1180 "./util/configparser.y" +#line 1215 "./util/configparser.y" { OUTYY(("P(server_dns64_synthall:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -3500,9 +3567,9 @@ yyreduce: } break; - case 236: + case 242: /* Line 1792 of yacc.c */ -#line 1189 "./util/configparser.y" +#line 1224 "./util/configparser.y" { OUTYY(("P(name:%s)\n", (yyvsp[(2) - (2)].str))); if(cfg_parser->cfg->stubs->name) @@ -3513,9 +3580,9 @@ yyreduce: } break; - case 237: + case 243: /* Line 1792 of yacc.c */ -#line 1199 "./util/configparser.y" +#line 1234 "./util/configparser.y" { OUTYY(("P(stub-host:%s)\n", (yyvsp[(2) - (2)].str))); if(!cfg_strlist_insert(&cfg_parser->cfg->stubs->hosts, (yyvsp[(2) - (2)].str))) @@ -3523,9 +3590,9 @@ yyreduce: } break; - case 238: + case 244: /* Line 1792 of yacc.c */ -#line 1206 "./util/configparser.y" +#line 1241 "./util/configparser.y" { OUTYY(("P(stub-addr:%s)\n", (yyvsp[(2) - (2)].str))); if(!cfg_strlist_insert(&cfg_parser->cfg->stubs->addrs, (yyvsp[(2) - (2)].str))) @@ -3533,9 +3600,9 @@ yyreduce: } break; - case 239: + case 245: /* Line 1792 of yacc.c */ -#line 1213 "./util/configparser.y" +#line 1248 "./util/configparser.y" { OUTYY(("P(stub-first:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -3545,9 +3612,9 @@ yyreduce: } break; - case 240: + case 246: /* Line 1792 of yacc.c */ -#line 1222 "./util/configparser.y" +#line 1257 "./util/configparser.y" { OUTYY(("P(stub-prime:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -3558,9 +3625,9 @@ yyreduce: } break; - case 241: + case 247: /* Line 1792 of yacc.c */ -#line 1232 "./util/configparser.y" +#line 1267 "./util/configparser.y" { OUTYY(("P(name:%s)\n", (yyvsp[(2) - (2)].str))); if(cfg_parser->cfg->forwards->name) @@ -3571,9 +3638,9 @@ yyreduce: } break; - case 242: + case 248: /* Line 1792 of yacc.c */ -#line 1242 "./util/configparser.y" +#line 1277 "./util/configparser.y" { OUTYY(("P(forward-host:%s)\n", (yyvsp[(2) - (2)].str))); if(!cfg_strlist_insert(&cfg_parser->cfg->forwards->hosts, (yyvsp[(2) - (2)].str))) @@ -3581,9 +3648,9 @@ yyreduce: } break; - case 243: + case 249: /* Line 1792 of yacc.c */ -#line 1249 "./util/configparser.y" +#line 1284 "./util/configparser.y" { OUTYY(("P(forward-addr:%s)\n", (yyvsp[(2) - (2)].str))); if(!cfg_strlist_insert(&cfg_parser->cfg->forwards->addrs, (yyvsp[(2) - (2)].str))) @@ -3591,9 +3658,9 @@ yyreduce: } break; - case 244: + case 250: /* Line 1792 of yacc.c */ -#line 1256 "./util/configparser.y" +#line 1291 "./util/configparser.y" { OUTYY(("P(forward-first:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -3603,17 +3670,17 @@ yyreduce: } break; - case 245: + case 251: /* Line 1792 of yacc.c */ -#line 1265 "./util/configparser.y" +#line 1300 "./util/configparser.y" { OUTYY(("\nP(remote-control:)\n")); } break; - case 255: + case 262: /* Line 1792 of yacc.c */ -#line 1276 "./util/configparser.y" +#line 1311 "./util/configparser.y" { OUTYY(("P(control_enable:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -3624,9 +3691,9 @@ yyreduce: } break; - case 256: + case 263: /* Line 1792 of yacc.c */ -#line 1286 "./util/configparser.y" +#line 1321 "./util/configparser.y" { OUTYY(("P(control_port:%s)\n", (yyvsp[(2) - (2)].str))); if(atoi((yyvsp[(2) - (2)].str)) == 0) @@ -3636,9 +3703,9 @@ yyreduce: } break; - case 257: + case 264: /* Line 1792 of yacc.c */ -#line 1295 "./util/configparser.y" +#line 1330 "./util/configparser.y" { OUTYY(("P(control_interface:%s)\n", (yyvsp[(2) - (2)].str))); if(!cfg_strlist_insert(&cfg_parser->cfg->control_ifs, (yyvsp[(2) - (2)].str))) @@ -3646,9 +3713,22 @@ yyreduce: } break; - case 258: + case 265: +/* Line 1792 of yacc.c */ +#line 1337 "./util/configparser.y" + { + OUTYY(("P(control_use_cert:%s)\n", (yyvsp[(2) - (2)].str))); + if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) + yyerror("expected yes or no."); + else cfg_parser->cfg->remote_control_use_cert = + (strcmp((yyvsp[(2) - (2)].str), "yes")==0); + free((yyvsp[(2) - (2)].str)); + } + break; + + case 266: /* Line 1792 of yacc.c */ -#line 1302 "./util/configparser.y" +#line 1347 "./util/configparser.y" { OUTYY(("P(rc_server_key_file:%s)\n", (yyvsp[(2) - (2)].str))); free(cfg_parser->cfg->server_key_file); @@ -3656,9 +3736,9 @@ yyreduce: } break; - case 259: + case 267: /* Line 1792 of yacc.c */ -#line 1309 "./util/configparser.y" +#line 1354 "./util/configparser.y" { OUTYY(("P(rc_server_cert_file:%s)\n", (yyvsp[(2) - (2)].str))); free(cfg_parser->cfg->server_cert_file); @@ -3666,9 +3746,9 @@ yyreduce: } break; - case 260: + case 268: /* Line 1792 of yacc.c */ -#line 1316 "./util/configparser.y" +#line 1361 "./util/configparser.y" { OUTYY(("P(rc_control_key_file:%s)\n", (yyvsp[(2) - (2)].str))); free(cfg_parser->cfg->control_key_file); @@ -3676,9 +3756,9 @@ yyreduce: } break; - case 261: + case 269: /* Line 1792 of yacc.c */ -#line 1323 "./util/configparser.y" +#line 1368 "./util/configparser.y" { OUTYY(("P(rc_control_cert_file:%s)\n", (yyvsp[(2) - (2)].str))); free(cfg_parser->cfg->control_cert_file); @@ -3686,17 +3766,17 @@ yyreduce: } break; - case 262: + case 270: /* Line 1792 of yacc.c */ -#line 1330 "./util/configparser.y" +#line 1375 "./util/configparser.y" { OUTYY(("\nP(dnstap:)\n")); } break; - case 277: + case 285: /* Line 1792 of yacc.c */ -#line 1347 "./util/configparser.y" +#line 1392 "./util/configparser.y" { OUTYY(("P(dt_dnstap_enable:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -3705,9 +3785,9 @@ yyreduce: } break; - case 278: + case 286: /* Line 1792 of yacc.c */ -#line 1355 "./util/configparser.y" +#line 1400 "./util/configparser.y" { OUTYY(("P(dt_dnstap_socket_path:%s)\n", (yyvsp[(2) - (2)].str))); free(cfg_parser->cfg->dnstap_socket_path); @@ -3715,9 +3795,9 @@ yyreduce: } break; - case 279: + case 287: /* Line 1792 of yacc.c */ -#line 1362 "./util/configparser.y" +#line 1407 "./util/configparser.y" { OUTYY(("P(dt_dnstap_send_identity:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -3726,9 +3806,9 @@ yyreduce: } break; - case 280: + case 288: /* Line 1792 of yacc.c */ -#line 1370 "./util/configparser.y" +#line 1415 "./util/configparser.y" { OUTYY(("P(dt_dnstap_send_version:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -3737,9 +3817,9 @@ yyreduce: } break; - case 281: + case 289: /* Line 1792 of yacc.c */ -#line 1378 "./util/configparser.y" +#line 1423 "./util/configparser.y" { OUTYY(("P(dt_dnstap_identity:%s)\n", (yyvsp[(2) - (2)].str))); free(cfg_parser->cfg->dnstap_identity); @@ -3747,9 +3827,9 @@ yyreduce: } break; - case 282: + case 290: /* Line 1792 of yacc.c */ -#line 1385 "./util/configparser.y" +#line 1430 "./util/configparser.y" { OUTYY(("P(dt_dnstap_version:%s)\n", (yyvsp[(2) - (2)].str))); free(cfg_parser->cfg->dnstap_version); @@ -3757,9 +3837,9 @@ yyreduce: } break; - case 283: + case 291: /* Line 1792 of yacc.c */ -#line 1392 "./util/configparser.y" +#line 1437 "./util/configparser.y" { OUTYY(("P(dt_dnstap_log_resolver_query_messages:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -3769,9 +3849,9 @@ yyreduce: } break; - case 284: + case 292: /* Line 1792 of yacc.c */ -#line 1401 "./util/configparser.y" +#line 1446 "./util/configparser.y" { OUTYY(("P(dt_dnstap_log_resolver_response_messages:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -3781,9 +3861,9 @@ yyreduce: } break; - case 285: + case 293: /* Line 1792 of yacc.c */ -#line 1410 "./util/configparser.y" +#line 1455 "./util/configparser.y" { OUTYY(("P(dt_dnstap_log_client_query_messages:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -3793,9 +3873,9 @@ yyreduce: } break; - case 286: + case 294: /* Line 1792 of yacc.c */ -#line 1419 "./util/configparser.y" +#line 1464 "./util/configparser.y" { OUTYY(("P(dt_dnstap_log_client_response_messages:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -3805,9 +3885,9 @@ yyreduce: } break; - case 287: + case 295: /* Line 1792 of yacc.c */ -#line 1428 "./util/configparser.y" +#line 1473 "./util/configparser.y" { OUTYY(("P(dt_dnstap_log_forwarder_query_messages:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -3817,9 +3897,9 @@ yyreduce: } break; - case 288: + case 296: /* Line 1792 of yacc.c */ -#line 1437 "./util/configparser.y" +#line 1482 "./util/configparser.y" { OUTYY(("P(dt_dnstap_log_forwarder_response_messages:%s)\n", (yyvsp[(2) - (2)].str))); if(strcmp((yyvsp[(2) - (2)].str), "yes") != 0 && strcmp((yyvsp[(2) - (2)].str), "no") != 0) @@ -3829,17 +3909,17 @@ yyreduce: } break; - case 289: + case 297: /* Line 1792 of yacc.c */ -#line 1446 "./util/configparser.y" +#line 1491 "./util/configparser.y" { OUTYY(("\nP(python:)\n")); } break; - case 293: + case 301: /* Line 1792 of yacc.c */ -#line 1455 "./util/configparser.y" +#line 1500 "./util/configparser.y" { OUTYY(("P(python-script:%s)\n", (yyvsp[(2) - (2)].str))); free(cfg_parser->cfg->python_script); @@ -3849,7 +3929,7 @@ yyreduce: /* Line 1792 of yacc.c */ -#line 3853 "util/configparser.c" +#line 3933 "util/configparser.c" default: break; } /* User semantic actions sometimes alter yychar, and that requires @@ -4081,7 +4161,7 @@ yyreturn: /* Line 2055 of yacc.c */ -#line 1460 "./util/configparser.y" +#line 1505 "./util/configparser.y" /* parse helper routines could be here */ diff --git a/external/unbound/util/configparser.h b/external/unbound/util/configparser.h index 0ee72661d..a8ab6ee59 100644 --- a/external/unbound/util/configparser.h +++ b/external/unbound/util/configparser.h @@ -139,59 +139,63 @@ extern int yydebug; VAR_SERVER_CERT_FILE = 348, VAR_CONTROL_KEY_FILE = 349, VAR_CONTROL_CERT_FILE = 350, - VAR_EXTENDED_STATISTICS = 351, - VAR_LOCAL_DATA_PTR = 352, - VAR_JOSTLE_TIMEOUT = 353, - VAR_STUB_PRIME = 354, - VAR_UNWANTED_REPLY_THRESHOLD = 355, - VAR_LOG_TIME_ASCII = 356, - VAR_DOMAIN_INSECURE = 357, - VAR_PYTHON = 358, - VAR_PYTHON_SCRIPT = 359, - VAR_VAL_SIG_SKEW_MIN = 360, - VAR_VAL_SIG_SKEW_MAX = 361, - VAR_CACHE_MIN_TTL = 362, - VAR_VAL_LOG_LEVEL = 363, - VAR_AUTO_TRUST_ANCHOR_FILE = 364, - VAR_KEEP_MISSING = 365, - VAR_ADD_HOLDDOWN = 366, - VAR_DEL_HOLDDOWN = 367, - VAR_SO_RCVBUF = 368, - VAR_EDNS_BUFFER_SIZE = 369, - VAR_PREFETCH = 370, - VAR_PREFETCH_KEY = 371, - VAR_SO_SNDBUF = 372, - VAR_SO_REUSEPORT = 373, - VAR_HARDEN_BELOW_NXDOMAIN = 374, - VAR_IGNORE_CD_FLAG = 375, - VAR_LOG_QUERIES = 376, - VAR_TCP_UPSTREAM = 377, - VAR_SSL_UPSTREAM = 378, - VAR_SSL_SERVICE_KEY = 379, - VAR_SSL_SERVICE_PEM = 380, - VAR_SSL_PORT = 381, - VAR_FORWARD_FIRST = 382, - VAR_STUB_FIRST = 383, - VAR_MINIMAL_RESPONSES = 384, - VAR_RRSET_ROUNDROBIN = 385, - VAR_MAX_UDP_SIZE = 386, - VAR_DELAY_CLOSE = 387, - VAR_UNBLOCK_LAN_ZONES = 388, - VAR_DNS64_PREFIX = 389, - VAR_DNS64_SYNTHALL = 390, - VAR_DNSTAP = 391, - VAR_DNSTAP_ENABLE = 392, - VAR_DNSTAP_SOCKET_PATH = 393, - VAR_DNSTAP_SEND_IDENTITY = 394, - VAR_DNSTAP_SEND_VERSION = 395, - VAR_DNSTAP_IDENTITY = 396, - VAR_DNSTAP_VERSION = 397, - VAR_DNSTAP_LOG_RESOLVER_QUERY_MESSAGES = 398, - VAR_DNSTAP_LOG_RESOLVER_RESPONSE_MESSAGES = 399, - VAR_DNSTAP_LOG_CLIENT_QUERY_MESSAGES = 400, - VAR_DNSTAP_LOG_CLIENT_RESPONSE_MESSAGES = 401, - VAR_DNSTAP_LOG_FORWARDER_QUERY_MESSAGES = 402, - VAR_DNSTAP_LOG_FORWARDER_RESPONSE_MESSAGES = 403 + VAR_CONTROL_USE_CERT = 351, + VAR_EXTENDED_STATISTICS = 352, + VAR_LOCAL_DATA_PTR = 353, + VAR_JOSTLE_TIMEOUT = 354, + VAR_STUB_PRIME = 355, + VAR_UNWANTED_REPLY_THRESHOLD = 356, + VAR_LOG_TIME_ASCII = 357, + VAR_DOMAIN_INSECURE = 358, + VAR_PYTHON = 359, + VAR_PYTHON_SCRIPT = 360, + VAR_VAL_SIG_SKEW_MIN = 361, + VAR_VAL_SIG_SKEW_MAX = 362, + VAR_CACHE_MIN_TTL = 363, + VAR_VAL_LOG_LEVEL = 364, + VAR_AUTO_TRUST_ANCHOR_FILE = 365, + VAR_KEEP_MISSING = 366, + VAR_ADD_HOLDDOWN = 367, + VAR_DEL_HOLDDOWN = 368, + VAR_SO_RCVBUF = 369, + VAR_EDNS_BUFFER_SIZE = 370, + VAR_PREFETCH = 371, + VAR_PREFETCH_KEY = 372, + VAR_SO_SNDBUF = 373, + VAR_SO_REUSEPORT = 374, + VAR_HARDEN_BELOW_NXDOMAIN = 375, + VAR_IGNORE_CD_FLAG = 376, + VAR_LOG_QUERIES = 377, + VAR_TCP_UPSTREAM = 378, + VAR_SSL_UPSTREAM = 379, + VAR_SSL_SERVICE_KEY = 380, + VAR_SSL_SERVICE_PEM = 381, + VAR_SSL_PORT = 382, + VAR_FORWARD_FIRST = 383, + VAR_STUB_FIRST = 384, + VAR_MINIMAL_RESPONSES = 385, + VAR_RRSET_ROUNDROBIN = 386, + VAR_MAX_UDP_SIZE = 387, + VAR_DELAY_CLOSE = 388, + VAR_UNBLOCK_LAN_ZONES = 389, + VAR_INFRA_CACHE_MIN_RTT = 390, + VAR_DNS64_PREFIX = 391, + VAR_DNS64_SYNTHALL = 392, + VAR_DNSTAP = 393, + VAR_DNSTAP_ENABLE = 394, + VAR_DNSTAP_SOCKET_PATH = 395, + VAR_DNSTAP_SEND_IDENTITY = 396, + VAR_DNSTAP_SEND_VERSION = 397, + VAR_DNSTAP_IDENTITY = 398, + VAR_DNSTAP_VERSION = 399, + VAR_DNSTAP_LOG_RESOLVER_QUERY_MESSAGES = 400, + VAR_DNSTAP_LOG_RESOLVER_RESPONSE_MESSAGES = 401, + VAR_DNSTAP_LOG_CLIENT_QUERY_MESSAGES = 402, + VAR_DNSTAP_LOG_CLIENT_RESPONSE_MESSAGES = 403, + VAR_DNSTAP_LOG_FORWARDER_QUERY_MESSAGES = 404, + VAR_DNSTAP_LOG_FORWARDER_RESPONSE_MESSAGES = 405, + VAR_HARDEN_ALGO_DOWNGRADE = 406, + VAR_IP_TRANSPARENT = 407 }; #endif /* Tokens. */ @@ -288,59 +292,63 @@ extern int yydebug; #define VAR_SERVER_CERT_FILE 348 #define VAR_CONTROL_KEY_FILE 349 #define VAR_CONTROL_CERT_FILE 350 -#define VAR_EXTENDED_STATISTICS 351 -#define VAR_LOCAL_DATA_PTR 352 -#define VAR_JOSTLE_TIMEOUT 353 -#define VAR_STUB_PRIME 354 -#define VAR_UNWANTED_REPLY_THRESHOLD 355 -#define VAR_LOG_TIME_ASCII 356 -#define VAR_DOMAIN_INSECURE 357 -#define VAR_PYTHON 358 -#define VAR_PYTHON_SCRIPT 359 -#define VAR_VAL_SIG_SKEW_MIN 360 -#define VAR_VAL_SIG_SKEW_MAX 361 -#define VAR_CACHE_MIN_TTL 362 -#define VAR_VAL_LOG_LEVEL 363 -#define VAR_AUTO_TRUST_ANCHOR_FILE 364 -#define VAR_KEEP_MISSING 365 -#define VAR_ADD_HOLDDOWN 366 -#define VAR_DEL_HOLDDOWN 367 -#define VAR_SO_RCVBUF 368 -#define VAR_EDNS_BUFFER_SIZE 369 -#define VAR_PREFETCH 370 -#define VAR_PREFETCH_KEY 371 -#define VAR_SO_SNDBUF 372 -#define VAR_SO_REUSEPORT 373 -#define VAR_HARDEN_BELOW_NXDOMAIN 374 -#define VAR_IGNORE_CD_FLAG 375 -#define VAR_LOG_QUERIES 376 -#define VAR_TCP_UPSTREAM 377 -#define VAR_SSL_UPSTREAM 378 -#define VAR_SSL_SERVICE_KEY 379 -#define VAR_SSL_SERVICE_PEM 380 -#define VAR_SSL_PORT 381 -#define VAR_FORWARD_FIRST 382 -#define VAR_STUB_FIRST 383 -#define VAR_MINIMAL_RESPONSES 384 -#define VAR_RRSET_ROUNDROBIN 385 -#define VAR_MAX_UDP_SIZE 386 -#define VAR_DELAY_CLOSE 387 -#define VAR_UNBLOCK_LAN_ZONES 388 -#define VAR_DNS64_PREFIX 389 -#define VAR_DNS64_SYNTHALL 390 -#define VAR_DNSTAP 391 -#define VAR_DNSTAP_ENABLE 392 -#define VAR_DNSTAP_SOCKET_PATH 393 -#define VAR_DNSTAP_SEND_IDENTITY 394 -#define VAR_DNSTAP_SEND_VERSION 395 -#define VAR_DNSTAP_IDENTITY 396 -#define VAR_DNSTAP_VERSION 397 -#define VAR_DNSTAP_LOG_RESOLVER_QUERY_MESSAGES 398 -#define VAR_DNSTAP_LOG_RESOLVER_RESPONSE_MESSAGES 399 -#define VAR_DNSTAP_LOG_CLIENT_QUERY_MESSAGES 400 -#define VAR_DNSTAP_LOG_CLIENT_RESPONSE_MESSAGES 401 -#define VAR_DNSTAP_LOG_FORWARDER_QUERY_MESSAGES 402 -#define VAR_DNSTAP_LOG_FORWARDER_RESPONSE_MESSAGES 403 +#define VAR_CONTROL_USE_CERT 351 +#define VAR_EXTENDED_STATISTICS 352 +#define VAR_LOCAL_DATA_PTR 353 +#define VAR_JOSTLE_TIMEOUT 354 +#define VAR_STUB_PRIME 355 +#define VAR_UNWANTED_REPLY_THRESHOLD 356 +#define VAR_LOG_TIME_ASCII 357 +#define VAR_DOMAIN_INSECURE 358 +#define VAR_PYTHON 359 +#define VAR_PYTHON_SCRIPT 360 +#define VAR_VAL_SIG_SKEW_MIN 361 +#define VAR_VAL_SIG_SKEW_MAX 362 +#define VAR_CACHE_MIN_TTL 363 +#define VAR_VAL_LOG_LEVEL 364 +#define VAR_AUTO_TRUST_ANCHOR_FILE 365 +#define VAR_KEEP_MISSING 366 +#define VAR_ADD_HOLDDOWN 367 +#define VAR_DEL_HOLDDOWN 368 +#define VAR_SO_RCVBUF 369 +#define VAR_EDNS_BUFFER_SIZE 370 +#define VAR_PREFETCH 371 +#define VAR_PREFETCH_KEY 372 +#define VAR_SO_SNDBUF 373 +#define VAR_SO_REUSEPORT 374 +#define VAR_HARDEN_BELOW_NXDOMAIN 375 +#define VAR_IGNORE_CD_FLAG 376 +#define VAR_LOG_QUERIES 377 +#define VAR_TCP_UPSTREAM 378 +#define VAR_SSL_UPSTREAM 379 +#define VAR_SSL_SERVICE_KEY 380 +#define VAR_SSL_SERVICE_PEM 381 +#define VAR_SSL_PORT 382 +#define VAR_FORWARD_FIRST 383 +#define VAR_STUB_FIRST 384 +#define VAR_MINIMAL_RESPONSES 385 +#define VAR_RRSET_ROUNDROBIN 386 +#define VAR_MAX_UDP_SIZE 387 +#define VAR_DELAY_CLOSE 388 +#define VAR_UNBLOCK_LAN_ZONES 389 +#define VAR_INFRA_CACHE_MIN_RTT 390 +#define VAR_DNS64_PREFIX 391 +#define VAR_DNS64_SYNTHALL 392 +#define VAR_DNSTAP 393 +#define VAR_DNSTAP_ENABLE 394 +#define VAR_DNSTAP_SOCKET_PATH 395 +#define VAR_DNSTAP_SEND_IDENTITY 396 +#define VAR_DNSTAP_SEND_VERSION 397 +#define VAR_DNSTAP_IDENTITY 398 +#define VAR_DNSTAP_VERSION 399 +#define VAR_DNSTAP_LOG_RESOLVER_QUERY_MESSAGES 400 +#define VAR_DNSTAP_LOG_RESOLVER_RESPONSE_MESSAGES 401 +#define VAR_DNSTAP_LOG_CLIENT_QUERY_MESSAGES 402 +#define VAR_DNSTAP_LOG_CLIENT_RESPONSE_MESSAGES 403 +#define VAR_DNSTAP_LOG_FORWARDER_QUERY_MESSAGES 404 +#define VAR_DNSTAP_LOG_FORWARDER_RESPONSE_MESSAGES 405 +#define VAR_HARDEN_ALGO_DOWNGRADE 406 +#define VAR_IP_TRANSPARENT 407 @@ -354,7 +362,7 @@ typedef union YYSTYPE /* Line 2058 of yacc.c */ -#line 358 "util/configparser.h" +#line 366 "util/configparser.h" } YYSTYPE; # define YYSTYPE_IS_TRIVIAL 1 # define yystype YYSTYPE /* obsolescent; will be withdrawn */ diff --git a/external/unbound/util/configparser.y b/external/unbound/util/configparser.y index 7a92d9ee7..fb94be9d4 100644 --- a/external/unbound/util/configparser.y +++ b/external/unbound/util/configparser.y @@ -95,6 +95,7 @@ extern struct config_parser_state* cfg_parser; %token VAR_PRIVATE_DOMAIN VAR_REMOTE_CONTROL VAR_CONTROL_ENABLE %token VAR_CONTROL_INTERFACE VAR_CONTROL_PORT VAR_SERVER_KEY_FILE %token VAR_SERVER_CERT_FILE VAR_CONTROL_KEY_FILE VAR_CONTROL_CERT_FILE +%token VAR_CONTROL_USE_CERT %token VAR_EXTENDED_STATISTICS VAR_LOCAL_DATA_PTR VAR_JOSTLE_TIMEOUT %token VAR_STUB_PRIME VAR_UNWANTED_REPLY_THRESHOLD VAR_LOG_TIME_ASCII %token VAR_DOMAIN_INSECURE VAR_PYTHON VAR_PYTHON_SCRIPT VAR_VAL_SIG_SKEW_MIN @@ -106,6 +107,7 @@ extern struct config_parser_state* cfg_parser; %token VAR_SSL_SERVICE_KEY VAR_SSL_SERVICE_PEM VAR_SSL_PORT VAR_FORWARD_FIRST %token VAR_STUB_FIRST VAR_MINIMAL_RESPONSES VAR_RRSET_ROUNDROBIN %token VAR_MAX_UDP_SIZE VAR_DELAY_CLOSE VAR_UNBLOCK_LAN_ZONES +%token VAR_INFRA_CACHE_MIN_RTT %token VAR_DNS64_PREFIX VAR_DNS64_SYNTHALL %token VAR_DNSTAP VAR_DNSTAP_ENABLE VAR_DNSTAP_SOCKET_PATH %token VAR_DNSTAP_SEND_IDENTITY VAR_DNSTAP_SEND_VERSION @@ -116,6 +118,7 @@ extern struct config_parser_state* cfg_parser; %token VAR_DNSTAP_LOG_CLIENT_RESPONSE_MESSAGES %token VAR_DNSTAP_LOG_FORWARDER_QUERY_MESSAGES %token VAR_DNSTAP_LOG_FORWARDER_RESPONSE_MESSAGES +%token VAR_HARDEN_ALGO_DOWNGRADE VAR_IP_TRANSPARENT %% toplevelvars: /* empty */ | toplevelvars toplevelvar ; @@ -174,7 +177,9 @@ content_server: server_num_threads | server_verbosity | server_port | server_ssl_service_key | server_ssl_service_pem | server_ssl_port | server_minimal_responses | server_rrset_roundrobin | server_max_udp_size | server_so_reuseport | server_delay_close | server_unblock_lan_zones | - server_dns64_prefix | server_dns64_synthall + server_dns64_prefix | server_dns64_synthall | + server_infra_cache_min_rtt | server_harden_algo_downgrade | + server_ip_transparent ; stubstart: VAR_STUB_ZONE { @@ -617,6 +622,16 @@ server_so_reuseport: VAR_SO_REUSEPORT STRING_ARG free($2); } ; +server_ip_transparent: VAR_IP_TRANSPARENT STRING_ARG + { + OUTYY(("P(server_ip_transparent:%s)\n", $2)); + if(strcmp($2, "yes") != 0 && strcmp($2, "no") != 0) + yyerror("expected yes or no."); + else cfg_parser->cfg->ip_transparent = + (strcmp($2, "yes")==0); + free($2); + } + ; server_edns_buffer_size: VAR_EDNS_BUFFER_SIZE STRING_ARG { OUTYY(("P(server_edns_buffer_size:%s)\n", $2)); @@ -767,6 +782,15 @@ server_infra_cache_slabs: VAR_INFRA_CACHE_SLABS STRING_ARG free($2); } ; +server_infra_cache_min_rtt: VAR_INFRA_CACHE_MIN_RTT STRING_ARG + { + OUTYY(("P(server_infra_cache_min_rtt:%s)\n", $2)); + if(atoi($2) == 0 && strcmp($2, "0") != 0) + yyerror("number expected"); + else cfg_parser->cfg->infra_cache_min_rtt = atoi($2); + free($2); + } + ; server_target_fetch_policy: VAR_TARGET_FETCH_POLICY STRING_ARG { OUTYY(("P(server_target_fetch_policy:%s)\n", $2)); @@ -834,6 +858,16 @@ server_harden_referral_path: VAR_HARDEN_REFERRAL_PATH STRING_ARG free($2); } ; +server_harden_algo_downgrade: VAR_HARDEN_ALGO_DOWNGRADE STRING_ARG + { + OUTYY(("P(server_harden_algo_downgrade:%s)\n", $2)); + if(strcmp($2, "yes") != 0 && strcmp($2, "no") != 0) + yyerror("expected yes or no."); + else cfg_parser->cfg->harden_algo_downgrade = + (strcmp($2, "yes")==0); + free($2); + } + ; server_use_caps_for_id: VAR_USE_CAPS_FOR_ID STRING_ARG { OUTYY(("P(server_use_caps_for_id:%s)\n", $2)); @@ -1104,10 +1138,11 @@ server_local_zone: VAR_LOCAL_ZONE STRING_ARG STRING_ARG if(strcmp($3, "static")!=0 && strcmp($3, "deny")!=0 && strcmp($3, "refuse")!=0 && strcmp($3, "redirect")!=0 && strcmp($3, "transparent")!=0 && strcmp($3, "nodefault")!=0 - && strcmp($3, "typetransparent")!=0) + && strcmp($3, "typetransparent")!=0 && + strcmp($3, "inform")!=0) yyerror("local-zone type: expected static, deny, " "refuse, redirect, transparent, " - "typetransparent or nodefault"); + "typetransparent, inform or nodefault"); else if(strcmp($3, "nodefault")==0) { if(!cfg_strlist_insert(&cfg_parser->cfg-> local_zones_nodefault, $2)) @@ -1270,7 +1305,7 @@ contents_rc: contents_rc content_rc | ; content_rc: rc_control_enable | rc_control_interface | rc_control_port | rc_server_key_file | rc_server_cert_file | rc_control_key_file | - rc_control_cert_file + rc_control_cert_file | rc_control_use_cert ; rc_control_enable: VAR_CONTROL_ENABLE STRING_ARG { @@ -1298,6 +1333,16 @@ rc_control_interface: VAR_CONTROL_INTERFACE STRING_ARG yyerror("out of memory"); } ; +rc_control_use_cert: VAR_CONTROL_USE_CERT STRING_ARG + { + OUTYY(("P(control_use_cert:%s)\n", $2)); + if(strcmp($2, "yes") != 0 && strcmp($2, "no") != 0) + yyerror("expected yes or no."); + else cfg_parser->cfg->remote_control_use_cert = + (strcmp($2, "yes")==0); + free($2); + } + ; rc_server_key_file: VAR_SERVER_KEY_FILE STRING_ARG { OUTYY(("P(rc_server_key_file:%s)\n", $2)); diff --git a/external/unbound/util/data/dname.c b/external/unbound/util/data/dname.c index d43bbf6d2..79bf52ad4 100644 --- a/external/unbound/util/data/dname.c +++ b/external/unbound/util/data/dname.c @@ -45,7 +45,7 @@ #include "util/data/msgparse.h" #include "util/log.h" #include "util/storage/lookup3.h" -#include "ldns/sbuffer.h" +#include "sldns/sbuffer.h" /* determine length of a dname in buffer, no compression pointers allowed */ size_t diff --git a/external/unbound/util/data/msgencode.c b/external/unbound/util/data/msgencode.c index 26b5deabe..f9a8c5f67 100644 --- a/external/unbound/util/data/msgencode.c +++ b/external/unbound/util/data/msgencode.c @@ -47,7 +47,7 @@ #include "util/log.h" #include "util/regional.h" #include "util/net_help.h" -#include "ldns/sbuffer.h" +#include "sldns/sbuffer.h" /** return code that means the function ran out of memory. negative so it does * not conflict with DNS rcodes. */ diff --git a/external/unbound/util/data/msgparse.c b/external/unbound/util/data/msgparse.c index abe778a89..108c9dacb 100644 --- a/external/unbound/util/data/msgparse.c +++ b/external/unbound/util/data/msgparse.c @@ -42,10 +42,10 @@ #include "util/data/packed_rrset.h" #include "util/storage/lookup3.h" #include "util/regional.h" -#include "ldns/rrdef.h" -#include "ldns/sbuffer.h" -#include "ldns/parseutil.h" -#include "ldns/wire2str.h" +#include "sldns/rrdef.h" +#include "sldns/sbuffer.h" +#include "sldns/parseutil.h" +#include "sldns/wire2str.h" /** smart comparison of (compressed, valid) dnames from packet */ static int diff --git a/external/unbound/util/data/msgparse.h b/external/unbound/util/data/msgparse.h index 221a45aad..1a5ced356 100644 --- a/external/unbound/util/data/msgparse.h +++ b/external/unbound/util/data/msgparse.h @@ -63,8 +63,8 @@ #ifndef UTIL_DATA_MSGPARSE_H #define UTIL_DATA_MSGPARSE_H #include "util/storage/lruhash.h" -#include "ldns/pkthdr.h" -#include "ldns/rrdef.h" +#include "sldns/pkthdr.h" +#include "sldns/rrdef.h" struct sldns_buffer; struct rrset_parse; struct rr_parse; diff --git a/external/unbound/util/data/msgreply.c b/external/unbound/util/data/msgreply.c index 68bcfd09e..dc27be905 100644 --- a/external/unbound/util/data/msgreply.c +++ b/external/unbound/util/data/msgreply.c @@ -50,8 +50,8 @@ #include "util/regional.h" #include "util/data/msgparse.h" #include "util/data/msgencode.h" -#include "ldns/sbuffer.h" -#include "ldns/wire2str.h" +#include "sldns/sbuffer.h" +#include "sldns/wire2str.h" /** MAX TTL default for messages and rrsets */ time_t MAX_TTL = 3600 * 24 * 10; /* ten days */ @@ -87,6 +87,7 @@ construct_reply_info_base(struct regional* region, uint16_t flags, size_t qd, /* rrset_count-1 because the first ref is part of the struct. */ size_t s = sizeof(struct reply_info) - sizeof(struct rrset_ref) + sizeof(struct ub_packed_rrset_key*) * total; + if(total >= RR_COUNT_MAX) return NULL; /* sanity check on numRRS*/ if(region) rep = (struct reply_info*)regional_alloc(region, s); else rep = (struct reply_info*)malloc(s + @@ -277,7 +278,11 @@ parse_create_rrset(sldns_buffer* pkt, struct rrset_parse* pset, struct packed_rrset_data** data, struct regional* region) { /* allocate */ - size_t s = sizeof(struct packed_rrset_data) + + size_t s; + if(pset->rr_count > RR_COUNT_MAX || pset->rrsig_count > RR_COUNT_MAX || + pset->size > RR_COUNT_MAX) + return 0; /* protect against integer overflow */ + s = sizeof(struct packed_rrset_data) + (pset->rr_count + pset->rrsig_count) * (sizeof(size_t)+sizeof(uint8_t*)+sizeof(time_t)) + pset->size; diff --git a/external/unbound/util/data/packed_rrset.c b/external/unbound/util/data/packed_rrset.c index 807468576..0a5c9d327 100644 --- a/external/unbound/util/data/packed_rrset.c +++ b/external/unbound/util/data/packed_rrset.c @@ -47,9 +47,9 @@ #include "util/alloc.h" #include "util/regional.h" #include "util/net_help.h" -#include "ldns/rrdef.h" -#include "ldns/sbuffer.h" -#include "ldns/wire2str.h" +#include "sldns/rrdef.h" +#include "sldns/sbuffer.h" +#include "sldns/wire2str.h" void ub_packed_rrset_parsedelete(struct ub_packed_rrset_key* pkey, diff --git a/external/unbound/util/data/packed_rrset.h b/external/unbound/util/data/packed_rrset.h index 5d7990a2b..6039aef24 100644 --- a/external/unbound/util/data/packed_rrset.h +++ b/external/unbound/util/data/packed_rrset.h @@ -58,6 +58,12 @@ typedef uint64_t rrset_id_t; * from the SOA in the answer section from a direct SOA query or ANY query. */ #define PACKED_RRSET_SOA_NEG 0x4 +/** number of rrs and rrsets for integer overflow protection. More than + * this is not really possible (64K packet has much less RRs and RRsets) in + * a message. And this is small enough that also multiplied there is no + * integer overflow. */ +#define RR_COUNT_MAX 0xffffff + /** * The identifying information for an RRset. */ diff --git a/external/unbound/util/iana_ports.inc b/external/unbound/util/iana_ports.inc index d318477e5..9cedc80ea 100644 --- a/external/unbound/util/iana_ports.inc +++ b/external/unbound/util/iana_ports.inc @@ -3819,6 +3819,7 @@ 4359, 4361, 4362, +4366, 4368, 4369, 4370, @@ -4358,6 +4359,7 @@ 6072, 6073, 6074, +6080, 6081, 6082, 6083, @@ -4399,6 +4401,7 @@ 6163, 6200, 6201, +6209, 6222, 6241, 6242, @@ -4488,6 +4491,8 @@ 6628, 6633, 6634, +6635, +6636, 6653, 6657, 6670, @@ -4671,6 +4676,7 @@ 7778, 7779, 7781, +7784, 7786, 7787, 7789, @@ -4839,6 +4845,8 @@ 8912, 8913, 8954, +8980, +8981, 8989, 8990, 8991, @@ -4846,6 +4854,7 @@ 9000, 9001, 9002, +9006, 9007, 9009, 9020, @@ -5230,6 +5239,7 @@ 22005, 22273, 22305, +22335, 22343, 22347, 22350, diff --git a/external/unbound/util/log.c b/external/unbound/util/log.c index f90efa71c..3ebd12025 100644 --- a/external/unbound/util/log.c +++ b/external/unbound/util/log.c @@ -40,7 +40,7 @@ #include "config.h" #include "util/log.h" #include "util/locks.h" -#include "ldns/sbuffer.h" +#include "sldns/sbuffer.h" #include <stdarg.h> #ifdef HAVE_TIME_H #include <time.h> @@ -164,6 +164,14 @@ void log_thread_set(int* num) ub_thread_key_set(logkey, num); } +int log_thread_get(void) +{ + unsigned int* tid; + if(!key_created) return 0; + tid = (unsigned int*)ub_thread_key_get(logkey); + return (int)(tid?*tid:0); +} + void log_ident_set(const char* id) { ident = id; diff --git a/external/unbound/util/log.h b/external/unbound/util/log.h index ea283da7b..8e85ee620 100644 --- a/external/unbound/util/log.h +++ b/external/unbound/util/log.h @@ -98,6 +98,15 @@ void log_file(FILE *f); void log_thread_set(int* num); /** + * Get the thread id from logging system. Set after log_init is + * initialised, or log_thread_set for newly created threads. + * This initialisation happens in unbound as a daemon, in daemon + * startup code, when that spawns threads. + * @return thread number, from 0 and up. Before initialised, returns 0. + */ +int log_thread_get(void); + +/** * Set identity to print, default is 'unbound'. * @param id: string to print. Name of executable. */ diff --git a/external/unbound/util/net_help.c b/external/unbound/util/net_help.c index 8c2bac737..8b39af6b3 100644 --- a/external/unbound/util/net_help.c +++ b/external/unbound/util/net_help.c @@ -43,8 +43,8 @@ #include "util/data/dname.h" #include "util/module.h" #include "util/regional.h" -#include "ldns/parseutil.h" -#include "ldns/wire2str.h" +#include "sldns/parseutil.h" +#include "sldns/wire2str.h" #include <fcntl.h> #ifdef HAVE_OPENSSL_SSL_H #include <openssl/ssl.h> @@ -156,7 +156,12 @@ log_addr(enum verbosity_value v, const char* str, case AF_INET6: family="ip6"; sinaddr = &((struct sockaddr_in6*)addr)->sin6_addr; break; - case AF_UNIX: family="unix"; break; + case AF_LOCAL: + dest[0]=0; + (void)inet_ntop(af, sinaddr, dest, + (socklen_t)sizeof(dest)); + verbose(v, "%s local %s", str, dest); + return; /* do not continue and try to get port */ default: break; } if(inet_ntop(af, sinaddr, dest, (socklen_t)sizeof(dest)) == 0) { @@ -313,7 +318,7 @@ void log_name_addr(enum verbosity_value v, const char* str, uint8_t* zone, case AF_INET6: family=""; sinaddr = &((struct sockaddr_in6*)addr)->sin6_addr; break; - case AF_UNIX: family="unix_family "; break; + case AF_LOCAL: family="local "; break; default: break; } if(inet_ntop(af, sinaddr, dest, (socklen_t)sizeof(dest)) == 0) { @@ -765,7 +770,7 @@ static lock_basic_t *ub_openssl_locks = NULL; static unsigned long ub_crypto_id_cb(void) { - return (unsigned long)ub_thread_self(); + return (unsigned long)log_thread_get(); } static void @@ -784,8 +789,8 @@ int ub_openssl_lock_init(void) { #if defined(HAVE_SSL) && defined(OPENSSL_THREADS) && !defined(THREADS_DISABLED) int i; - ub_openssl_locks = (lock_basic_t*)malloc( - sizeof(lock_basic_t)*CRYPTO_num_locks()); + ub_openssl_locks = (lock_basic_t*)reallocarray( + NULL, (size_t)CRYPTO_num_locks(), sizeof(lock_basic_t)); if(!ub_openssl_locks) return 0; for(i=0; i<CRYPTO_num_locks(); i++) { diff --git a/external/unbound/util/netevent.c b/external/unbound/util/netevent.c index c7ed30e6c..9f1e86952 100644 --- a/external/unbound/util/netevent.c +++ b/external/unbound/util/netevent.c @@ -43,8 +43,8 @@ #include "util/log.h" #include "util/net_help.h" #include "util/fptr_wlist.h" -#include "ldns/pkthdr.h" -#include "ldns/sbuffer.h" +#include "sldns/pkthdr.h" +#include "sldns/sbuffer.h" #include "dnstap/dnstap.h" #ifdef HAVE_OPENSSL_SSL_H #include <openssl/ssl.h> @@ -879,12 +879,12 @@ comm_point_tcp_accept_callback(int fd, short event, void* arg) } /* grab the tcp handler buffers */ + c->cur_tcp_count++; c->tcp_free = c_hdl->tcp_free; if(!c->tcp_free) { /* stop accepting incoming queries for now. */ comm_point_stop_listening(c); } - /* addr is dropped. Not needed for tcp reply. */ setup_tcp_handler(c_hdl, new_fd); } @@ -902,6 +902,7 @@ reclaim_tcp_handler(struct comm_point* c) } comm_point_close(c); if(c->tcp_parent) { + c->tcp_parent->cur_tcp_count--; c->tcp_free = c->tcp_parent->tcp_free; c->tcp_parent->tcp_free = c; if(!c->tcp_free) { @@ -1528,6 +1529,7 @@ comm_point_create_udp(struct comm_base *base, int fd, sldns_buffer* buffer, c->tcp_byte_count = 0; c->tcp_parent = NULL; c->max_tcp_count = 0; + c->cur_tcp_count = 0; c->tcp_handlers = NULL; c->tcp_free = NULL; c->type = comm_udp; @@ -1578,6 +1580,7 @@ comm_point_create_udp_ancil(struct comm_base *base, int fd, c->tcp_byte_count = 0; c->tcp_parent = NULL; c->max_tcp_count = 0; + c->cur_tcp_count = 0; c->tcp_handlers = NULL; c->tcp_free = NULL; c->type = comm_udp; @@ -1639,6 +1642,7 @@ comm_point_create_tcp_handler(struct comm_base *base, c->tcp_byte_count = 0; c->tcp_parent = parent; c->max_tcp_count = 0; + c->cur_tcp_count = 0; c->tcp_handlers = NULL; c->tcp_free = NULL; c->type = comm_tcp; @@ -1691,6 +1695,7 @@ comm_point_create_tcp(struct comm_base *base, int fd, int num, size_t bufsize, c->tcp_byte_count = 0; c->tcp_parent = NULL; c->max_tcp_count = num; + c->cur_tcp_count = 0; c->tcp_handlers = (struct comm_point**)calloc((size_t)num, sizeof(struct comm_point*)); if(!c->tcp_handlers) { @@ -1758,6 +1763,7 @@ comm_point_create_tcp_out(struct comm_base *base, size_t bufsize, c->tcp_byte_count = 0; c->tcp_parent = NULL; c->max_tcp_count = 0; + c->cur_tcp_count = 0; c->tcp_handlers = NULL; c->tcp_free = NULL; c->type = comm_tcp; @@ -1810,6 +1816,7 @@ comm_point_create_local(struct comm_base *base, int fd, size_t bufsize, c->tcp_byte_count = 0; c->tcp_parent = NULL; c->max_tcp_count = 0; + c->cur_tcp_count = 0; c->tcp_handlers = NULL; c->tcp_free = NULL; c->type = comm_local; @@ -1857,6 +1864,7 @@ comm_point_create_raw(struct comm_base* base, int fd, int writing, c->tcp_byte_count = 0; c->tcp_parent = NULL; c->max_tcp_count = 0; + c->cur_tcp_count = 0; c->tcp_handlers = NULL; c->tcp_free = NULL; c->type = comm_raw; diff --git a/external/unbound/util/netevent.h b/external/unbound/util/netevent.h index 37322ab93..4b87cdba9 100644 --- a/external/unbound/util/netevent.h +++ b/external/unbound/util/netevent.h @@ -164,6 +164,8 @@ struct comm_point { /* -------- TCP Accept -------- */ /** the number of TCP handlers for this tcp-accept socket */ int max_tcp_count; + /** current number of tcp handler in-use for this accept socket */ + int cur_tcp_count; /** malloced array of tcp handlers for a tcp-accept, of size max_tcp_count. */ struct comm_point** tcp_handlers; diff --git a/external/unbound/util/rtt.c b/external/unbound/util/rtt.c index 4b44fca50..5d86f1337 100644 --- a/external/unbound/util/rtt.c +++ b/external/unbound/util/rtt.c @@ -42,6 +42,8 @@ #include "config.h" #include "util/rtt.h" +/* overwritten by config: infra_cache_min_rtt: */ +int RTT_MIN_TIMEOUT = 50; /** calculate RTO from rtt information */ static int calc_rto(const struct rtt_info* rtt) diff --git a/external/unbound/util/rtt.h b/external/unbound/util/rtt.h index 57e904d14..d6da98606 100644 --- a/external/unbound/util/rtt.h +++ b/external/unbound/util/rtt.h @@ -56,7 +56,7 @@ struct rtt_info { }; /** min retransmit timeout value, in milliseconds */ -#define RTT_MIN_TIMEOUT 50 +extern int RTT_MIN_TIMEOUT; /** max retransmit timeout value, in milliseconds */ #define RTT_MAX_TIMEOUT 120000 diff --git a/external/unbound/validator/autotrust.c b/external/unbound/validator/autotrust.c index 5e1dc4ef3..bb5723468 100644 --- a/external/unbound/validator/autotrust.c +++ b/external/unbound/validator/autotrust.c @@ -57,11 +57,11 @@ #include "services/mesh.h" #include "services/cache/rrset.h" #include "validator/val_kcache.h" -#include "ldns/sbuffer.h" -#include "ldns/wire2str.h" -#include "ldns/str2wire.h" -#include "ldns/keyraw.h" -#include "ldns/rrdef.h" +#include "sldns/sbuffer.h" +#include "sldns/wire2str.h" +#include "sldns/str2wire.h" +#include "sldns/keyraw.h" +#include "sldns/rrdef.h" #include <stdarg.h> #include <ctype.h> diff --git a/external/unbound/validator/val_anchor.c b/external/unbound/validator/val_anchor.c index 3a67fff45..845b54a2e 100644 --- a/external/unbound/validator/val_anchor.c +++ b/external/unbound/validator/val_anchor.c @@ -48,9 +48,9 @@ #include "util/log.h" #include "util/net_help.h" #include "util/config_file.h" -#include "ldns/sbuffer.h" -#include "ldns/rrdef.h" -#include "ldns/str2wire.h" +#include "sldns/sbuffer.h" +#include "sldns/rrdef.h" +#include "sldns/str2wire.h" #ifdef HAVE_GLOB_H #include <glob.h> #endif @@ -882,14 +882,14 @@ assemble_it(struct trust_anchor* ta, size_t num, uint16_t type) memset(pd, 0, sizeof(*pd)); pd->count = num; pd->trust = rrset_trust_ultimate; - pd->rr_len = (size_t*)malloc(num*sizeof(size_t)); + pd->rr_len = (size_t*)reallocarray(NULL, num, sizeof(size_t)); if(!pd->rr_len) { free(pd); free(pkey->rk.dname); free(pkey); return NULL; } - pd->rr_ttl = (time_t*)malloc(num*sizeof(time_t)); + pd->rr_ttl = (time_t*)reallocarray(NULL, num, sizeof(time_t)); if(!pd->rr_ttl) { free(pd->rr_len); free(pd); @@ -897,7 +897,7 @@ assemble_it(struct trust_anchor* ta, size_t num, uint16_t type) free(pkey); return NULL; } - pd->rr_data = (uint8_t**)malloc(num*sizeof(uint8_t*)); + pd->rr_data = (uint8_t**)reallocarray(NULL, num, sizeof(uint8_t*)); if(!pd->rr_data) { free(pd->rr_ttl); free(pd->rr_len); @@ -1020,7 +1020,13 @@ anchors_assemble_rrsets(struct val_anchors* anchors) dname_str(ta->name, b); log_warn("trust anchor %s has no supported algorithms," " the anchor is ignored (check if you need to" - " upgrade unbound and openssl)", b); + " upgrade unbound and " +#ifdef HAVE_LIBRESSL + "libressl" +#else + "openssl" +#endif + ")", b); (void)rbtree_delete(anchors->tree, &ta->node); lock_basic_unlock(&ta->lock); anchors_delfunc(&ta->node, NULL); diff --git a/external/unbound/validator/val_kentry.c b/external/unbound/validator/val_kentry.c index f99f18e89..93fe2145e 100644 --- a/external/unbound/validator/val_kentry.c +++ b/external/unbound/validator/val_kentry.c @@ -45,8 +45,8 @@ #include "util/storage/lookup3.h" #include "util/regional.h" #include "util/net_help.h" -#include "ldns/rrdef.h" -#include "ldns/keyraw.h" +#include "sldns/rrdef.h" +#include "sldns/keyraw.h" size_t key_entry_sizefunc(void* key, void* data) diff --git a/external/unbound/validator/val_neg.c b/external/unbound/validator/val_neg.c index 1d7a5c56e..b1ff8d9a1 100644 --- a/external/unbound/validator/val_neg.c +++ b/external/unbound/validator/val_neg.c @@ -59,8 +59,8 @@ #include "util/config_file.h" #include "services/cache/rrset.h" #include "services/cache/dns.h" -#include "ldns/rrdef.h" -#include "ldns/sbuffer.h" +#include "sldns/rrdef.h" +#include "sldns/sbuffer.h" int val_neg_data_compare(const void* a, const void* b) { diff --git a/external/unbound/validator/val_nsec3.c b/external/unbound/validator/val_nsec3.c index 548daf2bf..80ca4d0ba 100644 --- a/external/unbound/validator/val_nsec3.c +++ b/external/unbound/validator/val_nsec3.c @@ -62,7 +62,7 @@ #include "util/data/msgreply.h" /* we include nsec.h for the bitmap_has_type function */ #include "validator/val_nsec.h" -#include "ldns/sbuffer.h" +#include "sldns/sbuffer.h" /** * This function we get from ldns-compat or from base system diff --git a/external/unbound/validator/val_secalgo.c b/external/unbound/validator/val_secalgo.c index d89675f83..8ed403dfc 100644 --- a/external/unbound/validator/val_secalgo.c +++ b/external/unbound/validator/val_secalgo.c @@ -41,12 +41,13 @@ * and do the library calls (for the crypto library in use). */ #include "config.h" -#include "validator/val_secalgo.h" +/* packed_rrset on top to define enum types (forced by c99 standard) */ #include "util/data/packed_rrset.h" +#include "validator/val_secalgo.h" #include "util/log.h" -#include "ldns/rrdef.h" -#include "ldns/keyraw.h" -#include "ldns/sbuffer.h" +#include "sldns/rrdef.h" +#include "sldns/keyraw.h" +#include "sldns/sbuffer.h" #if !defined(HAVE_SSL) && !defined(HAVE_NSS) #error "Need crypto library to do digital signature cryptography" diff --git a/external/unbound/validator/val_sigcrypt.c b/external/unbound/validator/val_sigcrypt.c index 5a4d0f471..7c643cab1 100644 --- a/external/unbound/validator/val_sigcrypt.c +++ b/external/unbound/validator/val_sigcrypt.c @@ -51,10 +51,10 @@ #include "util/module.h" #include "util/net_help.h" #include "util/regional.h" -#include "ldns/keyraw.h" -#include "ldns/sbuffer.h" -#include "ldns/parseutil.h" -#include "ldns/wire2str.h" +#include "sldns/keyraw.h" +#include "sldns/sbuffer.h" +#include "sldns/parseutil.h" +#include "sldns/wire2str.h" #include <ctype.h> #if !defined(HAVE_SSL) && !defined(HAVE_NSS) @@ -1079,6 +1079,8 @@ int rrset_canonical_equal(struct regional* region, fd.rr_data = fdata; rbtree_init(&sortree1, &canonical_tree_compare); rbtree_init(&sortree2, &canonical_tree_compare); + if(d1->count > RR_COUNT_MAX || d2->count > RR_COUNT_MAX) + return 1; /* protection against integer overflow */ rrs1 = regional_alloc(region, sizeof(struct canon_rr)*d1->count); rrs2 = regional_alloc(region, sizeof(struct canon_rr)*d2->count); if(!rrs1 || !rrs2) return 1; /* alloc failure */ @@ -1135,6 +1137,8 @@ rrset_canonical(struct regional* region, sldns_buffer* buf, sizeof(rbtree_t)); if(!*sortree) return 0; + if(d->count > RR_COUNT_MAX) + return 0; /* integer overflow protection */ rrs = regional_alloc(region, sizeof(struct canon_rr)*d->count); if(!rrs) { *sortree = NULL; diff --git a/external/unbound/validator/val_utils.c b/external/unbound/validator/val_utils.c index ecf2dfaf0..475b0c905 100644 --- a/external/unbound/validator/val_utils.c +++ b/external/unbound/validator/val_utils.c @@ -846,6 +846,18 @@ val_fill_reply(struct reply_info* chase, struct reply_info* orig, chase->ar_numrrsets; } +void val_reply_remove_auth(struct reply_info* rep, size_t index) +{ + log_assert(index < rep->rrset_count); + log_assert(index >= rep->an_numrrsets); + log_assert(index < rep->an_numrrsets+rep->ns_numrrsets); + memmove(rep->rrsets+index, rep->rrsets+index+1, + sizeof(struct ub_packed_rrset_key*)* + (rep->rrset_count - index - 1)); + rep->ns_numrrsets--; + rep->rrset_count--; +} + void val_check_nonsecure(struct val_env* ve, struct reply_info* rep) { diff --git a/external/unbound/validator/val_utils.h b/external/unbound/validator/val_utils.h index b0344eff7..cdb87697e 100644 --- a/external/unbound/validator/val_utils.h +++ b/external/unbound/validator/val_utils.h @@ -295,6 +295,13 @@ void val_fill_reply(struct reply_info* chase, struct reply_info* orig, size_t cname_skip, uint8_t* name, size_t len, uint8_t* signer); /** + * Remove rrset with index from reply, from the authority section. + * @param rep: reply to remove it from. + * @param index: rrset to remove, must be in the authority section. + */ +void val_reply_remove_auth(struct reply_info* rep, size_t index); + +/** * Remove all unsigned or non-secure status rrsets from NS and AR sections. * So that unsigned data does not get let through to clients, when we have * found the data to be secure. diff --git a/external/unbound/validator/validator.c b/external/unbound/validator/validator.c index 9d5d5c390..a02525fee 100644 --- a/external/unbound/validator/validator.c +++ b/external/unbound/validator/validator.c @@ -58,8 +58,8 @@ #include "util/regional.h" #include "util/config_file.h" #include "util/fptr_wlist.h" -#include "ldns/rrdef.h" -#include "ldns/wire2str.h" +#include "sldns/rrdef.h" +#include "sldns/wire2str.h" /* forward decl for cache response and normal super inform calls of a DS */ static void process_ds_response(struct module_qstate* qstate, @@ -226,6 +226,8 @@ val_new_getmsg(struct module_qstate* qstate, struct val_qstate* vq) sizeof(struct reply_info) - sizeof(struct rrset_ref)); if(!vq->chase_reply) return NULL; + if(vq->orig_msg->rep->rrset_count > RR_COUNT_MAX) + return NULL; /* protect against integer overflow */ vq->chase_reply->rrsets = regional_alloc_init(qstate->region, vq->orig_msg->rep->rrsets, sizeof(struct ub_packed_rrset_key*) * vq->orig_msg->rep->rrset_count); @@ -574,6 +576,61 @@ detect_wrongly_truncated(struct reply_info* rep) return 1; } +/** + * For messages that are not referrals, if the chase reply contains an + * unsigned NS record in the authority section it could have been + * inserted by a (BIND) forwarder that thinks the zone is insecure, and + * that has an NS record without signatures in cache. Remove the NS + * record since the reply does not hinge on that record (in the authority + * section), but do not remove it if it removes the last record from the + * answer+authority sections. + * @param chase_reply: the chased reply, we have a key for this contents, + * so we should have signatures for these rrsets and not having + * signatures means it will be bogus. + * @param orig_reply: original reply, remove NS from there as well because + * we cannot mark the NS record as DNSSEC valid because it is not + * validated by signatures. + */ +static void +remove_spurious_authority(struct reply_info* chase_reply, + struct reply_info* orig_reply) +{ + size_t i, found = 0; + int remove = 0; + /* if no answer and only 1 auth RRset, do not remove that one */ + if(chase_reply->an_numrrsets == 0 && chase_reply->ns_numrrsets == 1) + return; + /* search authority section for unsigned NS records */ + for(i = chase_reply->an_numrrsets; + i < chase_reply->an_numrrsets+chase_reply->ns_numrrsets; i++) { + struct packed_rrset_data* d = (struct packed_rrset_data*) + chase_reply->rrsets[i]->entry.data; + if(ntohs(chase_reply->rrsets[i]->rk.type) == LDNS_RR_TYPE_NS + && d->rrsig_count == 0) { + found = i; + remove = 1; + break; + } + } + /* see if we found the entry */ + if(!remove) return; + log_rrset_key(VERB_ALGO, "Removing spurious unsigned NS record " + "(likely inserted by forwarder)", chase_reply->rrsets[found]); + + /* find rrset in orig_reply */ + for(i = orig_reply->an_numrrsets; + i < orig_reply->an_numrrsets+orig_reply->ns_numrrsets; i++) { + if(ntohs(orig_reply->rrsets[i]->rk.type) == LDNS_RR_TYPE_NS + && query_dname_compare(orig_reply->rrsets[i]->rk.dname, + chase_reply->rrsets[found]->rk.dname) == 0) { + /* remove from orig_msg */ + val_reply_remove_auth(orig_reply, i); + break; + } + } + /* remove rrset from chase_reply */ + val_reply_remove_auth(chase_reply, found); +} /** * Given a "positive" response -- a response that contains an answer to the @@ -1642,6 +1699,8 @@ processValidate(struct module_qstate* qstate, struct val_qstate* vq, } subtype = val_classify_response(qstate->query_flags, &qstate->qinfo, &vq->qchase, vq->orig_msg->rep, vq->rrset_skip); + if(subtype != VAL_CLASS_REFERRAL) + remove_spurious_authority(vq->chase_reply, vq->orig_msg->rep); /* check signatures in the message; * answer and authority must be valid, additional is only checked. */ @@ -2295,7 +2354,7 @@ primeResponseToKE(struct ub_packed_rrset_key* dnskey_rrset, struct key_entry_key* kkey = NULL; enum sec_status sec = sec_status_unchecked; char* reason = NULL; - int downprot = 1; + int downprot = qstate->env->cfg->harden_algo_downgrade; if(!dnskey_rrset) { log_nametypeclass(VERB_OPS, "failed to prime trust anchor -- " diff --git a/external/unbound/winrc/README.txt b/external/unbound/winrc/README.txt index e40204bb0..d36fff14c 100644 --- a/external/unbound/winrc/README.txt +++ b/external/unbound/winrc/README.txt @@ -10,7 +10,7 @@ See LICENSE for the license text file. Unbound is a recursive DNS server. It does caching, full recursion, stub
recursion, DNSSEC validation, NSEC3, IPv6. More information can be found
at the http://unbound.net site. Unbound has been built and tested on
-Windows XP, Vista and 7.
+Windows XP, Vista, 7 and 8.
At http://unbound.net/documentation is an install and configuration manual
for windows.
diff --git a/external/unbound/winrc/anchor-update.c b/external/unbound/winrc/anchor-update.c index 2b22e0142..13d44fda6 100644 --- a/external/unbound/winrc/anchor-update.c +++ b/external/unbound/winrc/anchor-update.c @@ -41,9 +41,9 @@ */ #include "config.h" #include "libunbound/unbound.h" -#include "ldns/rrdef.h" -#include "ldns/pkthdr.h" -#include "ldns/wire2str.h" +#include "sldns/rrdef.h" +#include "sldns/pkthdr.h" +#include "sldns/wire2str.h" /** usage */ static void diff --git a/include/INode.h b/include/INode.h index d9ab27fc1..c7c559d63 100644 --- a/include/INode.h +++ b/include/INode.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/include/IWallet.h b/include/IWallet.h index 5c4925d55..bb8ac0607 100644 --- a/include/IWallet.h +++ b/include/IWallet.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 43c5740af..e454b92fd 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # @@ -89,11 +89,17 @@ endfunction () add_subdirectory(common) add_subdirectory(crypto) add_subdirectory(cryptonote_core) +add_subdirectory(blockchain_db) add_subdirectory(mnemonics) add_subdirectory(rpc) add_subdirectory(wallet) +add_subdirectory(p2p) +add_subdirectory(cryptonote_protocol) add_subdirectory(connectivity_tool) add_subdirectory(miner) add_subdirectory(simplewallet) +add_subdirectory(daemonizer) add_subdirectory(daemon) + +add_subdirectory(blockchain_converter) diff --git a/src/Changelog b/src/Changelog new file mode 100644 index 000000000..84da8068b --- /dev/null +++ b/src/Changelog @@ -0,0 +1,77 @@ + +[F] - fixes a bug from existing official version +[B] - important bug discovered in official version +[T] - testes +[n] - not used now (only in history), replaced or not accepted to master + +- adding support for user-local installed compiler (e.g. non-root users on older systems to use new compiler) +instructions to use monero on Debian 7 (build clang + deps) +- [n] faster build: cmake with fast build, and with cotire (precompiled headers), and limited targets +- fixed cmake local compiler e.g. BOOST_IGNORE_SYSTEM_PATHS after merges with the other new CMake + +- faster rebuild: separate out not-template base for main network classes: connection_basic.cpp class connection_basic +- faster rebuild: separate out hooks: connection_basic::do_send_handler_write() do_send_handler_write_from_queue() +- created library/cmake for the new no-templates-only networking code - libp2p + +- added command exit without DB save (only for testers) +- added option --no-igd to not wait for IGD on start (faster testing) + +- imported logging/debug tools from otshell, with macros like _note() _info() _warn() +- logging: added proper thread locking; showing thread number {1} in the output +- logging: also showing process number (PID) in short form (1,2,3,...) to debug forking daemon etc +- logging: fixed colors for normal windows text console (currently they do not work in msys windows text console) +- logging: added channels to have separate files created +- logging: option (compile-time) to debug the logging system itself +- logging: console colors work on windows/msys + +- created network throttle class + throttle manager +- cmdline options for network throttle class (limit-rate-up limit-rate-down limit-rate) +- option and command to in fact limit (outgoing) peers --out-peers out_peers [currently not working! after merge] [TODO] +- rpc commands for network throttle class (limit limit_up limit_down) +- setting ToS socket flag with option --tos-flag + +- connection type support, so that RPC connection is excluded from network limits + +- gather and save throughput statistics (speed vs limit) into data file +- statistics of details to tune implementation: sleep in network threads, number of peers +- optimized statistics: accumulate sum and limit disk writes + +(dr. monero code is not published in this commit, but prepared) +- dr. monero show the collected statistics (in real time) +- dr. monero many windows opened at once from predefined list +- dr. monero auto scale; selectable average window +- dr. monero colors, show both samples and average values (transparent) +- dr. monero showing current configured network limit (goal) +- dr. monero show date/git commit/comments for better info in screenshots +- dr. monero optimize speed and memory usage to handle files from long tests + +- [F] found few UBs in code like wrong initialization order +- [B] found and partial debugged deadlocks on existing program (faster testing) + +- [B] found locking problem with connection close race + +- debug option --test-drop-download to test no-DB version without running out of RAM +- also option --test-drop-download-heigh to start drop only after certain height + +- added again fast exit command (but it does not work too well yet. ONLY FOR DEVELOPERS!) + +- created new from-scratch clean network engine model (a new program/library) +- connected the logging, the network throttles and all code to network model +- [T] run experiments in fully controlled and very fast environment + +- fine tuned parameters of throttles (sleep, 3 window sizes) +- code to estimated current network-transfer size of block based get_avg_block_size() +- [T] tested sleep vs request-size limiting of downloads (request less or more then 200 blocks) +- for now we use sleep method + +- documented throttle classes and few other classes +- adjusted comment headers, copyrights, added the @monero tag to doxygen denoting monero maintainer/contact for not-monero code +- doxygen scripts fixed, tested and generated actual website + +- [n] rebased most of above when change to branch development was decided +- [n] again rebase when we returned to developing against branch master +- finall version was tested by developer on Windows 64-bit, Debian, Ubuntu +- fixed clock compilation problems for freebsd and mac os x; Fixed related problem again on mac os x. + + + diff --git a/src/blockchain_converter/CMakeLists.txt b/src/blockchain_converter/CMakeLists.txt new file mode 100644 index 000000000..660650980 --- /dev/null +++ b/src/blockchain_converter/CMakeLists.txt @@ -0,0 +1,115 @@ +# Copyright (c) 2014-2015, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set(blockchain_converter_sources + blockchain_converter.cpp + ) + +set(blockchain_converter_private_headers) + +bitmonero_private_headers(blockchain_converter + ${blockchain_converter_private_headers}) + +set(blockchain_import_sources + blockchain_import.cpp + ) + +set(blockchain_import_private_headers + import.h + fake_core.h + ) + +bitmonero_private_headers(blockchain_import + ${blockchain_import_private_headers}) + +set(blockchain_export_sources + blockchain_export.cpp + ) + +set(blockchain_export_private_headers + import.h + blockchain_export.h + ) + +bitmonero_private_headers(blockchain_export + ${blockchain_export_private_headers}) + + + +if (BLOCKCHAIN_DB STREQUAL DB_LMDB) +bitmonero_add_executable(blockchain_converter + ${blockchain_converter_sources} + ${blockchain_converter_private_headers}) + +target_link_libraries(blockchain_converter + LINK_PRIVATE + cryptonote_core + p2p + blockchain_db + ${CMAKE_THREAD_LIBS_INIT}) + +add_dependencies(blockchain_converter + version) +set_property(TARGET blockchain_converter + PROPERTY + OUTPUT_NAME "blockchain_converter") +endif () + +bitmonero_add_executable(blockchain_import + ${blockchain_import_sources} + ${blockchain_import_private_headers}) + +target_link_libraries(blockchain_import + LINK_PRIVATE + cryptonote_core + blockchain_db + p2p + ${CMAKE_THREAD_LIBS_INIT}) + +add_dependencies(blockchain_import + version) +set_property(TARGET blockchain_import + PROPERTY + OUTPUT_NAME "blockchain_import") + +bitmonero_add_executable(blockchain_export + ${blockchain_export_sources} + ${blockchain_export_private_headers}) + +target_link_libraries(blockchain_export + LINK_PRIVATE + cryptonote_core + p2p + blockchain_db + ${CMAKE_THREAD_LIBS_INIT}) + +add_dependencies(blockchain_export + version) +set_property(TARGET blockchain_export + PROPERTY + OUTPUT_NAME "blockchain_export") diff --git a/src/blockchain_converter/README.md b/src/blockchain_converter/README.md new file mode 100644 index 000000000..00160c6b9 --- /dev/null +++ b/src/blockchain_converter/README.md @@ -0,0 +1,54 @@ + +For importing into the LMDB database, compile with `DATABASE=lmdb` + +e.g. + +`DATABASE=lmdb make release` + +This is also the default compile setting on the blockchain branch. + +By default, the exporter will use the original in-memory database (blockchain.bin) as its source. +This default is to make migrating to an LMDB database easy, without having to recompile anything. +To change the source, adjust `SOURCE_DB` in `src/blockchain_converter/blockchain_export.h` according to the comments. + +# Usage: + +See also each utility's "--help" option. + +## Export an existing in-memory database + +`$ blockchain_export` + +This loads the existing blockchain, for whichever database type it was compiled for, and exports it to `$MONERO_DATA_DIR/export/blockchain.raw` + +## Import the exported file + +`$ blockchain_import` + +This imports blocks from `$MONERO_DATA_DIR/export/blockchain.raw` into the current database. + +Defaults: `--batch on`, `--batch size 20000`, `--verify on` + +Batch size refers to number of blocks and can be adjusted for performance based on available RAM. + +Verification should only be turned off if importing from a trusted blockchain. + +```bash +# use default settings to import blockchain.raw into database +$ blockchain_import + +# fast import with large batch size, verification off +$ blockchain_import --batch-size 100000 --verify off + +# LMDB flags can be set by appending them to the database type: +# flags: nosync, nometasync, writemap, mapasync +$ blockchain_import --database lmdb#nosync +$ blockchain_import --database lmdb#nosync,nometasync +``` + +## Blockchain converter with batching +`blockchain_converter` has also been updated and includes batching for faster writes. However, on lower RAM systems, this will be slower than using the exporter and importer utilities. The converter needs to keep the blockchain in memory for the duration of the conversion, like the original bitmonerod, thus leaving less memory available to the destination database to operate. + +```bash +$ blockchain_converter --batch on --batch-size 20000 +``` diff --git a/src/blockchain_converter/blockchain_converter.cpp b/src/blockchain_converter/blockchain_converter.cpp new file mode 100644 index 000000000..c3c6d0918 --- /dev/null +++ b/src/blockchain_converter/blockchain_converter.cpp @@ -0,0 +1,299 @@ +// Copyright (c) 2014-2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "include_base_utils.h" +#include "common/util.h" +#include "warnings.h" +#include "crypto/crypto.h" +#include "cryptonote_config.h" +#include "cryptonote_core/cryptonote_format_utils.h" +#include "misc_language.h" +#include "cryptonote_core/blockchain_storage.h" +#include "blockchain_db/blockchain_db.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/lmdb/db_lmdb.h" +#include "cryptonote_core/tx_pool.h" +#include "common/command_line.h" +#include "serialization/json_utils.h" +#include "include_base_utils.h" +#include "version.h" +#include <iostream> + + +unsigned int epee::g_test_dbg_lock_sleep = 0; + +namespace +{ + +// CONFIG +bool opt_batch = true; +bool opt_resume = true; +bool opt_testnet = false; + +// number of blocks per batch transaction +// adjustable through command-line argument according to available RAM +uint64_t db_batch_size = 20000; + +} + +namespace po = boost::program_options; + +using namespace cryptonote; +using namespace epee; + +struct fake_core +{ + Blockchain dummy; + tx_memory_pool m_pool; + + blockchain_storage m_storage; + +#if !defined(BLOCKCHAIN_DB) + // for multi_db_runtime: + fake_core(const boost::filesystem::path &path, const bool use_testnet) : dummy(m_pool), m_pool(&dummy), m_storage(m_pool) +#else + // for multi_db_compile: + fake_core(const boost::filesystem::path &path, const bool use_testnet) : dummy(m_pool), m_pool(dummy), m_storage(&m_pool) +#endif + { + m_pool.init(path.string()); + m_storage.init(path.string(), use_testnet); + } +}; + +int main(int argc, char* argv[]) +{ + uint64_t height = 0; + uint64_t start_block = 0; + uint64_t end_block = 0; + uint64_t num_blocks = 0; + + boost::filesystem::path default_data_path {tools::get_default_data_dir()}; + boost::filesystem::path default_testnet_data_path {default_data_path / "testnet"}; + + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor<uint32_t> arg_log_level = {"log-level", "", LOG_LEVEL_0}; + const command_line::arg_descriptor<uint64_t> arg_batch_size = {"batch-size", "", db_batch_size}; + const command_line::arg_descriptor<bool> arg_testnet_on = { + "testnet" + , "Run on testnet." + , opt_testnet + }; + const command_line::arg_descriptor<uint64_t> arg_block_number = + {"block-number", "Number of blocks (default: use entire source blockchain)", + 0}; + + command_line::add_arg(desc_cmd_sett, command_line::arg_data_dir, default_data_path.string()); + command_line::add_arg(desc_cmd_sett, command_line::arg_testnet_data_dir, default_testnet_data_path.string()); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_batch_size); + command_line::add_arg(desc_cmd_sett, arg_testnet_on); + command_line::add_arg(desc_cmd_sett, arg_block_number); + + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + const command_line::arg_descriptor<bool> arg_batch = {"batch", + "Batch transactions for faster import", true}; + const command_line::arg_descriptor<bool> arg_resume = {"resume", + "Resume from current height if output database already exists", true}; + + // call add_options() directly for these arguments since command_line helpers + // support only boolean switch, not boolean argument + desc_cmd_sett.add_options() + (arg_batch.name, make_semantic(arg_batch), arg_batch.description) + (arg_resume.name, make_semantic(arg_resume), arg_resume.description) + ; + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + po::store(po::parse_command_line(argc, argv, desc_options), vm); + po::notify(vm); + + return true; + }); + if (!r) + return 1; + + int log_level = command_line::get_arg(vm, arg_log_level); + opt_batch = command_line::get_arg(vm, arg_batch); + opt_resume = command_line::get_arg(vm, arg_resume); + db_batch_size = command_line::get_arg(vm, arg_batch_size); + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + if (! opt_batch && ! vm["batch-size"].defaulted()) + { + std::cerr << "Error: batch-size set, but batch option not enabled" << ENDL; + return 1; + } + if (! db_batch_size) + { + std::cerr << "Error: batch-size must be > 0" << ENDL; + return 1; + } + + log_space::get_set_log_detalisation_level(true, log_level); + log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL); + LOG_PRINT_L0("Starting..."); + + std::string src_folder; + opt_testnet = command_line::get_arg(vm, arg_testnet_on); + auto data_dir_arg = opt_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir; + src_folder = command_line::get_arg(vm, data_dir_arg); + boost::filesystem::path dest_folder(src_folder); + + num_blocks = command_line::get_arg(vm, arg_block_number); + + if (opt_batch) + { + LOG_PRINT_L0("batch: " << std::boolalpha << opt_batch << std::noboolalpha + << " batch size: " << db_batch_size); + } + else + { + LOG_PRINT_L0("batch: " << std::boolalpha << opt_batch << std::noboolalpha); + } + LOG_PRINT_L0("resume: " << std::boolalpha << opt_resume << std::noboolalpha); + LOG_PRINT_L0("testnet: " << std::boolalpha << opt_testnet << std::noboolalpha); + + fake_core c(src_folder, opt_testnet); + + height = c.m_storage.get_current_blockchain_height(); + + BlockchainDB *blockchain; + blockchain = new BlockchainLMDB(opt_batch); + dest_folder /= blockchain->get_db_name(); + LOG_PRINT_L0("Source blockchain: " << src_folder); + LOG_PRINT_L0("Dest blockchain: " << dest_folder.string()); + LOG_PRINT_L0("Opening dest blockchain (BlockchainDB " << blockchain->get_db_name() << ")"); + blockchain->open(dest_folder.string()); + LOG_PRINT_L0("Source blockchain height: " << height); + LOG_PRINT_L0("Dest blockchain height: " << blockchain->height()); + + if (opt_resume) + // next block number to add is same as current height + start_block = blockchain->height(); + + if (! num_blocks || (start_block + num_blocks > height)) + end_block = height - 1; + else + end_block = start_block + num_blocks - 1; + + LOG_PRINT_L0("start height: " << start_block+1 << " stop height: " << + end_block+1); + + if (start_block > end_block) + { + LOG_PRINT_L0("Finished: no blocks to add"); + delete blockchain; + return 0; + } + + if (opt_batch) + blockchain->batch_start(); + uint64_t i = 0; + for (i = start_block; i < end_block + 1; ++i) + { + // block: i height: i+1 end height: end_block + 1 + if ((i+1) % 10 == 0) + { + std::cout << "\r \r" << "height " << i+1 << "/" << + end_block+1 << " (" << (i+1)*100/(end_block+1)<< "%)" << std::flush; + } + // for debugging: + // std::cout << "height " << i+1 << "/" << end_block+1 + // << " ((" << i+1 << ")*100/(end_block+1))" << "%)" << ENDL; + + block b = c.m_storage.get_block(i); + size_t bsize = c.m_storage.get_block_size(i); + difficulty_type bdiff = c.m_storage.get_block_cumulative_difficulty(i); + uint64_t bcoins = c.m_storage.get_block_coins_generated(i); + std::vector<transaction> txs; + std::vector<crypto::hash> missed; + + c.m_storage.get_transactions(b.tx_hashes, txs, missed); + if (missed.size()) + { + std::cout << ENDL; + std::cerr << "Missed transaction(s) for block at height " << i + 1 << ", exiting" << ENDL; + delete blockchain; + return 1; + } + + try + { + blockchain->add_block(b, bsize, bdiff, bcoins, txs); + + if (opt_batch) + { + if ((i < end_block) && ((i + 1) % db_batch_size == 0)) + { + std::cout << "\r \r"; + std::cout << "[- batch commit at height " << i + 1 << " -]" << ENDL; + blockchain->batch_stop(); + blockchain->batch_start(); + std::cout << ENDL; + blockchain->show_stats(); + } + } + } + catch (const std::exception& e) + { + std::cout << ENDL; + std::cerr << "Error adding block " << i << " to new blockchain: " << e.what() << ENDL; + delete blockchain; + return 2; + } + } + if (opt_batch) + { + std::cout << "\r \r" << "height " << i << "/" << + end_block+1 << " (" << (i)*100/(end_block+1)<< "%)" << std::flush; + std::cout << ENDL; + std::cout << "[- batch commit at height " << i << " -]" << ENDL; + blockchain->batch_stop(); + } + std::cout << ENDL; + blockchain->show_stats(); + std::cout << "Finished at height: " << i << " block: " << i-1 << ENDL; + + delete blockchain; + return 0; +} diff --git a/src/blockchain_converter/blockchain_export.cpp b/src/blockchain_converter/blockchain_export.cpp new file mode 100644 index 000000000..dc5c7cadc --- /dev/null +++ b/src/blockchain_converter/blockchain_export.cpp @@ -0,0 +1,396 @@ +// Copyright (c) 2014-2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <algorithm> +#include <cstdio> +#include <fstream> +#include <boost/iostreams/copy.hpp> +#include <atomic> + +#include <boost/archive/binary_oarchive.hpp> +#include <boost/archive/binary_iarchive.hpp> +#include <boost/iostreams/stream_buffer.hpp> +#include <boost/iostreams/stream.hpp> +#include <boost/iostreams/device/back_inserter.hpp> +#include <boost/iostreams/filtering_streambuf.hpp> +#include <boost/iostreams/filter/bzip2.hpp> +#include "common/command_line.h" +#include "version.h" +#include "blockchain_export.h" +#include "cryptonote_core/cryptonote_boost_serialization.h" + +#include "import.h" + +unsigned int epee::g_test_dbg_lock_sleep = 0; + +static int max_chunk = 0; +static size_t height; + +namespace po = boost::program_options; + +using namespace cryptonote; +using namespace epee; + +bool BlockchainExport::open(const boost::filesystem::path& dir_path) +{ + if (boost::filesystem::exists(dir_path)) + { + if (!boost::filesystem::is_directory(dir_path)) + { + LOG_PRINT_RED_L0("export directory path is a file: " << dir_path); + return false; + } + } + else + { + if (!boost::filesystem::create_directory(dir_path)) + { + LOG_PRINT_RED_L0("Failed to create directory " << dir_path); + return false; + } + } + + std::string file_path = (dir_path / BLOCKCHAIN_RAW).string(); + m_raw_data_file = new std::ofstream(); + m_raw_data_file->open(file_path , std::ios_base::binary | std::ios_base::out| std::ios::trunc); + if (m_raw_data_file->fail()) + return false; + + m_output_stream = new boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>>(m_buffer); + m_raw_archive = new boost::archive::binary_oarchive(*m_output_stream); + if (m_raw_archive == NULL) + return false; + + return true; +} + +void BlockchainExport::flush_chunk() +{ + m_output_stream->flush(); + char buffer[STR_LENGTH_OF_INT + 1]; + int chunk_size = (int) m_buffer.size(); + if (chunk_size > BUFFER_SIZE) + { + LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE); + } + sprintf(buffer, STR_FORMAT_OF_INT, chunk_size); + m_raw_data_file->write(buffer, STR_LENGTH_OF_INT); + if (max_chunk < chunk_size) + { + max_chunk = chunk_size; + } + long pos_before = m_raw_data_file->tellp(); + std::copy(m_buffer.begin(), m_buffer.end(), std::ostreambuf_iterator<char>(*m_raw_data_file)); + m_raw_data_file->flush(); + long pos_after = m_raw_data_file->tellp(); + long num_chars_written = pos_after - pos_before; + if ((int) num_chars_written != chunk_size) + { + LOG_PRINT_RED_L0("INTERNAL ERROR: num chars wrote NEQ buffer size. height = " << height); + } + + m_buffer.clear(); + delete m_raw_archive; + delete m_output_stream; + m_output_stream = new boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>>(m_buffer); + m_raw_archive = new boost::archive::binary_oarchive(*m_output_stream); +} + +void BlockchainExport::serialize_block_to_text_buffer(const block& block) +{ + *m_raw_archive << block; +} + +void BlockchainExport::buffer_serialize_tx(const transaction& tx) +{ + *m_raw_archive << tx; +} + +void BlockchainExport::buffer_write_num_txs(const std::list<transaction> txs) +{ + int n = txs.size(); + *m_raw_archive << n; +} + +void BlockchainExport::write_block(block& block) +{ + serialize_block_to_text_buffer(block); + std::list<transaction> txs; + + uint64_t block_height = boost::get<txin_gen>(block.miner_tx.vin.front()).height; + + // put coinbase transaction first + transaction coinbase_tx = block.miner_tx; + crypto::hash coinbase_tx_hash = get_transaction_hash(coinbase_tx); +#if SOURCE_DB == DB_MEMORY + const transaction* cb_tx_full = m_blockchain_storage->get_tx(coinbase_tx_hash); +#else + transaction cb_tx_full = m_blockchain_storage->get_db().get_tx(coinbase_tx_hash); +#endif + +#if SOURCE_DB == DB_MEMORY + if (cb_tx_full != NULL) + { + txs.push_back(*cb_tx_full); + } +#else + // TODO: should check and abort if cb_tx_full equals null_hash? + txs.push_back(cb_tx_full); +#endif + + // now add all regular transactions + BOOST_FOREACH(const auto& tx_id, block.tx_hashes) + { +#if SOURCE_DB == DB_MEMORY + const transaction* tx = m_blockchain_storage->get_tx(tx_id); +#else + transaction tx = m_blockchain_storage->get_db().get_tx(tx_id); +#endif + +#if SOURCE_DB == DB_MEMORY + if(tx == NULL) + { + if (! m_tx_pool) + throw std::runtime_error("Aborting: tx == NULL, so memory pool required to get tx, but memory pool isn't enabled"); + else + { + transaction tx; + if(m_tx_pool->get_transaction(tx_id, tx)) + txs.push_back(tx); + else + throw std::runtime_error("Aborting: tx not found in pool"); + } + } + else + txs.push_back(*tx); +#else + txs.push_back(tx); +#endif + } + + // serialize all txs to the persistant storage + buffer_write_num_txs(txs); + BOOST_FOREACH(const auto& tx, txs) + { + buffer_serialize_tx(tx); + } + + // These three attributes are currently necessary for a fast import that adds blocks without verification. + bool include_extra_block_data = true; + if (include_extra_block_data) + { +#if SOURCE_DB == DB_MEMORY + size_t block_size = m_blockchain_storage->get_block_size(block_height); +#else + size_t block_size = m_blockchain_storage->get_db().get_block_size(block_height); +#endif +#if SOURCE_DB == DB_MEMORY + difficulty_type cumulative_difficulty = m_blockchain_storage->get_block_cumulative_difficulty(block_height); +#else + difficulty_type cumulative_difficulty = m_blockchain_storage->get_db().get_block_cumulative_difficulty(block_height); +#endif +#if SOURCE_DB == DB_MEMORY + uint64_t coins_generated = m_blockchain_storage->get_block_coins_generated(block_height); +#else + // TODO TEST to verify that this is the equivalent. make sure no off-by-one error with block height vs block number + uint64_t coins_generated = m_blockchain_storage->get_db().get_block_already_generated_coins(block_height); +#endif + + *m_raw_archive << block_size; + *m_raw_archive << cumulative_difficulty; + *m_raw_archive << coins_generated; + } +} + +bool BlockchainExport::BlockchainExport::close() +{ + if (m_raw_data_file->fail()) + return false; + + m_raw_data_file->flush(); + delete m_raw_archive; + delete m_output_stream; + delete m_raw_data_file; + return true; +} + + +#if SOURCE_DB == DB_MEMORY +bool BlockchainExport::store_blockchain_raw(blockchain_storage* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_dir, uint64_t requested_block_height) +#else +bool BlockchainExport::store_blockchain_raw(Blockchain* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_dir, uint64_t requested_block_height) +#endif +{ + uint64_t block_height = 0; + m_blockchain_storage = _blockchain_storage; + m_tx_pool = _tx_pool; + uint64_t progress_interval = 100; + std::string refresh_string = "\r \r"; + LOG_PRINT_L0("Storing blocks raw data..."); + if (!BlockchainExport::open(output_dir)) + { + LOG_PRINT_RED_L0("failed to open raw file for write"); + return false; + } + block b; + LOG_PRINT_L0("source blockchain height: " << m_blockchain_storage->get_current_blockchain_height()); + LOG_PRINT_L0("requested block height: " << requested_block_height); + if ((requested_block_height > 0) && (requested_block_height < m_blockchain_storage->get_current_blockchain_height())) + block_height = requested_block_height; + else + { + block_height = m_blockchain_storage->get_current_blockchain_height(); + LOG_PRINT_L0("Using block height of source blockchain: " << block_height); + } + for (height=0; height < block_height; ++height) + { + crypto::hash hash = m_blockchain_storage->get_block_id_by_height(height); + m_blockchain_storage->get_block_by_hash(hash, b); + write_block(b); + if (height % NUM_BLOCKS_PER_CHUNK == 0) { + flush_chunk(); + } + if (height % progress_interval == 0) { + std::cout << refresh_string; + std::cout << "height " << height << "/" << block_height << std::flush; + } + } + if (height % NUM_BLOCKS_PER_CHUNK != 0) + { + flush_chunk(); + } + std::cout << refresh_string; + std::cout << "height " << height << "/" << block_height << ENDL; + + LOG_PRINT_L0("longest chunk was " << max_chunk << " bytes"); + return BlockchainExport::close(); +} + + +int main(int argc, char* argv[]) +{ + uint32_t log_level = 0; + uint64_t block_height = 0; + std::string import_filename = BLOCKCHAIN_RAW; + + boost::filesystem::path default_data_path {tools::get_default_data_dir()}; + boost::filesystem::path default_testnet_data_path {default_data_path / "testnet"}; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor<uint32_t> arg_log_level = {"log-level", "", log_level}; + const command_line::arg_descriptor<uint64_t> arg_block_height = {"block-number", "", block_height}; + const command_line::arg_descriptor<bool> arg_testnet_on = { + "testnet" + , "Run on testnet." + , false + }; + + + command_line::add_arg(desc_cmd_sett, command_line::arg_data_dir, default_data_path.string()); + command_line::add_arg(desc_cmd_sett, command_line::arg_testnet_data_dir, default_testnet_data_path.string()); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_block_height); + command_line::add_arg(desc_cmd_sett, arg_testnet_on); + + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + po::store(po::parse_command_line(argc, argv, desc_options), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + log_level = command_line::get_arg(vm, arg_log_level); + block_height = command_line::get_arg(vm, arg_block_height); + + log_space::get_set_log_detalisation_level(true, log_level); + log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL); + LOG_PRINT_L0("Starting..."); + LOG_PRINT_L0("Setting log level = " << log_level); + + bool opt_testnet = command_line::get_arg(vm, arg_testnet_on); + + std::string m_config_folder; + + auto data_dir_arg = opt_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir; + m_config_folder = command_line::get_arg(vm, data_dir_arg); + boost::filesystem::path output_dir {m_config_folder}; + output_dir /= "export"; + LOG_PRINT_L0("Export directory: " << output_dir.string()); + + // If we wanted to use the memory pool, we would set up a fake_core. + +#if SOURCE_DB == DB_MEMORY + // blockchain_storage* core_storage = NULL; + // tx_memory_pool m_mempool(*core_storage); // is this fake anyway? just passing in NULL! so m_mempool can't be used anyway, right? + // core_storage = new blockchain_storage(&m_mempool); + + blockchain_storage* core_storage = new blockchain_storage(NULL); + LOG_PRINT_L0("Initializing source blockchain (in-memory database)"); + r = core_storage->init(m_config_folder, opt_testnet); +#else + // Use Blockchain instead of lower-level BlockchainDB for two reasons: + // 1. Blockchain has the init() method for easy setup + // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() + // + // cannot match blockchain_storage setup above with just one line, + // e.g. + // Blockchain* core_storage = new Blockchain(NULL); + // because unlike blockchain_storage constructor, which takes a pointer to + // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. + LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); + Blockchain* core_storage = NULL; + tx_memory_pool m_mempool(*core_storage); + core_storage = new Blockchain(m_mempool); + r = core_storage->init(m_config_folder, opt_testnet); +#endif + + CHECK_AND_ASSERT_MES(r, false, "Failed to initialize source blockchain storage"); + LOG_PRINT_L0("Source blockchain storage initialized OK"); + LOG_PRINT_L0("Exporting blockchain raw data..."); + + BlockchainExport be; + r = be.store_blockchain_raw(core_storage, NULL, output_dir, block_height); + CHECK_AND_ASSERT_MES(r, false, "Failed to export blockchain raw data"); + LOG_PRINT_L0("Blockchain raw data exported OK"); +} diff --git a/src/blockchain_converter/blockchain_export.h b/src/blockchain_converter/blockchain_export.h new file mode 100644 index 000000000..43e25c039 --- /dev/null +++ b/src/blockchain_converter/blockchain_export.h @@ -0,0 +1,87 @@ +// Copyright (c) 2014-2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <boost/archive/binary_oarchive.hpp> +#include <boost/iostreams/stream_buffer.hpp> +#include <boost/iostreams/stream.hpp> +#include <boost/iostreams/device/back_inserter.hpp> +#include "cryptonote_core/cryptonote_basic.h" +#include "cryptonote_core/blockchain_storage.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/lmdb/db_lmdb.h" + +// CONFIG: choose one of the three #define's +// +// DB_MEMORY is a sensible default for users migrating to LMDB, as it allows +// the exporter to use the in-memory blockchain while the other binaries +// work with LMDB, without recompiling anything. +// +#define SOURCE_DB DB_MEMORY +// #define SOURCE_DB DB_LMDB +// to use global compile-time setting (DB_MEMORY or DB_LMDB): +// #define SOURCE_DB BLOCKCHAIN_DB + +using namespace cryptonote; + +class BlockchainExport +{ +public: +#if SOURCE_DB == DB_MEMORY + bool store_blockchain_raw(cryptonote::blockchain_storage* cs, cryptonote::tx_memory_pool* txp, + boost::filesystem::path& output_dir, uint64_t use_block_height=0); +#else + bool store_blockchain_raw(cryptonote::Blockchain* cs, cryptonote::tx_memory_pool* txp, + boost::filesystem::path& output_dir, uint64_t use_block_height=0); +#endif + +protected: +#if SOURCE_DB == DB_MEMORY + blockchain_storage* m_blockchain_storage; +#else + Blockchain* m_blockchain_storage; +#endif + + tx_memory_pool* m_tx_pool; + typedef std::vector<char> buffer_type; + std::ofstream * m_raw_data_file; + boost::archive::binary_oarchive * m_raw_archive; + buffer_type m_buffer; + boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>>* m_output_stream; + + // open export file for write + bool open(const boost::filesystem::path& dir_path); + bool close(); + void write_block(block& block); + void serialize_block_to_text_buffer(const block& block); + void buffer_serialize_tx(const transaction& tx); + void buffer_write_num_txs(const std::list<transaction> txs); + void flush_chunk(); +}; diff --git a/src/blockchain_converter/blockchain_import.cpp b/src/blockchain_converter/blockchain_import.cpp new file mode 100644 index 000000000..6b432ccc6 --- /dev/null +++ b/src/blockchain_converter/blockchain_import.cpp @@ -0,0 +1,756 @@ +// Copyright (c) 2014-2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <atomic> +#include <cstdio> +#include <algorithm> +#include <fstream> + +#include <boost/filesystem.hpp> +#include <boost/iostreams/stream.hpp> +#include <boost/archive/binary_iarchive.hpp> +#include "cryptonote_core/cryptonote_basic.h" +#include "cryptonote_core/cryptonote_format_utils.h" +#include "cryptonote_core/cryptonote_boost_serialization.h" +#include "serialization/json_utils.h" // dump_json() +#include "include_base_utils.h" +#include "common/command_line.h" +#include "version.h" + +#include <lmdb.h> // for db flag arguments + +#include "import.h" +#include "fake_core.h" + +unsigned int epee::g_test_dbg_lock_sleep = 0; + +// CONFIG +static bool opt_batch = true; +static bool opt_verify = true; // use add_new_block, which does verification before calling add_block +static bool opt_resume = true; +static bool opt_testnet = true; + +// number of blocks per batch transaction +// adjustable through command-line argument according to available RAM +static uint64_t db_batch_size = 20000; + +static std::string refresh_string = "\r \r"; + + +namespace po = boost::program_options; + +using namespace cryptonote; +using namespace epee; + + +int parse_db_arguments(const std::string& db_arg_str, std::string& db_engine, int& mdb_flags) +{ + std::vector<std::string> db_args; + boost::split(db_args, db_arg_str, boost::is_any_of("#")); + db_engine = db_args.front(); + boost::algorithm::trim(db_engine); + + if (db_args.size() == 1) + { + return 0; + } + else if (db_args.size() > 2) + { + std::cerr << "unrecognized database argument format: " << db_arg_str << ENDL; + return 1; + } + + std::string db_arg_str2 = db_args[1]; + boost::split(db_args, db_arg_str2, boost::is_any_of(",")); + for (auto& it : db_args) + { + boost::algorithm::trim(it); + if (it.empty()) + continue; + LOG_PRINT_L1("LMDB flag: " << it); + if (it == "nosync") + { + mdb_flags |= MDB_NOSYNC; + } + else if (it == "nometasync") + { + mdb_flags |= MDB_NOMETASYNC; + } + else if (it == "writemap") + { + mdb_flags |= MDB_WRITEMAP; + } + else if (it == "mapasync") + { + mdb_flags |= MDB_MAPASYNC; + } + else + { + std::cerr << "unrecognized database flag: " << it << ENDL; + return 1; + } + } + return 0; +} + + +int count_blocks(std::string& import_file_path) +{ + boost::filesystem::path raw_file_path(import_file_path); + boost::system::error_code ec; + if (!boost::filesystem::exists(raw_file_path, ec)) + { + LOG_PRINT_L0("import file not found: " << raw_file_path); + throw std::runtime_error("Aborting"); + } + std::ifstream import_file; + import_file.open(import_file_path, std::ios_base::binary | std::ifstream::in); + + uint64_t h = 0; + if (import_file.fail()) + { + LOG_PRINT_L0("import_file.open() fail"); + throw std::runtime_error("Aborting"); + } + LOG_PRINT_L0("Scanning blockchain from import file..."); + char buffer1[STR_LENGTH_OF_INT + 1]; + block b; + transaction tx; + bool quit = false; + uint64_t bytes_read = 0; + int progress_interval = 10; + + while (! quit) + { + int chunk_size; + import_file.read(buffer1, STR_LENGTH_OF_INT); + if (!import_file) { + std::cout << refresh_string; + LOG_PRINT_L1("End of import file reached"); + quit = true; + break; + } + h += NUM_BLOCKS_PER_CHUNK; + if (h % progress_interval == 0) + { + std::cout << refresh_string << "block height: " << h << + std::flush; + } + bytes_read += STR_LENGTH_OF_INT; + buffer1[STR_LENGTH_OF_INT] = '\0'; + chunk_size = atoi(buffer1); + if (chunk_size > BUFFER_SIZE) + { + std::cout << refresh_string; + LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE + << " height: " << h); + throw std::runtime_error("Aborting: chunk size exceeds buffer size"); + } + if (chunk_size > 100000) + { + std::cout << refresh_string; + LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > 100000" << " height: " + << h); + } + else if (chunk_size <= 0) { + std::cout << refresh_string; + LOG_PRINT_L0("ERROR: chunk_size " << chunk_size << " <= 0" << " height: " << h); + throw std::runtime_error("Aborting"); + } + // skip to next expected block size value + import_file.seekg(chunk_size, std::ios_base::cur); + if (! import_file) { + std::cout << refresh_string; + LOG_PRINT_L0("ERROR: unexpected end of import file: bytes read before error: " + << import_file.gcount() << " of chunk_size " << chunk_size); + throw std::runtime_error("Aborting"); + } + bytes_read += chunk_size; + std::cout << refresh_string; + + LOG_PRINT_L3("Total bytes scanned: " << bytes_read); + } + + import_file.close(); + + std::cout << ENDL; + std::cout << "Done scanning import file" << ENDL; + std::cout << "Total bytes scanned: " << bytes_read << ENDL; + std::cout << "Height: " << h << ENDL; + + return h; +} + +template <typename FakeCore> +int import_from_file(FakeCore& simple_core, std::string& import_file_path) +{ +#if !defined(BLOCKCHAIN_DB) + static_assert(std::is_same<fake_core_memory, FakeCore>::value || std::is_same<fake_core_lmdb, FakeCore>::value, + "FakeCore constraint error"); +#endif +#if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB) + if (std::is_same<fake_core_lmdb, FakeCore>::value) + { + // Reset stats, in case we're using newly created db, accumulating stats + // from addition of genesis block. + // This aligns internal db counts with importer counts. + simple_core.m_storage.get_db().reset_stats(); + } +#endif + boost::filesystem::path raw_file_path(import_file_path); + boost::system::error_code ec; + if (!boost::filesystem::exists(raw_file_path, ec)) + { + LOG_PRINT_L0("import file not found: " << raw_file_path); + return false; + } + + uint64_t source_height = count_blocks(import_file_path); + LOG_PRINT_L0("import file blockchain height: " << source_height); + + std::ifstream import_file; + import_file.open(import_file_path, std::ios_base::binary | std::ifstream::in); + + uint64_t h = 0; + if (import_file.fail()) + { + LOG_PRINT_L0("import_file.open() fail"); + return false; + } + char buffer1[STR_LENGTH_OF_INT + 1]; + char buffer_block[BUFFER_SIZE]; + block b; + transaction tx; + int quit = 0; + uint64_t bytes_read = 0; + + uint64_t start_height = 1; + if (opt_resume) + start_height = simple_core.m_storage.get_current_blockchain_height(); + + // Note that a new blockchain will start with a height of 1 (block number 0) + // due to genesis block being added at initialization. + + // CONFIG + // TODO: can expand on this, e.g. with --block-number option + uint64_t stop_height = source_height; + + // These are what we'll try to use, and they don't have to be a determination + // from source and destination blockchains, but those are the current + // defaults. + LOG_PRINT_L0("start height: " << start_height << " stop height: " << + stop_height); + + bool use_batch = false; + if (opt_batch) + { + if (simple_core.support_batch) + use_batch = true; + else + LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database engine - ignoring"); + } + + if (use_batch) + simple_core.batch_start(); + + LOG_PRINT_L0("Reading blockchain from import file..."); + std::cout << ENDL; + + // Within the loop, we skip to start_height before we start adding. + // TODO: Not a bottleneck, but we can use what's done in count_blocks() and + // only do the chunk size reads, skipping the chunk content reads until we're + // at start_height. + while (! quit) + { + int chunk_size; + import_file.read(buffer1, STR_LENGTH_OF_INT); + if (! import_file) { + std::cout << refresh_string; + LOG_PRINT_L0("End of import file reached"); + quit = 1; + break; + } + bytes_read += STR_LENGTH_OF_INT; + buffer1[STR_LENGTH_OF_INT] = '\0'; + chunk_size = atoi(buffer1); + if (chunk_size > BUFFER_SIZE) + { + LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE); + throw std::runtime_error("Aborting: chunk size exceeds buffer size"); + } + if (chunk_size > 100000) + { + LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > 100000"); + } + else if (chunk_size < 0) { + LOG_PRINT_L0("ERROR: chunk_size " << chunk_size << " < 0"); + return 2; + } + import_file.read(buffer_block, chunk_size); + if (! import_file) { + LOG_PRINT_L0("ERROR: unexpected end of import file: bytes read before error: " + << import_file.gcount() << " of chunk_size " << chunk_size); + return 2; + } + bytes_read += chunk_size; + LOG_PRINT_L3("Total bytes read: " << bytes_read); + + if (h + NUM_BLOCKS_PER_CHUNK < start_height + 1) + { + h += NUM_BLOCKS_PER_CHUNK; + continue; + } + if (h > stop_height) + { + LOG_PRINT_L0("Specified height reached - stopping. height: " << h << " block: " << h-1); + quit = 1; + break; + } + + try + { + boost::iostreams::basic_array_source<char> device(buffer_block, chunk_size); + boost::iostreams::stream<boost::iostreams::basic_array_source<char>> s(device); + boost::archive::binary_iarchive a(s); + + int display_interval = 1000; + int progress_interval = 10; + for (int chunk_ind = 0; chunk_ind < NUM_BLOCKS_PER_CHUNK; chunk_ind++) + { + h++; + if (h % display_interval == 0) + { + std::cout << refresh_string; + LOG_PRINT_L0("loading block height " << h); + } + else + { + LOG_PRINT_L3("loading block height " << h); + } + try { + a >> b; + } + catch (const std::exception& e) + { + std::cout << refresh_string; + LOG_PRINT_RED_L0("exception while de-archiving block, height=" << h); + quit = 1; + break; + } + LOG_PRINT_L2("block prev_id: " << b.prev_id << ENDL); + + if (h % progress_interval == 0) + { + std::cout << refresh_string << "block " << h-1 + << std::flush; + } + + std::vector<transaction> txs; + + int num_txs; + try + { + a >> num_txs; + } + catch (const std::exception& e) + { + std::cout << refresh_string; + LOG_PRINT_RED_L0("exception while de-archiving tx-num, height=" << h); + quit = 1; + break; + } + for(int tx_num = 1; tx_num <= num_txs; tx_num++) + { + try { + a >> tx; + } + catch (const std::exception& e) + { + LOG_PRINT_RED_L0("exception while de-archiving tx, height=" << h <<", tx_num=" << tx_num); + quit = 1; + break; + } + // if (tx_num == 1) { + // std::cout << "coinbase transaction" << ENDL; + // } + // crypto::hash hsh = null_hash; + // size_t blob_size = 0; + // NOTE: all tx hashes except for coinbase tx are available in the block data + // get_transaction_hash(tx, hsh, blob_size); + // LOG_PRINT_L0("tx " << tx_num << " " << hsh << " : " << ENDL); + // LOG_PRINT_L0(obj_to_json_str(tx) << ENDL); + + // add blocks with verification. + // for Blockchain and blockchain_storage add_new_block(). + if (opt_verify) + { + if (tx_num == 1) { + continue; // coinbase transaction. no need to insert to tx_pool. + } + // crypto::hash hsh = null_hash; + // size_t blob_size = 0; + // get_transaction_hash(tx, hsh, blob_size); + tx_verification_context tvc = AUTO_VAL_INIT(tvc); + bool r = true; + r = simple_core.m_pool.add_tx(tx, tvc, true); + if (!r) + { + LOG_PRINT_RED_L0("failed to add transaction to transaction pool, height=" << h <<", tx_num=" << tx_num); + quit = 1; + break; + } + } + else + { + // for add_block() method, without (much) processing. + // don't add coinbase transaction to txs. + // + // because add_block() calls + // add_transaction(blk_hash, blk.miner_tx) first, and + // then a for loop for the transactions in txs. + if (tx_num > 1) + { + txs.push_back(tx); + } + } + } + + if (opt_verify) + { + block_verification_context bvc = boost::value_initialized<block_verification_context>(); + simple_core.m_storage.add_new_block(b, bvc); + + if (bvc.m_verifivation_failed) + { + LOG_PRINT_L0("Failed to add block to blockchain, verification failed, height = " << h); + LOG_PRINT_L0("skipping rest of import file"); + // ok to commit previously batched data because it failed only in + // verification of potential new block with nothing added to batch + // yet + quit = 1; + break; + } + if (! bvc.m_added_to_main_chain) + { + LOG_PRINT_L0("Failed to add block to blockchain, height = " << h); + LOG_PRINT_L0("skipping rest of import file"); + // make sure we don't commit partial block data + quit = 2; + break; + } + } + else + { + size_t block_size; + difficulty_type cumulative_difficulty; + uint64_t coins_generated; + + a >> block_size; + a >> cumulative_difficulty; + a >> coins_generated; + + std::cout << refresh_string; + LOG_PRINT_L2("block_size: " << block_size); + LOG_PRINT_L2("cumulative_difficulty: " << cumulative_difficulty); + LOG_PRINT_L2("coins_generated: " << coins_generated); + + try + { + simple_core.add_block(b, block_size, cumulative_difficulty, coins_generated, txs); + } + catch (const std::exception& e) + { + std::cout << refresh_string; + LOG_PRINT_RED_L0("Error adding block to blockchain: " << e.what()); + quit = 2; // make sure we don't commit partial block data + break; + } + } + + if (use_batch) + { + if (h % db_batch_size == 0) + { + std::cout << refresh_string; + std::cout << ENDL << "[- batch commit at height " << h << " -]" << ENDL; + simple_core.batch_stop(); + simple_core.batch_start(); + std::cout << ENDL; +#if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB) + simple_core.m_storage.get_db().show_stats(); +#endif + } + } + } + } + catch (const std::exception& e) + { + std::cout << refresh_string; + LOG_PRINT_RED_L0("exception while reading from import file, height=" << h); + return 2; + } + } // while + + import_file.close(); + + if (use_batch) + { + if (quit > 1) + { + // There was an error, so don't commit pending data. + // Destructor will abort write txn. + } + else + { + simple_core.batch_stop(); + } +#if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB) + simple_core.m_storage.get_db().show_stats(); +#endif + if (h > 0) + LOG_PRINT_L0("Finished at height: " << h << " block: " << h-1); + } + std::cout << ENDL; + return 0; +} + +int main(int argc, char* argv[]) +{ + std::string import_filename = BLOCKCHAIN_RAW; +#if defined(BLOCKCHAIN_DB) && (BLOCKCHAIN_DB == DB_MEMORY) + std::string default_db_engine = "memory"; +#else + std::string default_db_engine = "lmdb"; +#endif + + uint32_t log_level = LOG_LEVEL_0; + std::string dirname; + std::string db_arg_str; + + boost::filesystem::path default_data_path {tools::get_default_data_dir()}; + boost::filesystem::path default_testnet_data_path {default_data_path / "testnet"}; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor<uint32_t> arg_log_level = {"log-level", "", log_level}; + const command_line::arg_descriptor<uint64_t> arg_batch_size = {"batch-size", "", db_batch_size}; + const command_line::arg_descriptor<bool> arg_testnet_on = { + "testnet" + , "Run on testnet." + , false + }; + const command_line::arg_descriptor<bool> arg_count_blocks = { + "count-blocks" + , "Count blocks in import file and exit" + , false + }; + const command_line::arg_descriptor<std::string> arg_database = { + "database", "available: memory, lmdb" + , default_db_engine + }; + const command_line::arg_descriptor<bool> arg_verify = {"verify", + "Verify blocks and transactions during import", true}; + const command_line::arg_descriptor<bool> arg_batch = {"batch", + "Batch transactions for faster import", true}; + const command_line::arg_descriptor<bool> arg_resume = {"resume", + "Resume from current height if output database already exists", true}; + + command_line::add_arg(desc_cmd_sett, command_line::arg_data_dir, default_data_path.string()); + command_line::add_arg(desc_cmd_sett, command_line::arg_testnet_data_dir, default_testnet_data_path.string()); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_batch_size); + command_line::add_arg(desc_cmd_sett, arg_testnet_on); + command_line::add_arg(desc_cmd_sett, arg_database); + + command_line::add_arg(desc_cmd_only, arg_count_blocks); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + // call add_options() directly for these arguments since + // command_line helpers support only boolean switch, not boolean argument + desc_cmd_sett.add_options() + (arg_verify.name, make_semantic(arg_verify), arg_verify.description) + (arg_batch.name, make_semantic(arg_batch), arg_batch.description) + (arg_resume.name, make_semantic(arg_resume), arg_resume.description) + ; + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + po::store(po::parse_command_line(argc, argv, desc_options), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + log_level = command_line::get_arg(vm, arg_log_level); + opt_verify = command_line::get_arg(vm, arg_verify); + opt_batch = command_line::get_arg(vm, arg_batch); + opt_resume = command_line::get_arg(vm, arg_resume); + db_batch_size = command_line::get_arg(vm, arg_batch_size); + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + if (! opt_batch && ! vm["batch-size"].defaulted()) + { + std::cerr << "Error: batch-size set, but batch option not enabled" << ENDL; + exit(1); + } + if (! db_batch_size) + { + std::cerr << "Error: batch-size must be > 0" << ENDL; + exit(1); + } + + std::vector<std::string> db_engines {"memory", "lmdb"}; + + opt_testnet = command_line::get_arg(vm, arg_testnet_on); + auto data_dir_arg = opt_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir; + dirname = command_line::get_arg(vm, data_dir_arg); + db_arg_str = command_line::get_arg(vm, arg_database); + + log_space::get_set_log_detalisation_level(true, log_level); + log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL); + LOG_PRINT_L0("Starting..."); + LOG_PRINT_L0("Setting log level = " << log_level); + + boost::filesystem::path file_path {dirname}; + std::string import_file_path; + + import_file_path = (file_path / "export" / import_filename).string(); + + if (command_line::has_arg(vm, arg_count_blocks)) + { + count_blocks(import_file_path); + exit(0); + } + + + std::string db_engine; + int mdb_flags = 0; + int res = 0; + res = parse_db_arguments(db_arg_str, db_engine, mdb_flags); + if (res) + { + std::cerr << "Error parsing database argument(s)" << ENDL; + exit(1); + } + + if (std::find(db_engines.begin(), db_engines.end(), db_engine) == db_engines.end()) + { + std::cerr << "Invalid database engine: " << db_engine << std::endl; + exit(1); + } + + LOG_PRINT_L0("database: " << db_engine); + LOG_PRINT_L0("verify: " << std::boolalpha << opt_verify << std::noboolalpha); + if (opt_batch) + { + LOG_PRINT_L0("batch: " << std::boolalpha << opt_batch << std::noboolalpha + << " batch size: " << db_batch_size); + } + else + { + LOG_PRINT_L0("batch: " << std::boolalpha << opt_batch << std::noboolalpha); + } + LOG_PRINT_L0("resume: " << std::boolalpha << opt_resume << std::noboolalpha); + LOG_PRINT_L0("testnet: " << std::boolalpha << opt_testnet << std::noboolalpha); + + std::cout << "import file path: " << import_file_path << ENDL; + std::cout << "database path: " << file_path.string() << ENDL; + + try + { + + // fake_core needed for verification to work when enabled. + // + // NOTE: don't need fake_core method of doing things when we're going to call + // BlockchainDB add_block() directly and have available the 3 block + // properties to do so. Both ways work, but fake core isn't necessary in that + // circumstance. + + // for multi_db_runtime: +#if !defined(BLOCKCHAIN_DB) + if (db_engine == "lmdb") + { + fake_core_lmdb simple_core(dirname, opt_testnet, opt_batch, mdb_flags); + import_from_file(simple_core, import_file_path); + } + else if (db_engine == "memory") + { + fake_core_memory simple_core(dirname, opt_testnet); + import_from_file(simple_core, import_file_path); + } + else + { + std::cerr << "database engine unrecognized" << ENDL; + exit(1); + } + + // for multi_db_compile: +#else + if (db_engine != default_db_engine) + { + std::cerr << "Invalid database engine for compiled version: " << db_engine << std::endl; + exit(1); + } +#if BLOCKCHAIN_DB == DB_LMDB + fake_core_lmdb simple_core(dirname, opt_testnet, opt_batch, mdb_flags); +#else + fake_core_memory simple_core(dirname, opt_testnet); +#endif + + import_from_file(simple_core, import_file_path); +#endif + + } + catch (const DB_ERROR& e) + { + std::cout << std::string("Error loading blockchain db: ") + e.what() + " -- shutting down now" << ENDL; + exit(1); + } + + // destructors called at exit: + // + // ensure db closed + // - transactions properly checked and handled + // - disk sync if needed + // + // fake_core object's destructor is called when it goes out of scope. For an + // LMDB fake_core, it calls Blockchain::deinit() on its object, which in turn + // calls delete on its BlockchainDB derived class' object, which closes its + // files. +} diff --git a/src/blockchain_converter/fake_core.h b/src/blockchain_converter/fake_core.h new file mode 100644 index 000000000..f82b05d04 --- /dev/null +++ b/src/blockchain_converter/fake_core.h @@ -0,0 +1,165 @@ +// Copyright (c) 2014-2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <boost/filesystem.hpp> +#include "cryptonote_core/blockchain.h" // BlockchainDB +#include "cryptonote_core/blockchain_storage.h" // in-memory DB +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/lmdb/db_lmdb.h" + +using namespace cryptonote; + + +#if !defined(BLOCKCHAIN_DB) || BLOCKCHAIN_DB == DB_LMDB + +struct fake_core_lmdb +{ + Blockchain m_storage; + tx_memory_pool m_pool; + bool support_batch; + bool support_add_block; + + // for multi_db_runtime: +#if !defined(BLOCKCHAIN_DB) + fake_core_lmdb(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const int mdb_flags=0) : m_pool(&m_storage), m_storage(m_pool) + // for multi_db_compile: +#else + fake_core_lmdb(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const int mdb_flags=0) : m_pool(m_storage), m_storage(m_pool) +#endif + { + m_pool.init(path.string()); + + BlockchainDB* db = new BlockchainLMDB(); + + boost::filesystem::path folder(path); + + folder /= db->get_db_name(); + + LOG_PRINT_L0("Loading blockchain from folder " << folder.c_str() << " ..."); + + const std::string filename = folder.string(); + try + { + db->open(filename, mdb_flags); + } + catch (const std::exception& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + throw; + } + + m_storage.init(db, use_testnet); + if (do_batch) + m_storage.get_db().set_batch_transactions(do_batch); + support_batch = true; + support_add_block = true; + } + ~fake_core_lmdb() + { + m_storage.deinit(); + } + + uint64_t add_block(const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector<transaction>& txs + ) + { + return m_storage.get_db().add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); + } + + void batch_start() + { + m_storage.get_db().batch_start(); + } + + void batch_stop() + { + m_storage.get_db().batch_stop(); + } + +}; +#endif + +#if !defined(BLOCKCHAIN_DB) || BLOCKCHAIN_DB == DB_MEMORY + +struct fake_core_memory +{ + blockchain_storage m_storage; + tx_memory_pool m_pool; + bool support_batch; + bool support_add_block; + + // for multi_db_runtime: +#if !defined(BLOCKCHAIN_DB) + fake_core_memory(const boost::filesystem::path &path, const bool use_testnet=false) : m_pool(&m_storage), m_storage(m_pool) +#else + // for multi_db_compile: + fake_core_memory(const boost::filesystem::path &path, const bool use_testnet=false) : m_pool(m_storage), m_storage(&m_pool) +#endif + { + m_pool.init(path.string()); + m_storage.init(path.string(), use_testnet); + support_batch = false; + support_add_block = false; + } + ~fake_core_memory() + { + LOG_PRINT_L3("fake_core_memory() destructor called - want to see it ripple down"); + m_storage.deinit(); + } + + uint64_t add_block(const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector<transaction>& txs + ) + { + // TODO: + // would need to refactor handle_block_to_main_chain() to have a direct add_block() method like Blockchain class + throw std::runtime_error("direct add_block() method not implemented for in-memory db"); + return 2; + } + + void batch_start() + { + LOG_PRINT_L0("WARNING: [batch_start] opt_batch set, but this database doesn't support/need transactions - ignoring"); + } + + void batch_stop() + { + LOG_PRINT_L0("WARNING: [batch_stop] opt_batch set, but this database doesn't support/need transactions - ignoring"); + } + +}; + +#endif diff --git a/src/blockchain_converter/import.h b/src/blockchain_converter/import.h new file mode 100644 index 000000000..632b4c0d9 --- /dev/null +++ b/src/blockchain_converter/import.h @@ -0,0 +1,37 @@ +// Copyright (c) 2014-2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +// TODO: bounds checking is done before writing to buffer, but buffer size +// should be a sensible maximum +#define BUFFER_SIZE 1000000 +#define NUM_BLOCKS_PER_CHUNK 1 +#define STR_LENGTH_OF_INT 9 +#define STR_FORMAT_OF_INT "%09d" +#define BLOCKCHAIN_RAW "blockchain.raw" diff --git a/src/blockchain_db/CMakeLists.txt b/src/blockchain_db/CMakeLists.txt new file mode 100644 index 000000000..adbe804aa --- /dev/null +++ b/src/blockchain_db/CMakeLists.txt @@ -0,0 +1,76 @@ +# Copyright (c) 2014-2015, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set(blockchain_db_sources + blockchain_db.cpp + lmdb/db_lmdb.cpp + ) + +if (NOT STATIC) + set(blockchain_db_sources + ${blockchain_db_sources} + berkeleydb/db_bdb.cpp + ) +endif() + + +set(blockchain_db_headers) + +set(blockchain_db_private_headers + blockchain_db.h + lmdb/db_lmdb.h + ) + +if (NOT STATIC) + set(blockchain_db_private_headers + ${blockchain_db_private_headers} + berkeleydb/db_bdb.h + ) +endif() + +bitmonero_private_headers(blockchain_db + ${crypto_private_headers}) +bitmonero_add_library(blockchain_db + ${blockchain_db_sources} + ${blockchain_db_headers} + ${blockchain_db_private_headers}) +target_link_libraries(blockchain_db + LINK_PUBLIC + common + crypto + cryptonote_core + ${Boost_DATE_TIME_LIBRARY} + ${Boost_PROGRAM_OPTIONS_LIBRARY} + ${Boost_SERIALIZATION_LIBRARY} + ${LMDB_LIBRARY} + ${BDB_LIBRARY} + LINK_PRIVATE + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${EXTRA_LIBRARIES}) diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp new file mode 100644 index 000000000..4b254500b --- /dev/null +++ b/src/blockchain_db/berkeleydb/db_bdb.cpp @@ -0,0 +1,1689 @@ +// Copyright (c) 2014, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "db_bdb.h" + +#include <boost/filesystem.hpp> +#include <memory> // std::unique_ptr +#include <cstring> // memcpy + +#include "cryptonote_core/cryptonote_format_utils.h" +#include "crypto/crypto.h" +#include "profile_tools.h" + +using epee::string_tools::pod_to_hex; + +namespace +{ + +template <typename T> +inline void throw0(const T &e) +{ + LOG_PRINT_L0(e.what()); + throw e; +} + +template <typename T> +inline void throw1(const T &e) +{ + LOG_PRINT_L1(e.what()); + throw e; +} + +// cursor needs to be closed when it goes out of scope, +// this helps if the function using it throws +struct bdb_cur +{ + bdb_cur(DbTxn* txn, Db* dbi) + { + if (dbi->cursor(txn, &m_cur, 0)) + throw0(cryptonote::DB_ERROR("Error opening db cursor")); + done = false; + } + + ~bdb_cur() { close(); } + + operator Dbc*() { return m_cur; } + operator Dbc**() { return &m_cur; } + Dbc* operator->() { return m_cur; } + + void close() + { + if (!done) + { + m_cur->close(); + done = true; + } + } + +private: + Dbc* m_cur; + bool done; +}; + +const char* const BDB_BLOCKS = "blocks"; +const char* const BDB_BLOCK_TIMESTAMPS = "block_timestamps"; +const char* const BDB_BLOCK_HEIGHTS = "block_heights"; +const char* const BDB_BLOCK_HASHES = "block_hashes"; +const char* const BDB_BLOCK_SIZES = "block_sizes"; +const char* const BDB_BLOCK_DIFFS = "block_diffs"; +const char* const BDB_BLOCK_COINS = "block_coins"; + +const char* const BDB_TXS = "txs"; +const char* const BDB_TX_UNLOCKS = "tx_unlocks"; +const char* const BDB_TX_HEIGHTS = "tx_heights"; +const char* const BDB_TX_OUTPUTS = "tx_outputs"; + +const char* const BDB_OUTPUT_TXS = "output_txs"; +const char* const BDB_OUTPUT_INDICES = "output_indices"; +const char* const BDB_OUTPUT_AMOUNTS = "output_amounts"; +const char* const BDB_OUTPUT_KEYS = "output_keys"; + +const char* const BDB_SPENT_KEYS = "spent_keys"; + +template<typename T> +struct Dbt_copy: public Dbt +{ + Dbt_copy(const T &t): t_copy(t) + { + init(); + } + + Dbt_copy() + { + init(); + } + + void init() + { + set_data(&t_copy); + set_size(sizeof(T)); + set_ulen(sizeof(T)); + set_flags(DB_DBT_USERMEM); + } + + operator T() + { + return t_copy; + } +private: + T t_copy; +}; + +template<> +struct Dbt_copy<cryptonote::blobdata>: public Dbt +{ + Dbt_copy(const cryptonote::blobdata &bd) : m_data(new char[bd.size()]) + { + memcpy(m_data.get(), bd.data(), bd.size()); + set_data(m_data.get()); + set_size(bd.size()); + set_ulen(bd.size()); + set_flags(DB_DBT_USERMEM); + } +private: + std::unique_ptr<char[]> m_data; +}; + +struct Dbt_safe : public Dbt +{ + Dbt_safe() + { + set_data(NULL); + set_flags(DB_DBT_MALLOC); + } + ~Dbt_safe() + { + void* buf = get_data(); + if (buf != NULL) + { + free(buf); + } + } +}; + +} // anonymous namespace + +namespace cryptonote +{ + +void BlockchainBDB::add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const crypto::hash& blk_hash + ) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + Dbt_copy<crypto::hash> val_h(blk_hash); + if (m_block_heights->exists(*m_write_txn, &val_h, 0) == 0) + throw1(BLOCK_EXISTS("Attempting to add block that's already in the db")); + + if (m_height > 0) + { + Dbt_copy<crypto::hash> parent_key(blk.prev_id); + Dbt_copy<uint32_t> parent_h; + if (m_block_heights->get(*m_write_txn, &parent_key, &parent_h, 0)) + { + LOG_PRINT_L3("m_height: " << m_height); + LOG_PRINT_L3("parent_key: " << blk.prev_id); + throw0(DB_ERROR("Failed to get top block hash to check for new block's parent")); + } + uint32_t parent_height = parent_h; + if (parent_height != m_height) + throw0(BLOCK_PARENT_DNE("Top block is not new block's parent")); + } + + Dbt_copy<uint32_t> key(m_height + 1); + + Dbt_copy<blobdata> blob(block_to_blob(blk)); + auto res = m_blocks->put(*m_write_txn, &key, &blob, 0); + if (res) + throw0(DB_ERROR("Failed to add block blob to db transaction.")); + + Dbt_copy<size_t> sz(block_size); + if (m_block_sizes->put(*m_write_txn, &key, &sz, 0)) + throw0(DB_ERROR("Failed to add block size to db transaction.")); + + Dbt_copy<uint64_t> ts(blk.timestamp); + if (m_block_timestamps->put(*m_write_txn, &key, &ts, 0)) + throw0(DB_ERROR("Failed to add block timestamp to db transaction.")); + + Dbt_copy<difficulty_type> diff(cumulative_difficulty); + if (m_block_diffs->put(*m_write_txn, &key, &diff, 0)) + throw0(DB_ERROR("Failed to add block cumulative difficulty to db transaction.")); + + Dbt_copy<uint64_t> coinsgen(coins_generated); + if (m_block_coins->put(*m_write_txn, &key, &coinsgen, 0)) + throw0(DB_ERROR("Failed to add block total generated coins to db transaction.")); + + if (m_block_heights->put(*m_write_txn, &val_h, &key, 0)) + throw0(DB_ERROR("Failed to add block height by hash to db transaction.")); + + if (m_block_hashes->put(*m_write_txn, &key, &val_h, 0)) + throw0(DB_ERROR("Failed to add block hash to db transaction.")); +} + +void BlockchainBDB::remove_block() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + if (m_height == 0) + throw0(BLOCK_DNE ("Attempting to remove block from an empty blockchain")); + + Dbt_copy<uint32_t> k(m_height); + Dbt_copy<crypto::hash> h; + if (m_block_hashes->get(*m_write_txn, &k, &h, 0)) + throw1(BLOCK_DNE("Attempting to remove block that's not in the db")); + + if (m_blocks->del(*m_write_txn, &k, 0)) + throw1(DB_ERROR("Failed to add removal of block to db transaction")); + + if (m_block_sizes->del(*m_write_txn, &k, 0)) + throw1(DB_ERROR("Failed to add removal of block size to db transaction")); + + if (m_block_diffs->del(*m_write_txn, &k, 0)) + throw1(DB_ERROR("Failed to add removal of block cumulative difficulty to db transaction")); + + if (m_block_coins->del(*m_write_txn, &k, 0)) + throw1(DB_ERROR("Failed to add removal of block total generated coins to db transaction")); + + if (m_block_timestamps->del(*m_write_txn, &k, 0)) + throw1(DB_ERROR("Failed to add removal of block timestamp to db transaction")); + + if (m_block_heights->del(*m_write_txn, &h, 0)) + throw1(DB_ERROR("Failed to add removal of block height by hash to db transaction")); + + if (m_block_hashes->del(*m_write_txn, &k, 0)) + throw1(DB_ERROR("Failed to add removal of block hash to db transaction")); +} + +void BlockchainBDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + Dbt_copy<crypto::hash> val_h(tx_hash); + + if (m_txs->exists(*m_write_txn, &val_h, 0) == 0) + throw1(TX_EXISTS("Attempting to add transaction that's already in the db")); + + Dbt_copy<blobdata> blob(tx_to_blob(tx)); + if (m_txs->put(*m_write_txn, &val_h, &blob, 0)) + throw0(DB_ERROR("Failed to add tx blob to db transaction")); + + Dbt_copy<uint64_t> height(m_height + 1); + if (m_tx_heights->put(*m_write_txn, &val_h, &height, 0)) + throw0(DB_ERROR("Failed to add tx block height to db transaction")); + + Dbt_copy<uint64_t> unlock_time(tx.unlock_time); + if (m_tx_unlocks->put(*m_write_txn, &val_h, &unlock_time, 0)) + throw0(DB_ERROR("Failed to add tx unlock time to db transaction")); +} + +void BlockchainBDB::remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + Dbt_copy<crypto::hash> val_h(tx_hash); + if (m_txs->exists(*m_write_txn, &val_h, 0)) + throw1(TX_DNE("Attempting to remove transaction that isn't in the db")); + + if (m_txs->del(*m_write_txn, &val_h, 0)) + throw1(DB_ERROR("Failed to add removal of tx to db transaction")); + if (m_tx_unlocks->del(*m_write_txn, &val_h, 0)) + throw1(DB_ERROR("Failed to add removal of tx unlock time to db transaction")); + if (m_tx_heights->del(*m_write_txn, &val_h, 0)) + throw1(DB_ERROR("Failed to add removal of tx block height to db transaction")); + + remove_tx_outputs(tx_hash, tx); + + if (m_tx_outputs->del(*m_write_txn, &val_h, 0)) + throw1(DB_ERROR("Failed to add removal of tx outputs to db transaction")); + +} + +void BlockchainBDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + Dbt_copy<uint32_t> k(m_num_outputs + 1); + Dbt_copy<crypto::hash> v(tx_hash); + + if (m_output_txs->put(*m_write_txn, &k, &v, 0)) + throw0(DB_ERROR("Failed to add output tx hash to db transaction")); + if (m_tx_outputs->put(*m_write_txn, &v, &k, 0)) + throw0(DB_ERROR("Failed to add tx output index to db transaction")); + + Dbt_copy<uint64_t> val_local_index(local_index); + if (m_output_indices->put(*m_write_txn, &k, &val_local_index, 0)) + throw0(DB_ERROR("Failed to add tx output index to db transaction")); + + Dbt_copy<uint64_t> val_amount(tx_output.amount); + if (m_output_amounts->put(*m_write_txn, &val_amount, &k, 0)) + throw0(DB_ERROR("Failed to add output amount to db transaction.")); + + if (tx_output.target.type() == typeid(txout_to_key)) + { + Dbt_copy<crypto::public_key> val_pubkey(boost::get<txout_to_key>(tx_output.target).key); + if (m_output_keys->put(*m_write_txn, &k, &val_pubkey, 0)) + throw0(DB_ERROR("Failed to add output pubkey to db transaction")); + } + + m_num_outputs++; +} + +void BlockchainBDB::remove_tx_outputs(const crypto::hash& tx_hash, const transaction& tx) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + + bdb_cur cur(*m_write_txn, m_tx_outputs); + + Dbt_copy<crypto::hash> k(tx_hash); + Dbt_copy<uint32_t> v; + + auto result = cur->get(&k, &v, DB_SET); + if (result == DB_NOTFOUND) + { + throw0(DB_ERROR("Attempting to remove a tx's outputs, but none found.")); + } + else if (result) + { + throw0(DB_ERROR("DB error attempting to get an output")); + } + else + { + db_recno_t num_elems = 0; + cur->count(&num_elems, 0); + + for (uint64_t i = 0; i < num_elems; ++i) + { + const tx_out tx_output = tx.vout[i]; + remove_output(v, tx_output.amount); + if (i < num_elems - 1) + { + cur->get(&k, &v, DB_NEXT_DUP); + } + } + } + + cur.close(); +} + +// TODO: probably remove this function +void BlockchainBDB::remove_output(const tx_out& tx_output) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__ << " (unused version - does nothing)"); + return; +} + +void BlockchainBDB::remove_output(const uint64_t& out_index, const uint64_t amount) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + Dbt_copy<uint32_t> k(out_index); + + auto result = m_output_indices->del(*m_write_txn, &k, 0); + if (result == DB_NOTFOUND) + { + LOG_PRINT_L0("Unexpected: global output index not found in m_output_indices"); + } + else if (result) + { + throw1(DB_ERROR("Error adding removal of output tx index to db transaction")); + } + + result = m_output_txs->del(*m_write_txn, &k, 0); + // if (result != 0 && result != DB_NOTFOUND) + // throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); + if (result == DB_NOTFOUND) + { + LOG_PRINT_L0("Unexpected: global output index not found in m_output_txs"); + } + else if (result) + { + throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); + } + + result = m_output_keys->del(*m_write_txn, &k, 0); + if (result == DB_NOTFOUND) + { + LOG_PRINT_L0("Unexpected: global output index not found in m_output_keys"); + } + else if (result) + throw1(DB_ERROR("Error adding removal of output pubkey to db transaction")); + + remove_amount_output_index(amount, out_index); + + m_num_outputs--; +} + +void BlockchainBDB::remove_amount_output_index(const uint64_t amount, const uint64_t global_output_index) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_cur cur(*m_write_txn, m_output_amounts); + + Dbt_copy<uint64_t> k(amount); + Dbt_copy<uint32_t> v; + + auto result = cur->get(&k, &v, DB_SET); + if (result == DB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + db_recno_t num_elems = 0; + cur->count(&num_elems, 0); + + uint64_t amount_output_index = 0; + uint64_t goi = 0; + bool found_index = false; + for (uint64_t i = 0; i < num_elems; ++i) + { + goi = v; + if (goi == global_output_index) + { + amount_output_index = i; + found_index = true; + break; + } + cur->get(&k, &v, DB_NEXT_DUP); + } + if (found_index) + { + // found the amount output index + // now delete it + result = cur->del(0); + if (result) + throw0(DB_ERROR(std::string("Error deleting amount output index ").append(boost::lexical_cast<std::string>(amount_output_index)).c_str())); + } + else + { + // not found + throw1(OUTPUT_DNE("Failed to find amount output index")); + } + cur.close(); +} + +void BlockchainBDB::add_spent_key(const crypto::key_image& k_image) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + Dbt_copy<crypto::key_image> val_key(k_image); + if (m_spent_keys->exists(*m_write_txn, &val_key, 0) == 0) + throw1(KEY_IMAGE_EXISTS("Attempting to add spent key image that's already in the db")); + + Dbt_copy<char> val('\0'); + if (m_spent_keys->put(*m_write_txn, &val_key, &val, 0)) + throw1(DB_ERROR("Error adding spent key image to db transaction.")); +} + +void BlockchainBDB::remove_spent_key(const crypto::key_image& k_image) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + Dbt_copy<crypto::key_image> k(k_image); + auto result = m_spent_keys->del(*m_write_txn, &k, 0); + if (result != 0 && result != DB_NOTFOUND) + throw1(DB_ERROR("Error adding removal of key image to db transaction")); +} + +blobdata BlockchainBDB::output_to_blob(const tx_out& output) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + blobdata b; + if (!t_serializable_object_to_blob(output, b)) + throw1(DB_ERROR("Error serializing output to blob")); + return b; +} + +tx_out BlockchainBDB::output_from_blob(const blobdata& blob) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + std::stringstream ss; + ss << blob; + binary_archive<false> ba(ss); + tx_out o; + + if (!(::serialization::serialize(ba, o))) + throw1(DB_ERROR("Error deserializing tx output blob")); + + return o; +} + +uint64_t BlockchainBDB::get_output_global_index(const uint64_t& amount, const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + bdb_cur cur(txn, m_output_amounts); + + Dbt_copy<uint64_t> k(amount); + Dbt_copy<uint32_t> v; + + auto result = cur->get(&k, &v, DB_SET); + if (result == DB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + db_recno_t num_elems; + cur->count(&num_elems, 0); + + if (num_elems <= index) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); + + for (uint64_t i = 0; i < index; ++i) + { + cur->get(&k, &v, DB_NEXT_DUP); + } + + uint64_t glob_index = v; + + cur.close(); + + txn.commit(); + + return glob_index; +} + +void BlockchainBDB::check_open() const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + if (!m_open) + throw0(DB_ERROR("DB operation attempted on a not-open DB instance")); +} + +BlockchainBDB::~BlockchainBDB() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + + if (m_open) + { + close(); + } +} + +BlockchainBDB::BlockchainBDB(bool batch_transactions) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + // initialize folder to something "safe" just in case + // someone accidentally misuses this class... + m_folder = "thishsouldnotexistbecauseitisgibberish"; + m_open = false; + + m_batch_transactions = batch_transactions; + m_write_txn = nullptr; + m_height = 0; +} + +void BlockchainBDB::open(const std::string& filename, const int db_flags) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + + if (m_open) + throw0(DB_OPEN_FAILURE("Attempted to open db, but it's already open")); + + boost::filesystem::path direc(filename); + if (boost::filesystem::exists(direc)) + { + if (!boost::filesystem::is_directory(direc)) + throw0(DB_OPEN_FAILURE("DB needs a directory path, but a file was passed")); + } + else + { + if (!boost::filesystem::create_directory(direc)) + throw0(DB_OPEN_FAILURE(std::string("Failed to create directory ").append(filename).c_str())); + } + + m_folder = filename; + + try + { + + //Create BerkeleyDB environment + m_env = new DbEnv(0); // no flags needed for DbEnv + + uint32_t db_env_open_flags = DB_CREATE | DB_INIT_MPOOL | DB_INIT_LOCK + | DB_INIT_LOG | DB_INIT_TXN | DB_RECOVER + | DB_THREAD; + + // last parameter left 0, files will be created with default rw access + m_env->open(filename.c_str(), db_env_open_flags, 0); + + // begin transaction to init dbs + bdb_txn_safe txn; + m_env->txn_begin(NULL, txn, 0); + + // create Dbs in the environment + m_blocks = new Db(m_env, 0); + m_block_heights = new Db(m_env, 0); + m_block_hashes = new Db(m_env, 0); + m_block_timestamps = new Db(m_env, 0); + m_block_sizes = new Db(m_env, 0); + m_block_diffs = new Db(m_env, 0); + m_block_coins = new Db(m_env, 0); + + m_txs = new Db(m_env, 0); + m_tx_unlocks = new Db(m_env, 0); + m_tx_heights = new Db(m_env, 0); + m_tx_outputs = new Db(m_env, 0); + + m_output_txs = new Db(m_env, 0); + m_output_indices = new Db(m_env, 0); + m_output_amounts = new Db(m_env, 0); + m_output_keys = new Db(m_env, 0); + + m_spent_keys = new Db(m_env, 0); + + // Tell DB about Dbs that need duplicate support + // Note: no need to tell about sorting, + // as the default is insertion order, which we want + m_tx_outputs->set_flags(DB_DUP); + m_output_amounts->set_flags(DB_DUP); + + // Tell DB about fixed-size values. + m_block_hashes->set_re_len(sizeof(crypto::hash)); + m_block_timestamps->set_re_len(sizeof(uint64_t)); + m_block_sizes->set_re_len(sizeof(size_t)); // should really store block size as uint64_t... + m_block_diffs->set_re_len(sizeof(difficulty_type)); + m_block_coins->set_re_len(sizeof(uint64_t)); + + m_output_txs->set_re_len(sizeof(crypto::hash)); + m_output_indices->set_re_len(sizeof(uint64_t)); + m_output_keys->set_re_len(sizeof(crypto::public_key)); + + //TODO: Find out if we need to do Db::set_flags(DB_RENUMBER) + // for the RECNO databases. We shouldn't as we're only + // inserting/removing from the end, but we'll see. + + // open Dbs in the environment + // m_tx_outputs and m_output_amounts must be DB_HASH or DB_BTREE + // because they need duplicate entry support. The rest are DB_RECNO, + // as it seems that will be the most performant choice. + m_blocks->open(txn, BDB_BLOCKS, NULL, DB_RECNO, DB_CREATE, 0); + + m_block_timestamps->open(txn, BDB_BLOCK_TIMESTAMPS, NULL, DB_RECNO, DB_CREATE, 0); + m_block_heights->open(txn, BDB_BLOCK_HEIGHTS, NULL, DB_HASH, DB_CREATE, 0); + m_block_hashes->open(txn, BDB_BLOCK_HASHES, NULL, DB_RECNO, DB_CREATE, 0); + m_block_sizes->open(txn, BDB_BLOCK_SIZES, NULL, DB_RECNO, DB_CREATE, 0); + m_block_diffs->open(txn, BDB_BLOCK_DIFFS, NULL, DB_RECNO, DB_CREATE, 0); + m_block_coins->open(txn, BDB_BLOCK_COINS, NULL, DB_RECNO, DB_CREATE, 0); + + m_txs->open(txn, BDB_TXS, NULL, DB_HASH, DB_CREATE, 0); + m_tx_unlocks->open(txn, BDB_TX_UNLOCKS, NULL, DB_HASH, DB_CREATE, 0); + m_tx_heights->open(txn, BDB_TX_HEIGHTS, NULL, DB_HASH, DB_CREATE, 0); + m_tx_outputs->open(txn, BDB_TX_OUTPUTS, NULL, DB_HASH, DB_CREATE, 0); + + m_output_txs->open(txn, BDB_OUTPUT_TXS, NULL, DB_RECNO, DB_CREATE, 0); + m_output_indices->open(txn, BDB_OUTPUT_INDICES, NULL, DB_RECNO, DB_CREATE, 0); + m_output_amounts->open(txn, BDB_OUTPUT_AMOUNTS, NULL, DB_HASH, DB_CREATE, 0); + m_output_keys->open(txn, BDB_OUTPUT_KEYS, NULL, DB_RECNO, DB_CREATE, 0); + + m_spent_keys->open(txn, BDB_SPENT_KEYS, NULL, DB_HASH, DB_CREATE, 0); + + DB_BTREE_STAT* stats; + + // DB_FAST_STAT can apparently cause an incorrect number of records + // to be returned. The flag should be set to 0 instead if this proves + // to be the case. + m_blocks->stat(txn, &stats, DB_FAST_STAT); + m_height = stats->bt_nkeys; + delete stats; + + // see above comment about DB_FAST_STAT + m_output_indices->stat(txn, &stats, DB_FAST_STAT); + m_num_outputs = stats->bt_nkeys; + delete stats; + + txn.commit(); + } + catch (const std::exception& e) + { + throw0(DB_OPEN_FAILURE(e.what())); + } + + m_open = true; +} + +void BlockchainBDB::close() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + this->sync(); + + // FIXME: not yet thread safe!!! Use with care. + m_open = false; + m_env->close(DB_FORCESYNC); +} + +void BlockchainBDB::sync() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + try + { + m_blocks->sync(0); + m_block_heights->sync(0); + m_block_hashes->sync(0); + m_block_timestamps->sync(0); + m_block_sizes->sync(0); + m_block_diffs->sync(0); + m_block_coins->sync(0); + + m_txs->sync(0); + m_tx_unlocks->sync(0); + m_tx_heights->sync(0); + m_tx_outputs->sync(0); + + m_output_txs->sync(0); + m_output_indices->sync(0); + m_output_amounts->sync(0); + m_output_keys->sync(0); + + m_spent_keys->sync(0); + } + catch (const std::exception& e) + { + throw0(DB_ERROR(std::string("Failed to sync database: ").append(e.what()).c_str())); + } +} + +void BlockchainBDB::reset() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + // TODO: this +} + +std::vector<std::string> BlockchainBDB::get_filenames() const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + std::vector<std::string> filenames; + + char *fname, *dbname; + const char **pfname, **pdbname; + + pfname = (const char **)&fname; + pdbname = (const char **)&dbname; + + m_blocks->get_dbname(pfname, pdbname); + filenames.push_back(fname); + + m_block_heights->get_dbname(pfname, pdbname); + filenames.push_back(fname); + + m_block_hashes->get_dbname(pfname, pdbname); + filenames.push_back(fname); + + m_block_timestamps->get_dbname(pfname, pdbname); + filenames.push_back(fname); + + m_block_sizes->get_dbname(pfname, pdbname); + filenames.push_back(fname); + + m_block_diffs->get_dbname(pfname, pdbname); + filenames.push_back(fname); + + m_block_coins->get_dbname(pfname, pdbname); + filenames.push_back(fname); + + m_txs->get_dbname(pfname, pdbname); + filenames.push_back(fname); + + m_tx_unlocks->get_dbname(pfname, pdbname); + filenames.push_back(fname); + + m_tx_heights->get_dbname(pfname, pdbname); + filenames.push_back(fname); + + m_tx_outputs->get_dbname(pfname, pdbname); + filenames.push_back(fname); + + m_output_txs->get_dbname(pfname, pdbname); + filenames.push_back(fname); + + m_output_indices->get_dbname(pfname, pdbname); + filenames.push_back(fname); + + m_output_amounts->get_dbname(pfname, pdbname); + filenames.push_back(fname); + + m_output_keys->get_dbname(pfname, pdbname); + filenames.push_back(fname); + + m_spent_keys->get_dbname(pfname, pdbname); + filenames.push_back(fname); + + std::vector<std::string> full_paths; + + for (auto& filename : filenames) + { + boost::filesystem::path p(m_folder); + p /= filename; + full_paths.push_back(p.string()); + } + + return full_paths; +} + +std::string BlockchainBDB::get_db_name() const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + + return std::string("BerkeleyDB"); +} + +// TODO: this? +bool BlockchainBDB::lock() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + return false; +} + +// TODO: this? +void BlockchainBDB::unlock() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +bool BlockchainBDB::block_exists(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy<crypto::hash> key(h); + + auto get_result = m_block_heights->exists(txn, &key, 0); + if (get_result == DB_NOTFOUND) + { + txn.commit(); + LOG_PRINT_L3("Block with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); + return false; + } + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch block index from hash")); + + txn.commit(); + return true; +} + +block BlockchainBDB::get_block(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + return get_block_from_height(get_block_height(h)); +} + +uint64_t BlockchainBDB::get_block_height(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy<crypto::hash> key(h); + Dbt_copy<uint64_t> result; + + auto get_result = m_block_heights->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + throw1(BLOCK_DNE("Attempted to retrieve non-existent block height")); + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a block height from the db")); + + txn.commit(); + + return (uint64_t)result - 1; +} + +block_header BlockchainBDB::get_block_header(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + // block_header object is automatically cast from block object + return get_block(h); +} + +block BlockchainBDB::get_block_from_height(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy<uint32_t> key(height + 1); + Dbt_safe result; + auto get_result = m_blocks->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get block from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a block from the db")); + + txn.commit(); + + blobdata bd; + bd.assign(reinterpret_cast<char*>(result.get_data()), result.get_size()); + + block b; + if (!parse_and_validate_block_from_blob(bd, b)) + throw0(DB_ERROR("Failed to parse block from blob retrieved from the db")); + + return b; +} + +uint64_t BlockchainBDB::get_block_timestamp(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy<uint32_t> key(height + 1); + Dbt_copy<uint64_t> result; + auto get_result = m_block_timestamps->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get timestamp from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- timestamp not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a timestamp from the db")); + + txn.commit(); + return result; +} + +uint64_t BlockchainBDB::get_top_block_timestamp() const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + // if no blocks, return 0 + if (m_height == 0) + { + return 0; + } + + return get_block_timestamp(m_height - 1); +} + +size_t BlockchainBDB::get_block_size(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy<uint32_t> key(height + 1); + Dbt_copy<size_t> result; + auto get_result = m_block_sizes->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get block size from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block size not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a block size from the db")); + + txn.commit(); + return result; +} + +difficulty_type BlockchainBDB::get_block_cumulative_difficulty(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__ << " height: " << height); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy<uint32_t> key(height + 1); + Dbt_copy<difficulty_type> result; + auto get_result = m_block_diffs->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get cumulative difficulty from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- difficulty not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a cumulative difficulty from the db")); + + txn.commit(); + return result; +} + +difficulty_type BlockchainBDB::get_block_difficulty(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + difficulty_type diff1 = 0; + difficulty_type diff2 = 0; + + diff1 = get_block_cumulative_difficulty(height); + if (height != 0) + { + diff2 = get_block_cumulative_difficulty(height - 1); + } + + return diff1 - diff2; +} + +uint64_t BlockchainBDB::get_block_already_generated_coins(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy<uint32_t> key(height + 1); + Dbt_copy<uint64_t> result; + auto get_result = m_block_coins->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get generated coins from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block size not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a total generated coins from the db")); + + txn.commit(); + return result; +} + +crypto::hash BlockchainBDB::get_block_hash_from_height(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy<uint32_t> key(height + 1); + Dbt_copy<crypto::hash> result; + auto get_result = m_block_hashes->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + { + throw0(BLOCK_DNE(std::string("Attempt to get hash from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- hash not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a block hash from the db.")); + + txn.commit(); + return result; +} + +std::vector<block> BlockchainBDB::get_blocks_range(const uint64_t& h1, const uint64_t& h2) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + std::vector<block> v; + + for (uint64_t height = h1; height <= h2; ++height) + { + v.push_back(get_block_from_height(height)); + } + + return v; +} + +std::vector<crypto::hash> BlockchainBDB::get_hashes_range(const uint64_t& h1, const uint64_t& h2) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + std::vector<crypto::hash> v; + + for (uint64_t height = h1; height <= h2; ++height) + { + v.push_back(get_block_hash_from_height(height)); + } + + return v; +} + +crypto::hash BlockchainBDB::top_block_hash() const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + if (m_height > 0) + { + return get_block_hash_from_height(m_height - 1); + } + + return null_hash; +} + +block BlockchainBDB::get_top_block() const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + if (m_height > 0) + { + return get_block_from_height(m_height - 1); + } + + block b; + return b; +} + +uint64_t BlockchainBDB::height() const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + return m_height; +} + +bool BlockchainBDB::tx_exists(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy<crypto::hash> key(h); + + TIME_MEASURE_START(time1); + auto get_result = m_txs->exists(txn, &key, 0); + TIME_MEASURE_FINISH(time1); + time_tx_exists += time1; + if (get_result == DB_NOTFOUND) + { + txn.commit(); + LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); + return false; + } + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch transaction from hash")); + + return true; +} + +uint64_t BlockchainBDB::get_tx_unlock_time(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy<crypto::hash> key(h); + Dbt_copy<uint64_t> result; + auto get_result = m_tx_unlocks->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + throw1(TX_DNE(std::string("tx unlock time with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch tx unlock time from hash")); + + return result; +} + +transaction BlockchainBDB::get_tx(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy<crypto::hash> key(h); + Dbt_safe result; + auto get_result = m_txs->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch tx from hash")); + + blobdata bd; + bd.assign(reinterpret_cast<char*>(result.get_data()), result.get_size()); + + transaction tx; + if (!parse_and_validate_tx_from_blob(bd, tx)) + throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); + + txn.commit(); + + return tx; +} + +uint64_t BlockchainBDB::get_tx_count() const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + DB_BTREE_STAT* stats; + + // DB_FAST_STAT can apparently cause an incorrect number of records + // to be returned. The flag should be set to 0 instead if this proves + // to be the case. + m_txs->stat(txn, &stats, DB_FAST_STAT); + auto num_txs = stats->bt_nkeys; + delete stats; + + txn.commit(); + + return num_txs; +} + +std::vector<transaction> BlockchainBDB::get_tx_list(const std::vector<crypto::hash>& hlist) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + std::vector<transaction> v; + + for (auto& h : hlist) + { + v.push_back(get_tx(h)); + } + + return v; +} + +uint64_t BlockchainBDB::get_tx_block_height(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy<crypto::hash> key(h); + Dbt_copy<uint64_t> result; + auto get_result = m_tx_heights->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + { + throw1(TX_DNE(std::string("tx height with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch tx height from hash")); + + txn.commit(); + + return (uint64_t)result - 1; +} + +//FIXME: make sure the random method used here is appropriate +uint64_t BlockchainBDB::get_random_output(const uint64_t& amount) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + uint64_t num_outputs = get_num_outputs(amount); + if (num_outputs == 0) + throw1(OUTPUT_DNE("Attempting to get a random output for an amount, but none exist")); + + return crypto::rand<uint64_t>() % num_outputs; +} + +uint64_t BlockchainBDB::get_num_outputs(const uint64_t& amount) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + bdb_cur cur(txn, m_output_amounts); + + Dbt_copy<uint64_t> k(amount); + Dbt_copy<uint64_t> v; + auto result = cur->get(&k, &v, DB_SET); + if (result == DB_NOTFOUND) + { + return 0; + } + else if (result) + throw0(DB_ERROR("DB error attempting to get number of outputs of an amount")); + + db_recno_t num_elems = 0; + cur->count(&num_elems, 0); + + txn.commit(); + + return num_elems; +} + +crypto::public_key BlockchainBDB::get_output_key(const uint64_t& amount, const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + uint64_t glob_index = get_output_global_index(amount, index); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy<uint32_t> k(glob_index); + Dbt_copy<crypto::public_key> v; + auto get_result = m_output_keys->get(txn, &k, &v, 0); + if (get_result == DB_NOTFOUND) + throw0(DB_ERROR("Attempting to get output pubkey by global index, but key does not exist")); + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db")); + + return v; +} + +// As this is not used, its return is now a blank output. +// This will save on space in the db. +tx_out BlockchainBDB::get_output(const crypto::hash& h, const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + return tx_out(); +} + +// As this is not used, its return is now a blank output. +// This will save on space in the db. +tx_out BlockchainBDB::get_output(const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + return tx_out(); +} + +tx_out_index BlockchainBDB::get_output_tx_and_index_from_global(const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy<uint32_t> k(index); + Dbt_copy<crypto::hash > v; + + auto get_result = m_output_txs->get(txn, &k, &v, 0); + if (get_result == DB_NOTFOUND) + throw1(OUTPUT_DNE("output with given index not in db")); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch output tx hash")); + + crypto::hash tx_hash = v; + + Dbt_copy<uint64_t> result; + get_result = m_output_indices->get(txn, &k, &result, 0); + if (get_result == DB_NOTFOUND) + throw1(OUTPUT_DNE("output with given index not in db")); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch output tx index")); + + txn.commit(); + + return tx_out_index(tx_hash, result); +} + +tx_out_index BlockchainBDB::get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + bdb_cur cur(txn, m_output_amounts); + + Dbt_copy<uint64_t> k(amount); + Dbt_copy<uint64_t> v; + + auto result = cur->get(&k, &v, DB_SET); + if (result == DB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + db_recno_t num_elems = 0; + cur->count(&num_elems, 0); + + if (num_elems <= index) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); + + for (uint64_t i = 0; i < index; ++i) + { + cur->get(&k, &v, DB_NEXT_DUP); + } + + uint64_t glob_index = v; + + cur.close(); + + txn.commit(); + + return get_output_tx_and_index_from_global(glob_index); +} + +std::vector<uint64_t> BlockchainBDB::get_tx_output_indices(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + std::vector<uint64_t> index_vec; + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + bdb_cur cur(txn, m_tx_outputs); + + Dbt_copy<crypto::hash> k(h); + Dbt_copy<uint64_t> v; + auto result = cur->get(&k, &v, DB_SET); + if (result == DB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + db_recno_t num_elems = 0; + cur->count(&num_elems, 0); + + for (uint64_t i = 0; i < num_elems; ++i) + { + index_vec.push_back(v); + cur->get(&k, &v, DB_NEXT_DUP); + } + + cur.close(); + txn.commit(); + + return index_vec; +} + +std::vector<uint64_t> BlockchainBDB::get_tx_amount_output_indices(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + std::vector<uint64_t> index_vec; + std::vector<uint64_t> index_vec2; + + // get the transaction's global output indices first + index_vec = get_tx_output_indices(h); + // these are next used to obtain the amount output indices + + transaction tx = get_tx(h); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + uint64_t i = 0; + uint64_t global_index; + for (const auto& vout : tx.vout) + { + uint64_t amount = vout.amount; + + global_index = index_vec[i]; + + bdb_cur cur(txn, m_output_amounts); + + Dbt_copy<uint64_t> k(amount); + Dbt_copy<uint64_t> v; + + auto result = cur->get(&k, &v, DB_SET); + if (result == DB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + db_recno_t num_elems = 0; + cur->count(&num_elems, 0); + + uint64_t amount_output_index = 0; + uint64_t output_index = 0; + bool found_index = false; + for (uint64_t j = 0; j < num_elems; ++j) + { + output_index = v; + if (output_index == global_index) + { + amount_output_index = j; + found_index = true; + break; + } + cur->get(&k, &v, DB_NEXT_DUP); + } + if (found_index) + { + index_vec2.push_back(amount_output_index); + } + else + { + // not found + cur.close(); + txn.commit(); + throw1(OUTPUT_DNE("specified output not found in db")); + } + + cur.close(); + ++i; + } + + txn.commit(); + + return index_vec2; +} + + + +bool BlockchainBDB::has_key_image(const crypto::key_image& img) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy<crypto::key_image> val_key(img); + if (m_spent_keys->exists(txn, &val_key, 0) == 0) + { + txn.commit(); + return true; + } + + txn.commit(); + return false; +} + +// Ostensibly BerkeleyDB has batch transaction support built-in, +// so the following few functions will be NOP. + +void BlockchainBDB::batch_start() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); +} + +void BlockchainBDB::batch_commit() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); +} + +void BlockchainBDB::batch_stop() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); +} + +void BlockchainBDB::batch_abort() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); +} + +void BlockchainBDB::set_batch_transactions(bool batch_transactions) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + m_batch_transactions = batch_transactions; + LOG_PRINT_L3("batch transactions " << (m_batch_transactions ? "enabled" : "disabled")); +} + +uint64_t BlockchainBDB::add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector<transaction>& txs + ) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + m_write_txn = &txn; + + uint64_t num_outputs = m_num_outputs; + try + { + BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); + m_write_txn = NULL; + + TIME_MEASURE_START(time1); + txn.commit(); + TIME_MEASURE_FINISH(time1); + time_commit1 += time1; + } + catch (const std::exception& e) + { + m_num_outputs = num_outputs; + m_write_txn = NULL; + throw; + } + + return ++m_height; +} + +void BlockchainBDB::pop_block(block& blk, std::vector<transaction>& txs) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + m_write_txn = &txn; + + uint64_t num_outputs = m_num_outputs; + try + { + BlockchainDB::pop_block(blk, txs); + + m_write_txn = NULL; + txn.commit(); + } + catch (...) + { + m_num_outputs = num_outputs; + m_write_txn = NULL; + throw; + } + + --m_height; +} + +} // namespace cryptonote diff --git a/src/blockchain_db/berkeleydb/db_bdb.h b/src/blockchain_db/berkeleydb/db_bdb.h new file mode 100644 index 000000000..d4eb5434c --- /dev/null +++ b/src/blockchain_db/berkeleydb/db_bdb.h @@ -0,0 +1,288 @@ +// Copyright (c) 2014, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <db_cxx.h> + +#include "blockchain_db/blockchain_db.h" +#include "cryptonote_protocol/blobdatatype.h" // for type blobdata + +namespace cryptonote +{ + +struct bdb_txn_safe +{ + bdb_txn_safe() : m_txn(NULL) { } + ~bdb_txn_safe() + { + LOG_PRINT_L3("bdb_txn_safe: destructor"); + + if (m_txn != NULL) + abort(); + } + + void commit(std::string message = "") + { + if (message.size() == 0) + { + message = "Failed to commit a transaction to the db"; + } + + if (m_txn->commit(0)) + { + m_txn = NULL; + LOG_PRINT_L0(message); + throw DB_ERROR(message.c_str()); + } + m_txn = NULL; + } + + void abort() + { + LOG_PRINT_L3("bdb_txn_safe: abort()"); + if(m_txn != NULL) + { + m_txn->abort(); + m_txn = NULL; + } + else + { + LOG_PRINT_L0("WARNING: bdb_txn_safe: abort() called, but m_txn is NULL"); + } + } + + operator DbTxn*() + { + return m_txn; + } + + operator DbTxn**() + { + return &m_txn; + } + + DbTxn* m_txn; +}; + +class BlockchainBDB : public BlockchainDB +{ +public: + BlockchainBDB(bool batch_transactions=false); + ~BlockchainBDB(); + + virtual void open(const std::string& filename, const int db_flags); + + virtual void close(); + + virtual void sync(); + + virtual void reset(); + + virtual std::vector<std::string> get_filenames() const; + + virtual std::string get_db_name() const; + + virtual bool lock(); + + virtual void unlock(); + + virtual bool block_exists(const crypto::hash& h) const; + + virtual block get_block(const crypto::hash& h) const; + + virtual uint64_t get_block_height(const crypto::hash& h) const; + + virtual block_header get_block_header(const crypto::hash& h) const; + + virtual block get_block_from_height(const uint64_t& height) const; + + virtual uint64_t get_block_timestamp(const uint64_t& height) const; + + virtual uint64_t get_top_block_timestamp() const; + + virtual size_t get_block_size(const uint64_t& height) const; + + virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const; + + virtual difficulty_type get_block_difficulty(const uint64_t& height) const; + + virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const; + + virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const; + + virtual std::vector<block> get_blocks_range(const uint64_t& h1, const uint64_t& h2) const; + + virtual std::vector<crypto::hash> get_hashes_range(const uint64_t& h1, const uint64_t& h2) const; + + virtual crypto::hash top_block_hash() const; + + virtual block get_top_block() const; + + virtual uint64_t height() const; + + virtual bool tx_exists(const crypto::hash& h) const; + + virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const; + + virtual transaction get_tx(const crypto::hash& h) const; + + virtual uint64_t get_tx_count() const; + + virtual std::vector<transaction> get_tx_list(const std::vector<crypto::hash>& hlist) const; + + virtual uint64_t get_tx_block_height(const crypto::hash& h) const; + + virtual uint64_t get_random_output(const uint64_t& amount) const; + + virtual uint64_t get_num_outputs(const uint64_t& amount) const; + + virtual crypto::public_key get_output_key(const uint64_t& amount, const uint64_t& index) const; + + virtual tx_out get_output(const crypto::hash& h, const uint64_t& index) const; + + /** + * @brief get an output from its global index + * + * @param index global index of the output desired + * + * @return the output associated with the index. + * Will throw OUTPUT_DNE if not output has that global index. + * Will throw DB_ERROR if there is a non-specific LMDB error in fetching + */ + tx_out get_output(const uint64_t& index) const; + + virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const; + + virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const; + + virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const; + virtual std::vector<uint64_t> get_tx_amount_output_indices(const crypto::hash& h) const; + + virtual bool has_key_image(const crypto::key_image& img) const; + + virtual uint64_t add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector<transaction>& txs + ); + + virtual void set_batch_transactions(bool batch_transactions); + virtual void batch_start(); + virtual void batch_commit(); + virtual void batch_stop(); + virtual void batch_abort(); + + virtual void pop_block(block& blk, std::vector<transaction>& txs); + +private: + virtual void add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const crypto::hash& block_hash + ); + + virtual void remove_block(); + + virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash); + + virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx); + + virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index); + + virtual void remove_output(const tx_out& tx_output); + + void remove_tx_outputs(const crypto::hash& tx_hash, const transaction& tx); + + void remove_output(const uint64_t& out_index, const uint64_t amount); + void remove_amount_output_index(const uint64_t amount, const uint64_t global_output_index); + + virtual void add_spent_key(const crypto::key_image& k_image); + + virtual void remove_spent_key(const crypto::key_image& k_image); + + /** + * @brief convert a tx output to a blob for storage + * + * @param output the output to convert + * + * @return the resultant blob + */ + blobdata output_to_blob(const tx_out& output); + + /** + * @brief convert a tx output blob to a tx output + * + * @param blob the blob to convert + * + * @return the resultant tx output + */ + tx_out output_from_blob(const blobdata& blob) const; + + /** + * @brief get the global index of the index-th output of the given amount + * + * @param amount the output amount + * @param index the index into the set of outputs of that amount + * + * @return the global index of the desired output + */ + uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index) const; + + void check_open() const; + + DbEnv* m_env; + + Db* m_blocks; + Db* m_block_heights; + Db* m_block_hashes; + Db* m_block_timestamps; + Db* m_block_sizes; + Db* m_block_diffs; + Db* m_block_coins; + + Db* m_txs; + Db* m_tx_unlocks; + Db* m_tx_heights; + Db* m_tx_outputs; + + Db* m_output_txs; + Db* m_output_indices; + Db* m_output_amounts; + Db* m_output_keys; + + Db* m_spent_keys; + + uint64_t m_height; + uint64_t m_num_outputs; + std::string m_folder; + bdb_txn_safe *m_write_txn; + + bool m_batch_transactions; // support for batch transactions +}; + +} // namespace cryptonote diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp new file mode 100644 index 000000000..d648be44e --- /dev/null +++ b/src/blockchain_db/blockchain_db.cpp @@ -0,0 +1,185 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "blockchain_db.h" +#include "cryptonote_core/cryptonote_format_utils.h" +#include "profile_tools.h" + +using epee::string_tools::pod_to_hex; + +namespace cryptonote +{ + +void BlockchainDB::pop_block() +{ + block blk; + std::vector<transaction> txs; + pop_block(blk, txs); +} + +void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr) +{ + crypto::hash tx_hash; + if (!tx_hash_ptr) + { + // should only need to compute hash for miner transactions + tx_hash = get_transaction_hash(tx); + LOG_PRINT_L3("null tx_hash_ptr - needed to compute: " << tx_hash); + } + else + { + tx_hash = *tx_hash_ptr; + } + + add_transaction_data(blk_hash, tx, tx_hash); + + // iterate tx.vout using indices instead of C++11 foreach syntax because + // we need the index + if (tx.vout.size() != 0) // it may be technically possible for a tx to have no outputs + { + for (uint64_t i = 0; i < tx.vout.size(); ++i) + { + add_output(tx_hash, tx.vout[i], i); + } + + for (const txin_v& tx_input : tx.vin) + { + if (tx_input.type() == typeid(txin_to_key)) + { + add_spent_key(boost::get<txin_to_key>(tx_input).k_image); + } + } + } +} + +uint64_t BlockchainDB::add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector<transaction>& txs + ) +{ + TIME_MEASURE_START(time1); + crypto::hash blk_hash = get_block_hash(blk); + TIME_MEASURE_FINISH(time1); + time_blk_hash += time1; + + // call out to subclass implementation to add the block & metadata + time1 = epee::misc_utils::get_tick_count(); + add_block(blk, block_size, cumulative_difficulty, coins_generated, blk_hash); + TIME_MEASURE_FINISH(time1); + time_add_block1 += time1; + + // call out to add the transactions + + time1 = epee::misc_utils::get_tick_count(); + add_transaction(blk_hash, blk.miner_tx); + int tx_i = 0; + crypto::hash tx_hash = null_hash; + for (const transaction& tx : txs) + { + tx_hash = blk.tx_hashes[tx_i]; + add_transaction(blk_hash, tx, &tx_hash); + ++tx_i; + } + TIME_MEASURE_FINISH(time1); + time_add_transaction += time1; + + ++num_calls; + + return height(); +} + +void BlockchainDB::pop_block(block& blk, std::vector<transaction>& txs) +{ + blk = get_top_block(); + + remove_block(); + + remove_transaction(get_transaction_hash(blk.miner_tx)); + for (const auto& h : blk.tx_hashes) + { + txs.push_back(get_tx(h)); + remove_transaction(h); + } +} + +bool BlockchainDB::is_open() +{ + return m_open; +} + +void BlockchainDB::remove_transaction(const crypto::hash& tx_hash) +{ + transaction tx = get_tx(tx_hash); + + for (const txin_v& tx_input : tx.vin) + { + if (tx_input.type() == typeid(txin_to_key)) + { + remove_spent_key(boost::get<txin_to_key>(tx_input).k_image); + } + } + + // need tx as tx.vout has the tx outputs, and the output amounts are needed + remove_transaction_data(tx_hash, tx); +} + +void BlockchainDB::reset_stats() +{ + num_calls = 0; + time_blk_hash = 0; + time_tx_exists = 0; + time_add_block1 = 0; + time_add_transaction = 0; + time_commit1 = 0; +} + +void BlockchainDB::show_stats() +{ + LOG_PRINT_L1(ENDL + << "*********************************" + << ENDL + << "num_calls: " << num_calls + << ENDL + << "time_blk_hash: " << time_blk_hash << "ms" + << ENDL + << "time_tx_exists: " << time_tx_exists << "ms" + << ENDL + << "time_add_block1: " << time_add_block1 << "ms" + << ENDL + << "time_add_transaction: " << time_add_transaction << "ms" + << ENDL + << "time_commit1: " << time_commit1 << "ms" + << ENDL + << "*********************************" + << ENDL + ); +} + +} // namespace cryptonote diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h new file mode 100644 index 000000000..04d9c5384 --- /dev/null +++ b/src/blockchain_db/blockchain_db.h @@ -0,0 +1,492 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#ifndef BLOCKCHAIN_DB_H +#define BLOCKCHAIN_DB_H + +#include <list> +#include <string> +#include <exception> +#include "crypto/hash.h" +#include "cryptonote_core/cryptonote_basic.h" +#include "cryptonote_core/difficulty.h" + +/* DB Driver Interface + * + * The DB interface is a store for the canonical block chain. + * It serves as a persistent storage for the blockchain. + * + * For the sake of efficiency, the reference implementation will also + * store some blockchain data outside of the blocks, such as spent + * transfer key images, unspent transaction outputs, etc. + * + * Transactions are duplicated so that we don't have to fetch a whole block + * in order to fetch a transaction from that block. If this is deemed + * unnecessary later, this can change. + * + * Spent key images are duplicated outside of the blocks so it is quick + * to verify an output hasn't already been spent + * + * Unspent transaction outputs are duplicated to quickly gather random + * outputs to use for mixins + * + * IMPORTANT: + * A concrete implementation of this interface should populate these + * duplicated members! It is possible to have a partial implementation + * of this interface call to private members of the interface to be added + * later that will then populate as needed. + * + * General: + * open() + * is_open() + * close() + * sync() + * reset() + * + * Lock and unlock provided for reorg externally, and for block + * additions internally, this way threaded reads are completely fine + * unless the blockchain is changing. + * bool lock() + * unlock() + * + * vector<str> get_filenames() + * + * Blocks: + * bool block_exists(hash) + * height add_block(block, block_size, cumulative_difficulty, coins_generated, transactions) + * block get_block(hash) + * height get_block_height(hash) + * header get_block_header(hash) + * block get_block_from_height(height) + * size_t get_block_size(height) + * difficulty get_block_cumulative_difficulty(height) + * uint64_t get_block_already_generated_coins(height) + * uint64_t get_block_timestamp(height) + * uint64_t get_top_block_timestamp() + * hash get_block_hash_from_height(height) + * blocks get_blocks_range(height1, height2) + * hashes get_hashes_range(height1, height2) + * hash top_block_hash() + * block get_top_block() + * height height() + * void pop_block(block&, tx_list&) + * + * Transactions: + * bool tx_exists(hash) + * uint64_t get_tx_unlock_time(hash) + * tx get_tx(hash) + * uint64_t get_tx_count() + * tx_list get_tx_list(hash_list) + * height get_tx_block_height(hash) + * + * Outputs: + * index get_random_output(amount) + * uint64_t get_num_outputs(amount) + * pub_key get_output_key(amount, index) + * tx_out get_output(tx_hash, index) + * hash,index get_output_tx_and_index_from_global(index) + * hash,index get_output_tx_and_index(amount, index) + * vec<uint64> get_tx_output_indices(tx_hash) + * + * + * Spent Output Key Images: + * bool has_key_image(key_image) + * + * Exceptions: + * DB_ERROR -- generic + * DB_OPEN_FAILURE + * DB_CREATE_FAILURE + * DB_SYNC_FAILURE + * BLOCK_DNE + * BLOCK_PARENT_DNE + * BLOCK_EXISTS + * BLOCK_INVALID -- considering making this multiple errors + * TX_DNE + * TX_EXISTS + * OUTPUT_DNE + * OUTPUT_EXISTS + * KEY_IMAGE_EXISTS + */ + +namespace cryptonote +{ + +// typedef for convenience +typedef std::pair<crypto::hash, uint64_t> tx_out_index; + +/*********************************** + * Exception Definitions + ***********************************/ +class DB_EXCEPTION : public std::exception +{ + private: + std::string m; + + protected: + DB_EXCEPTION(const char *s) : m(s) { } + + public: + virtual ~DB_EXCEPTION() { } + + const char* what() const throw() + { + return m.c_str(); + } +}; + +class DB_ERROR : public DB_EXCEPTION +{ + public: + DB_ERROR() : DB_EXCEPTION("Generic DB Error") { } + DB_ERROR(const char* s) : DB_EXCEPTION(s) { } +}; + +class DB_OPEN_FAILURE : public DB_EXCEPTION +{ + public: + DB_OPEN_FAILURE() : DB_EXCEPTION("Failed to open the db") { } + DB_OPEN_FAILURE(const char* s) : DB_EXCEPTION(s) { } +}; + +class DB_CREATE_FAILURE : public DB_EXCEPTION +{ + public: + DB_CREATE_FAILURE() : DB_EXCEPTION("Failed to create the db") { } + DB_CREATE_FAILURE(const char* s) : DB_EXCEPTION(s) { } +}; + +class DB_SYNC_FAILURE : public DB_EXCEPTION +{ + public: + DB_SYNC_FAILURE() : DB_EXCEPTION("Failed to sync the db") { } + DB_SYNC_FAILURE(const char* s) : DB_EXCEPTION(s) { } +}; + +class BLOCK_DNE : public DB_EXCEPTION +{ + public: + BLOCK_DNE() : DB_EXCEPTION("The block requested does not exist") { } + BLOCK_DNE(const char* s) : DB_EXCEPTION(s) { } +}; + +class BLOCK_PARENT_DNE : public DB_EXCEPTION +{ + public: + BLOCK_PARENT_DNE() : DB_EXCEPTION("The parent of the block does not exist") { } + BLOCK_PARENT_DNE(const char* s) : DB_EXCEPTION(s) { } +}; + +class BLOCK_EXISTS : public DB_EXCEPTION +{ + public: + BLOCK_EXISTS() : DB_EXCEPTION("The block to be added already exists!") { } + BLOCK_EXISTS(const char* s) : DB_EXCEPTION(s) { } +}; + +class BLOCK_INVALID : public DB_EXCEPTION +{ + public: + BLOCK_INVALID() : DB_EXCEPTION("The block to be added did not pass validation!") { } + BLOCK_INVALID(const char* s) : DB_EXCEPTION(s) { } +}; + +class TX_DNE : public DB_EXCEPTION +{ + public: + TX_DNE() : DB_EXCEPTION("The transaction requested does not exist") { } + TX_DNE(const char* s) : DB_EXCEPTION(s) { } +}; + +class TX_EXISTS : public DB_EXCEPTION +{ + public: + TX_EXISTS() : DB_EXCEPTION("The transaction to be added already exists!") { } + TX_EXISTS(const char* s) : DB_EXCEPTION(s) { } +}; + +class OUTPUT_DNE : public DB_EXCEPTION +{ + public: + OUTPUT_DNE() : DB_EXCEPTION("The output requested does not exist!") { } + OUTPUT_DNE(const char* s) : DB_EXCEPTION(s) { } +}; + +class OUTPUT_EXISTS : public DB_EXCEPTION +{ + public: + OUTPUT_EXISTS() : DB_EXCEPTION("The output to be added already exists!") { } + OUTPUT_EXISTS(const char* s) : DB_EXCEPTION(s) { } +}; + +class KEY_IMAGE_EXISTS : public DB_EXCEPTION +{ + public: + KEY_IMAGE_EXISTS() : DB_EXCEPTION("The spent key image to be added already exists!") { } + KEY_IMAGE_EXISTS(const char* s) : DB_EXCEPTION(s) { } +}; + +/*********************************** + * End of Exception Definitions + ***********************************/ + + +class BlockchainDB +{ +private: + /********************************************************************* + * private virtual members + *********************************************************************/ + + // tells the subclass to add the block and metadata to storage + virtual void add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const crypto::hash& blk_hash + ) = 0; + + // tells the subclass to remove data about the top block + virtual void remove_block() = 0; + + // tells the subclass to store the transaction and its metadata + virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) = 0; + + // tells the subclass to remove data about a transaction + virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) = 0; + + // tells the subclass to store an output + virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index) = 0; + + // tells the subclass to remove an output + virtual void remove_output(const tx_out& tx_output) = 0; + + // tells the subclass to store a spent key + virtual void add_spent_key(const crypto::key_image& k_image) = 0; + + // tells the subclass to remove a spent key + virtual void remove_spent_key(const crypto::key_image& k_image) = 0; + + + /********************************************************************* + * private concrete members + *********************************************************************/ + // private version of pop_block, for undoing if an add_block goes tits up + void pop_block(); + + // helper function for add_transactions, to add each individual tx + void add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr = NULL); + + // helper function to remove transaction from blockchain + void remove_transaction(const crypto::hash& tx_hash); + + uint64_t num_calls = 0; + uint64_t time_blk_hash = 0; + uint64_t time_add_block1 = 0; + uint64_t time_add_transaction = 0; + + +protected: + + mutable uint64_t time_tx_exists = 0; + uint64_t time_commit1 = 0; + + +public: + + // virtual dtor + virtual ~BlockchainDB() { }; + + // reset profiling stats + void reset_stats(); + + // show profiling stats + void show_stats(); + + // open the db at location <filename>, or create it if there isn't one. + virtual void open(const std::string& filename, const int db_flags = 0) = 0; + + // returns true of the db is open/ready, else false + bool is_open(); + + // close and sync the db + virtual void close() = 0; + + // sync the db + virtual void sync() = 0; + + // reset the db -- USE WITH CARE + virtual void reset() = 0; + + // get all files used by this db (if any) + virtual std::vector<std::string> get_filenames() const = 0; + + // return the name of the folder the db's file(s) should reside in + virtual std::string get_db_name() const = 0; + + + // FIXME: these are just for functionality mocking, need to implement + // RAII-friendly and multi-read one-write friendly locking mechanism + // + // acquire db lock + virtual bool lock() = 0; + + // release db lock + virtual void unlock() = 0; + + virtual void batch_start() = 0; + virtual void batch_stop() = 0; + virtual void set_batch_transactions(bool) = 0; + + // adds a block with the given metadata to the top of the blockchain, returns the new height + // NOTE: subclass implementations of this (or the functions it calls) need + // to handle undoing any partially-added blocks in the event of a failure. + virtual uint64_t add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector<transaction>& txs + ); + + // return true if a block with hash <h> exists in the blockchain + virtual bool block_exists(const crypto::hash& h) const = 0; + + // return block with hash <h> + virtual block get_block(const crypto::hash& h) const = 0; + + // return the height of the block with hash <h> on the blockchain, + // throw if it doesn't exist + virtual uint64_t get_block_height(const crypto::hash& h) const = 0; + + // return header for block with hash <h> + virtual block_header get_block_header(const crypto::hash& h) const = 0; + + // return block at height <height> + virtual block get_block_from_height(const uint64_t& height) const = 0; + + // return timestamp of block at height <height> + virtual uint64_t get_block_timestamp(const uint64_t& height) const = 0; + + // return timestamp of most recent block + virtual uint64_t get_top_block_timestamp() const = 0; + + // return block size of block at height <height> + virtual size_t get_block_size(const uint64_t& height) const = 0; + + // return cumulative difficulty up to and including block at height <height> + virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const = 0; + + // return difficulty of block at height <height> + virtual difficulty_type get_block_difficulty(const uint64_t& height) const = 0; + + // return number of coins generated up to and including block at height <height> + virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const = 0; + + // return hash of block at height <height> + virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const = 0; + + // return vector of blocks in range <h1,h2> of height (inclusively) + virtual std::vector<block> get_blocks_range(const uint64_t& h1, const uint64_t& h2) const = 0; + + // return vector of block hashes in range <h1, h2> of height (inclusively) + virtual std::vector<crypto::hash> get_hashes_range(const uint64_t& h1, const uint64_t& h2) const = 0; + + // return the hash of the top block on the chain + virtual crypto::hash top_block_hash() const = 0; + + // return the block at the top of the blockchain + virtual block get_top_block() const = 0; + + // return the height of the chain + virtual uint64_t height() const = 0; + + // pops the top block off the blockchain. + // Returns by reference the popped block and its associated transactions + // + // IMPORTANT: + // When a block is popped, the transactions associated with it need to be + // removed, as well as outputs and spent key images associated with + // those transactions. + virtual void pop_block(block& blk, std::vector<transaction>& txs); + + + // return true if a transaction with hash <h> exists + virtual bool tx_exists(const crypto::hash& h) const = 0; + + // return unlock time of tx with hash <h> + virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const = 0; + + // return tx with hash <h> + // throw if no such tx exists + virtual transaction get_tx(const crypto::hash& h) const = 0; + + // returns the total number of transactions in all blocks + virtual uint64_t get_tx_count() const = 0; + + // return list of tx with hashes <hlist>. + // TODO: decide if a missing hash means return empty list + // or just skip that hash + virtual std::vector<transaction> get_tx_list(const std::vector<crypto::hash>& hlist) const = 0; + + // returns height of block that contains transaction with hash <h> + virtual uint64_t get_tx_block_height(const crypto::hash& h) const = 0; + + // return global output index of a random output of amount <amount> + virtual uint64_t get_random_output(const uint64_t& amount) const = 0; + + // returns the total number of outputs of amount <amount> + virtual uint64_t get_num_outputs(const uint64_t& amount) const = 0; + + // return public key for output with global output amount <amount> and index <index> + virtual crypto::public_key get_output_key(const uint64_t& amount, const uint64_t& index) const = 0; + + // returns the output indexed by <index> in the transaction with hash <h> + virtual tx_out get_output(const crypto::hash& h, const uint64_t& index) const = 0; + + // returns the tx hash associated with an output, referenced by global output index + virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const = 0; + + // returns the transaction-local reference for the output with <amount> at <index> + // return type is pair of tx hash and index + virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const = 0; + + // return a vector of indices corresponding to the global output index for + // each output in the transaction with hash <h> + virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const = 0; + // return a vector of indices corresponding to the amount output index for + // each output in the transaction with hash <h> + virtual std::vector<uint64_t> get_tx_amount_output_indices(const crypto::hash& h) const = 0; + + // returns true if key image <img> is present in spent key images storage + virtual bool has_key_image(const crypto::key_image& img) const = 0; + + bool m_open; +}; // class BlockchainDB + + +} // namespace cryptonote + +#endif // BLOCKCHAIN_DB_H diff --git a/src/blockchain_db/db_types.h b/src/blockchain_db/db_types.h new file mode 100644 index 000000000..b13007df4 --- /dev/null +++ b/src/blockchain_db/db_types.h @@ -0,0 +1,40 @@ +// Copyright (c) 2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers +#pragma once + +namespace cryptonote +{ + + const std::unordered_set<std::string> blockchain_db_types = + { "lmdb" + , "berkeley" + }; + +} // namespace cryptonote diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp new file mode 100644 index 000000000..8e09dfab2 --- /dev/null +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -0,0 +1,1814 @@ +// Copyright (c) 2014, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "db_lmdb.h" + +#include <boost/filesystem.hpp> +#include <memory> // std::unique_ptr +#include <cstring> // memcpy + +#include "cryptonote_core/cryptonote_format_utils.h" +#include "crypto/crypto.h" +#include "profile_tools.h" + +using epee::string_tools::pod_to_hex; + +namespace +{ + +template <typename T> +inline void throw0(const T &e) +{ + LOG_PRINT_L0(e.what()); + throw e; +} + +template <typename T> +inline void throw1(const T &e) +{ + LOG_PRINT_L1(e.what()); + throw e; +} + +// cursor needs to be closed when it goes out of scope, +// this helps if the function using it throws +struct lmdb_cur +{ + lmdb_cur(MDB_txn* txn, MDB_dbi dbi) + { + if (mdb_cursor_open(txn, dbi, &m_cur)) + throw0(cryptonote::DB_ERROR("Error opening db cursor")); + done = false; + } + + ~lmdb_cur() { close(); } + + operator MDB_cursor*() { return m_cur; } + operator MDB_cursor**() { return &m_cur; } + + void close() + { + if (!done) + { + mdb_cursor_close(m_cur); + done = true; + } + } + +private: + MDB_cursor* m_cur; + bool done; +}; + +template<typename T> +struct MDB_val_copy: public MDB_val +{ + MDB_val_copy(const T &t): t_copy(t) + { + mv_size = sizeof (T); + mv_data = &t_copy; + } +private: + T t_copy; +}; + +template<> +struct MDB_val_copy<cryptonote::blobdata>: public MDB_val +{ + MDB_val_copy(const cryptonote::blobdata &bd): data(new char[bd.size()]) + { + memcpy(data.get(), bd.data(), bd.size()); + mv_size = bd.size(); + mv_data = data.get(); + } +private: + std::unique_ptr<char[]> data; +}; + +auto compare_uint64 = [](const MDB_val *a, const MDB_val *b) { + const uint64_t va = *(const uint64_t*)a->mv_data; + const uint64_t vb = *(const uint64_t*)b->mv_data; + if (va < vb) return -1; + else if (va == vb) return 0; + else return 1; +}; + +const char* const LMDB_BLOCKS = "blocks"; +const char* const LMDB_BLOCK_TIMESTAMPS = "block_timestamps"; +const char* const LMDB_BLOCK_HEIGHTS = "block_heights"; +const char* const LMDB_BLOCK_HASHES = "block_hashes"; +const char* const LMDB_BLOCK_SIZES = "block_sizes"; +const char* const LMDB_BLOCK_DIFFS = "block_diffs"; +const char* const LMDB_BLOCK_COINS = "block_coins"; + +const char* const LMDB_TXS = "txs"; +const char* const LMDB_TX_UNLOCKS = "tx_unlocks"; +const char* const LMDB_TX_HEIGHTS = "tx_heights"; +const char* const LMDB_TX_OUTPUTS = "tx_outputs"; + +const char* const LMDB_OUTPUT_TXS = "output_txs"; +const char* const LMDB_OUTPUT_INDICES = "output_indices"; +const char* const LMDB_OUTPUT_AMOUNTS = "output_amounts"; +const char* const LMDB_OUTPUT_KEYS = "output_keys"; +const char* const LMDB_OUTPUTS = "outputs"; +const char* const LMDB_OUTPUT_GINDICES = "output_gindices"; +const char* const LMDB_SPENT_KEYS = "spent_keys"; + +inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi, const std::string& error_string) +{ + if (mdb_dbi_open(txn, name, flags, &dbi)) + throw0(cryptonote::DB_OPEN_FAILURE(error_string.c_str())); +} + +} // anonymous namespace + +namespace cryptonote +{ + +// If m_batch_active is set, a batch transaction exists beyond this class, such +// as a batch import with verification enabled, or possibly (later) a batch +// network sync. +// +// For some of the lookup methods, such as get_block_timestamp(), tx_exists(), +// and get_tx(), when m_batch_active is set, the lookup uses the batch +// transaction. This isn't only because the transaction is available, but it's +// necessary so that lookups include the database updates only present in the +// current batch write. +// +// A regular network sync without batch writes is expected to open a new read +// transaction, as those lookups are part of the validation done prior to the +// write for block and tx data, so no write transaction is open at the time. + +void BlockchainLMDB::add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const crypto::hash& blk_hash + ) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + MDB_val_copy<crypto::hash> val_h(blk_hash); + MDB_val unused; + if (mdb_get(*m_write_txn, m_block_heights, &val_h, &unused) == 0) + throw1(BLOCK_EXISTS("Attempting to add block that's already in the db")); + + if (m_height > 0) + { + MDB_val_copy<crypto::hash> parent_key(blk.prev_id); + MDB_val parent_h; + if (mdb_get(*m_write_txn, m_block_heights, &parent_key, &parent_h)) + { + LOG_PRINT_L3("m_height: " << m_height); + LOG_PRINT_L3("parent_key: " << blk.prev_id); + throw0(DB_ERROR("Failed to get top block hash to check for new block's parent")); + } + uint64_t parent_height = *(const uint64_t *)parent_h.mv_data; + if (parent_height != m_height - 1) + throw0(BLOCK_PARENT_DNE("Top block is not new block's parent")); + } + + MDB_val_copy<uint64_t> key(m_height); + + MDB_val_copy<blobdata> blob(block_to_blob(blk)); + auto res = mdb_put(*m_write_txn, m_blocks, &key, &blob, 0); + if (res) + throw0(DB_ERROR(std::string("Failed to add block blob to db transaction: ").append(mdb_strerror(res)).c_str())); + + MDB_val_copy<size_t> sz(block_size); + if (mdb_put(*m_write_txn, m_block_sizes, &key, &sz, 0)) + throw0(DB_ERROR("Failed to add block size to db transaction")); + + MDB_val_copy<uint64_t> ts(blk.timestamp); + if (mdb_put(*m_write_txn, m_block_timestamps, &key, &ts, 0)) + throw0(DB_ERROR("Failed to add block timestamp to db transaction")); + + MDB_val_copy<difficulty_type> diff(cumulative_difficulty); + if (mdb_put(*m_write_txn, m_block_diffs, &key, &diff, 0)) + throw0(DB_ERROR("Failed to add block cumulative difficulty to db transaction")); + + MDB_val_copy<uint64_t> coinsgen(coins_generated); + if (mdb_put(*m_write_txn, m_block_coins, &key, &coinsgen, 0)) + throw0(DB_ERROR("Failed to add block total generated coins to db transaction")); + + if (mdb_put(*m_write_txn, m_block_heights, &val_h, &key, 0)) + throw0(DB_ERROR("Failed to add block height by hash to db transaction")); + + if (mdb_put(*m_write_txn, m_block_hashes, &key, &val_h, 0)) + throw0(DB_ERROR("Failed to add block hash to db transaction")); + +} + +void BlockchainLMDB::remove_block() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + if (m_height == 0) + throw0(BLOCK_DNE ("Attempting to remove block from an empty blockchain")); + + MDB_val_copy<uint64_t> k(m_height - 1); + MDB_val h; + if (mdb_get(*m_write_txn, m_block_hashes, &k, &h)) + throw1(BLOCK_DNE("Attempting to remove block that's not in the db")); + + if (mdb_del(*m_write_txn, m_blocks, &k, NULL)) + throw1(DB_ERROR("Failed to add removal of block to db transaction")); + + if (mdb_del(*m_write_txn, m_block_sizes, &k, NULL)) + throw1(DB_ERROR("Failed to add removal of block size to db transaction")); + + if (mdb_del(*m_write_txn, m_block_diffs, &k, NULL)) + throw1(DB_ERROR("Failed to add removal of block cumulative difficulty to db transaction")); + + if (mdb_del(*m_write_txn, m_block_coins, &k, NULL)) + throw1(DB_ERROR("Failed to add removal of block total generated coins to db transaction")); + + if (mdb_del(*m_write_txn, m_block_timestamps, &k, NULL)) + throw1(DB_ERROR("Failed to add removal of block timestamp to db transaction")); + + if (mdb_del(*m_write_txn, m_block_heights, &h, NULL)) + throw1(DB_ERROR("Failed to add removal of block height by hash to db transaction")); + + if (mdb_del(*m_write_txn, m_block_hashes, &k, NULL)) + throw1(DB_ERROR("Failed to add removal of block hash to db transaction")); +} + +void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + MDB_val_copy<crypto::hash> val_h(tx_hash); + MDB_val unused; + if (mdb_get(*m_write_txn, m_txs, &val_h, &unused) == 0) + throw1(TX_EXISTS("Attempting to add transaction that's already in the db")); + + MDB_val_copy<blobdata> blob(tx_to_blob(tx)); + if (mdb_put(*m_write_txn, m_txs, &val_h, &blob, 0)) + throw0(DB_ERROR("Failed to add tx blob to db transaction")); + + MDB_val_copy<uint64_t> height(m_height); + if (mdb_put(*m_write_txn, m_tx_heights, &val_h, &height, 0)) + throw0(DB_ERROR("Failed to add tx block height to db transaction")); + + MDB_val_copy<uint64_t> unlock_time(tx.unlock_time); + if (mdb_put(*m_write_txn, m_tx_unlocks, &val_h, &unlock_time, 0)) + throw0(DB_ERROR("Failed to add tx unlock time to db transaction")); +} + +void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + MDB_val_copy<crypto::hash> val_h(tx_hash); + MDB_val unused; + if (mdb_get(*m_write_txn, m_txs, &val_h, &unused)) + throw1(TX_DNE("Attempting to remove transaction that isn't in the db")); + + if (mdb_del(*m_write_txn, m_txs, &val_h, NULL)) + throw1(DB_ERROR("Failed to add removal of tx to db transaction")); + if (mdb_del(*m_write_txn, m_tx_unlocks, &val_h, NULL)) + throw1(DB_ERROR("Failed to add removal of tx unlock time to db transaction")); + if (mdb_del(*m_write_txn, m_tx_heights, &val_h, NULL)) + throw1(DB_ERROR("Failed to add removal of tx block height to db transaction")); + + remove_tx_outputs(tx_hash, tx); + + if (mdb_del(*m_write_txn, m_tx_outputs, &val_h, NULL)) + throw1(DB_ERROR("Failed to add removal of tx outputs to db transaction")); + +} + +void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + MDB_val_copy<uint64_t> k(m_num_outputs); + MDB_val_copy<crypto::hash> v(tx_hash); + + if (mdb_put(*m_write_txn, m_output_txs, &k, &v, 0)) + throw0(DB_ERROR("Failed to add output tx hash to db transaction")); + if (mdb_put(*m_write_txn, m_tx_outputs, &v, &k, 0)) + throw0(DB_ERROR("Failed to add tx output index to db transaction")); + + MDB_val_copy<uint64_t> val_local_index(local_index); + if (mdb_put(*m_write_txn, m_output_indices, &k, &val_local_index, 0)) + throw0(DB_ERROR("Failed to add tx output index to db transaction")); + + MDB_val_copy<uint64_t> val_amount(tx_output.amount); + if (auto result = mdb_put(*m_write_txn, m_output_amounts, &val_amount, &k, 0)) + throw0(DB_ERROR(std::string("Failed to add output amount to db transaction").append(mdb_strerror(result)).c_str())); + + if (tx_output.target.type() == typeid(txout_to_key)) + { + MDB_val_copy<crypto::public_key> val_pubkey(boost::get<txout_to_key>(tx_output.target).key); + if (mdb_put(*m_write_txn, m_output_keys, &k, &val_pubkey, 0)) + throw0(DB_ERROR("Failed to add output pubkey to db transaction")); + } + + +/****** Uncomment if ever outputs actually need to be stored in this manner + * + blobdata b = output_to_blob(tx_output); + + v.mv_size = b.size(); + v.mv_data = &b; + if (mdb_put(*m_write_txn, m_outputs, &k, &v, 0)) + throw0(DB_ERROR("Failed to add output to db transaction")); + if (mdb_put(*m_write_txn, m_output_gindices, &v, &k, 0)) + throw0(DB_ERROR("Failed to add output global index to db transaction")); +************************************************************************/ + + m_num_outputs++; +} + +void BlockchainLMDB::remove_tx_outputs(const crypto::hash& tx_hash, const transaction& tx) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + + lmdb_cur cur(*m_write_txn, m_tx_outputs); + + MDB_val_copy<crypto::hash> k(tx_hash); + MDB_val v; + + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + { + LOG_ERROR("Attempting to remove a tx's outputs, but none found. Continuing, but...be wary, because that's weird."); + } + else if (result) + { + throw0(DB_ERROR("DB error attempting to get an output")); + } + else + { + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + + mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); + + for (uint64_t i = 0; i < num_elems; ++i) + { + const tx_out tx_output = tx.vout[i]; + remove_output(*(const uint64_t*)v.mv_data, tx_output.amount); + if (i < num_elems - 1) + { + mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); + } + } + } + + cur.close(); +} + +// TODO: probably remove this function +void BlockchainLMDB::remove_output(const tx_out& tx_output) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " (unused version - does nothing)"); + return; +} + +void BlockchainLMDB::remove_output(const uint64_t& out_index, const uint64_t amount) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + MDB_val_copy<uint64_t> k(out_index); + +/****** Uncomment if ever outputs actually need to be stored in this manner + blobdata b; + t_serializable_object_to_blob(tx_output, b); + k.mv_size = b.size(); + k.mv_data = &b; + + if (mdb_get(*m_write_txn, m_output_gindices, &k, &v)) + throw1(OUTPUT_DNE("Attempting to remove output that does not exist")); + + uint64_t gindex = *(uint64_t*)v.mv_data; + + auto result = mdb_del(*m_write_txn, m_output_gindices, &k, NULL); + if (result != 0 && result != MDB_NOTFOUND) + throw1(DB_ERROR("Error adding removal of output global index to db transaction")); + + result = mdb_del(*m_write_txn, m_outputs, &v, NULL); + if (result != 0 && result != MDB_NOTFOUND) + throw1(DB_ERROR("Error adding removal of output to db transaction")); +*********************************************************************/ + + auto result = mdb_del(*m_write_txn, m_output_indices, &k, NULL); + if (result == MDB_NOTFOUND) + { + LOG_PRINT_L0("Unexpected: global output index not found in m_output_indices"); + } + else if (result) + { + throw1(DB_ERROR("Error adding removal of output tx index to db transaction")); + } + + result = mdb_del(*m_write_txn, m_output_txs, &k, NULL); + // if (result != 0 && result != MDB_NOTFOUND) + // throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); + if (result == MDB_NOTFOUND) + { + LOG_PRINT_L0("Unexpected: global output index not found in m_output_txs"); + } + else if (result) + { + throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); + } + + result = mdb_del(*m_write_txn, m_output_keys, &k, NULL); + if (result == MDB_NOTFOUND) + { + LOG_PRINT_L0("Unexpected: global output index not found in m_output_keys"); + } + else if (result) + throw1(DB_ERROR("Error adding removal of output pubkey to db transaction")); + + remove_amount_output_index(amount, out_index); + + m_num_outputs--; +} + +void BlockchainLMDB::remove_amount_output_index(const uint64_t amount, const uint64_t global_output_index) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + lmdb_cur cur(*m_write_txn, m_output_amounts); + + MDB_val_copy<uint64_t> k(amount); + MDB_val v; + + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + + mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); + + uint64_t amount_output_index = 0; + uint64_t goi = 0; + bool found_index = false; + for (uint64_t i = 0; i < num_elems; ++i) + { + mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); + goi = *(const uint64_t *)v.mv_data; + if (goi == global_output_index) + { + amount_output_index = i; + found_index = true; + break; + } + mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); + } + if (found_index) + { + // found the amount output index + // now delete it + result = mdb_cursor_del(cur, 0); + if (result) + throw0(DB_ERROR(std::string("Error deleting amount output index ").append(boost::lexical_cast<std::string>(amount_output_index)).c_str())); + } + else + { + // not found + cur.close(); + throw1(OUTPUT_DNE("Failed to find amount output index")); + } + cur.close(); +} + +void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + MDB_val_copy<crypto::key_image> val_key(k_image); + MDB_val unused; + if (mdb_get(*m_write_txn, m_spent_keys, &val_key, &unused) == 0) + throw1(KEY_IMAGE_EXISTS("Attempting to add spent key image that's already in the db")); + + char anything = '\0'; + unused.mv_size = sizeof(char); + unused.mv_data = &anything; + if (auto result = mdb_put(*m_write_txn, m_spent_keys, &val_key, &unused, 0)) + throw1(DB_ERROR(std::string("Error adding spent key image to db transaction: ").append(mdb_strerror(result)).c_str())); +} + +void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + MDB_val_copy<crypto::key_image> k(k_image); + auto result = mdb_del(*m_write_txn, m_spent_keys, &k, NULL); + if (result != 0 && result != MDB_NOTFOUND) + throw1(DB_ERROR("Error adding removal of key image to db transaction")); +} + +blobdata BlockchainLMDB::output_to_blob(const tx_out& output) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + blobdata b; + if (!t_serializable_object_to_blob(output, b)) + throw1(DB_ERROR("Error serializing output to blob")); + return b; +} + +tx_out BlockchainLMDB::output_from_blob(const blobdata& blob) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + std::stringstream ss; + ss << blob; + binary_archive<false> ba(ss); + tx_out o; + + if (!(::serialization::serialize(ba, o))) + throw1(DB_ERROR("Error deserializing tx output blob")); + + return o; +} + +uint64_t BlockchainLMDB::get_output_global_index(const uint64_t& amount, const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + lmdb_cur cur(txn, m_output_amounts); + + MDB_val_copy<uint64_t> k(amount); + MDB_val v; + + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + if (num_elems <= index) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); + + mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); + + for (uint64_t i = 0; i < index; ++i) + { + mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); + } + + mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); + + uint64_t glob_index = *(const uint64_t*)v.mv_data; + + cur.close(); + + txn.commit(); + + return glob_index; +} + +void BlockchainLMDB::check_open() const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (!m_open) + throw0(DB_ERROR("DB operation attempted on a not-open DB instance")); +} + +BlockchainLMDB::~BlockchainLMDB() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + + // batch transaction shouldn't be active at this point. If it is, consider it aborted. + if (m_batch_active) + batch_abort(); +} + +BlockchainLMDB::BlockchainLMDB(bool batch_transactions) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + // initialize folder to something "safe" just in case + // someone accidentally misuses this class... + m_folder = "thishsouldnotexistbecauseitisgibberish"; + m_open = false; + + m_batch_transactions = batch_transactions; + m_write_txn = nullptr; + m_batch_active = false; + m_height = 0; +} + +void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + + if (m_open) + throw0(DB_OPEN_FAILURE("Attempted to open db, but it's already open")); + + boost::filesystem::path direc(filename); + if (boost::filesystem::exists(direc)) + { + if (!boost::filesystem::is_directory(direc)) + throw0(DB_OPEN_FAILURE("LMDB needs a directory path, but a file was passed")); + } + else + { + if (!boost::filesystem::create_directory(direc)) + throw0(DB_OPEN_FAILURE(std::string("Failed to create directory ").append(filename).c_str())); + } + + // check for existing LMDB files in base directory + boost::filesystem::path old_files = direc.parent_path(); + if (boost::filesystem::exists(old_files / "data.mdb") || + boost::filesystem::exists(old_files / "lock.mdb")) + { + LOG_PRINT_L0("Found existing LMDB files in " << old_files.c_str()); + LOG_PRINT_L0("Move data.mdb and/or lock.mdb to " << filename << ", or delete them, and then restart"); + throw DB_ERROR("Database could not be opened"); + } + + m_folder = filename; + + // set up lmdb environment + if (mdb_env_create(&m_env)) + throw0(DB_ERROR("Failed to create lmdb environment")); + if (mdb_env_set_maxdbs(m_env, 20)) + throw0(DB_ERROR("Failed to set max number of dbs")); + + size_t mapsize = 1LL << 34; + if (auto result = mdb_env_set_mapsize(m_env, mapsize)) + throw0(DB_ERROR(std::string("Failed to set max memory map size: ").append(mdb_strerror(result)).c_str())); + if (auto result = mdb_env_open(m_env, filename.c_str(), mdb_flags, 0644)) + throw0(DB_ERROR(std::string("Failed to open lmdb environment: ").append(mdb_strerror(result)).c_str())); + + // get a read/write MDB_txn + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, 0, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + // open necessary databases, and set properties as needed + // uses macros to avoid having to change things too many places + lmdb_db_open(txn, LMDB_BLOCKS, MDB_INTEGERKEY | MDB_CREATE, m_blocks, "Failed to open db handle for m_blocks"); + + lmdb_db_open(txn, LMDB_BLOCK_TIMESTAMPS, MDB_INTEGERKEY | MDB_CREATE, m_block_timestamps, "Failed to open db handle for m_block_timestamps"); + lmdb_db_open(txn, LMDB_BLOCK_HEIGHTS, MDB_CREATE, m_block_heights, "Failed to open db handle for m_block_heights"); + lmdb_db_open(txn, LMDB_BLOCK_HASHES, MDB_INTEGERKEY | MDB_CREATE, m_block_hashes, "Failed to open db handle for m_block_hashes"); + lmdb_db_open(txn, LMDB_BLOCK_SIZES, MDB_INTEGERKEY | MDB_CREATE, m_block_sizes, "Failed to open db handle for m_block_sizes"); + lmdb_db_open(txn, LMDB_BLOCK_DIFFS, MDB_INTEGERKEY | MDB_CREATE, m_block_diffs, "Failed to open db handle for m_block_diffs"); + lmdb_db_open(txn, LMDB_BLOCK_COINS, MDB_INTEGERKEY | MDB_CREATE, m_block_coins, "Failed to open db handle for m_block_coins"); + + lmdb_db_open(txn, LMDB_TXS, MDB_CREATE, m_txs, "Failed to open db handle for m_txs"); + lmdb_db_open(txn, LMDB_TX_UNLOCKS, MDB_CREATE, m_tx_unlocks, "Failed to open db handle for m_tx_unlocks"); + lmdb_db_open(txn, LMDB_TX_HEIGHTS, MDB_CREATE, m_tx_heights, "Failed to open db handle for m_tx_heights"); + lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_DUPSORT | MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs"); + + lmdb_db_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE, m_output_txs, "Failed to open db handle for m_output_txs"); + lmdb_db_open(txn, LMDB_OUTPUT_INDICES, MDB_INTEGERKEY | MDB_CREATE, m_output_indices, "Failed to open db handle for m_output_indices"); + lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_DUPSORT | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts"); + lmdb_db_open(txn, LMDB_OUTPUT_KEYS, MDB_INTEGERKEY | MDB_CREATE, m_output_keys, "Failed to open db handle for m_output_keys"); + +/*************** not used, but kept for posterity + lmdb_db_open(txn, LMDB_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, m_outputs, "Failed to open db handle for m_outputs"); + lmdb_db_open(txn, LMDB_OUTPUT_GINDICES, MDB_CREATE, m_output_gindices, "Failed to open db handle for m_output_gindices"); +*************************************************/ + + lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_CREATE, m_spent_keys, "Failed to open db handle for m_spent_keys"); + + mdb_set_dupsort(txn, m_output_amounts, compare_uint64); + mdb_set_dupsort(txn, m_tx_outputs, compare_uint64); + + // get and keep current height + MDB_stat db_stats; + if (mdb_stat(txn, m_blocks, &db_stats)) + throw0(DB_ERROR("Failed to query m_blocks")); + LOG_PRINT_L2("Setting m_height to: " << db_stats.ms_entries); + m_height = db_stats.ms_entries; + + // get and keep current number of outputs + if (mdb_stat(txn, m_output_indices, &db_stats)) + throw0(DB_ERROR("Failed to query m_output_indices")); + m_num_outputs = db_stats.ms_entries; + + // commit the transaction + txn.commit(); + + m_open = true; + // from here, init should be finished +} + +void BlockchainLMDB::close() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (m_batch_active) + { + LOG_PRINT_L3("close() first calling batch_abort() due to active batch transaction"); + batch_abort(); + } + this->sync(); + + // FIXME: not yet thread safe!!! Use with care. + mdb_env_close(m_env); +} + +void BlockchainLMDB::sync() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + // Does nothing unless LMDB environment was opened with MDB_NOSYNC or in part + // MDB_NOMETASYNC. Force flush to be synchronous. + if (auto result = mdb_env_sync(m_env, true)) + { + throw0(DB_ERROR(std::string("Failed to sync database").append(mdb_strerror(result)).c_str())); + } +} + +void BlockchainLMDB::reset() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + // TODO: this +} + +std::vector<std::string> BlockchainLMDB::get_filenames() const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + std::vector<std::string> filenames; + + boost::filesystem::path datafile(m_folder); + datafile /= "data.mdb"; + boost::filesystem::path lockfile(m_folder); + lockfile /= "lock.mdb"; + + filenames.push_back(datafile.string()); + filenames.push_back(lockfile.string()); + + return filenames; +} + +std::string BlockchainLMDB::get_db_name() const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + + return std::string("lmdb"); +} + +// TODO: this? +bool BlockchainLMDB::lock() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + return false; +} + +// TODO: this? +void BlockchainLMDB::unlock() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); +} + +bool BlockchainLMDB::block_exists(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + MDB_val_copy<crypto::hash> key(h); + MDB_val result; + auto get_result = mdb_get(txn, m_block_heights, &key, &result); + if (get_result == MDB_NOTFOUND) + { + txn.commit(); + LOG_PRINT_L3("Block with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); + return false; + } + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch block index from hash")); + + txn.commit(); + return true; +} + +block BlockchainLMDB::get_block(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + return get_block_from_height(get_block_height(h)); +} + +uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + MDB_val_copy<crypto::hash> key(h); + MDB_val result; + auto get_result = mdb_get(txn, m_block_heights, &key, &result); + if (get_result == MDB_NOTFOUND) + throw1(BLOCK_DNE("Attempted to retrieve non-existent block height")); + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a block height from the db")); + + txn.commit(); + return *(const uint64_t*)result.mv_data; +} + +block_header BlockchainLMDB::get_block_header(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + // block_header object is automatically cast from block object + return get_block(h); +} + +block BlockchainLMDB::get_block_from_height(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + MDB_val_copy<uint64_t> key(height); + MDB_val result; + auto get_result = mdb_get(txn, m_blocks, &key, &result); + if (get_result == MDB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get block from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a block from the db")); + + txn.commit(); + + blobdata bd; + bd.assign(reinterpret_cast<char*>(result.mv_data), result.mv_size); + + block b; + if (!parse_and_validate_block_from_blob(bd, b)) + throw0(DB_ERROR("Failed to parse block from blob retrieved from the db")); + + return b; +} + +uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + MDB_val_copy<uint64_t> key(height); + MDB_val result; + auto get_result = mdb_get(*txn_ptr, m_block_timestamps, &key, &result); + if (get_result == MDB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get timestamp from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- timestamp not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a timestamp from the db")); + + if (! m_batch_active) + txn.commit(); + return *(const uint64_t*)result.mv_data; +} + +uint64_t BlockchainLMDB::get_top_block_timestamp() const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + // if no blocks, return 0 + if (m_height == 0) + { + return 0; + } + + return get_block_timestamp(m_height - 1); +} + +size_t BlockchainLMDB::get_block_size(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + + MDB_val_copy<uint64_t> key(height); + MDB_val result; + auto get_result = mdb_get(*txn_ptr, m_block_sizes, &key, &result); + if (get_result == MDB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get block size from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block size not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a block size from the db")); + + if (! m_batch_active) + txn.commit(); + return *(const size_t*)result.mv_data; +} + +difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " height: " << height); + check_open(); + + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + MDB_val_copy<uint64_t> key(height); + MDB_val result; + auto get_result = mdb_get(*txn_ptr, m_block_diffs, &key, &result); + if (get_result == MDB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get cumulative difficulty from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- difficulty not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a cumulative difficulty from the db")); + + if (! m_batch_active) + txn.commit(); + return *(difficulty_type*)result.mv_data; +} + +difficulty_type BlockchainLMDB::get_block_difficulty(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + difficulty_type diff1 = 0; + difficulty_type diff2 = 0; + + diff1 = get_block_cumulative_difficulty(height); + if (height != 0) + { + diff2 = get_block_cumulative_difficulty(height - 1); + } + + return diff1 - diff2; +} + +uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + + MDB_val_copy<uint64_t> key(height); + MDB_val result; + auto get_result = mdb_get(*txn_ptr, m_block_coins, &key, &result); + if (get_result == MDB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get generated coins from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block size not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a total generated coins from the db")); + + if (! m_batch_active) + txn.commit(); + return *(const uint64_t*)result.mv_data; +} + +crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + + MDB_val_copy<uint64_t> key(height); + MDB_val result; + auto get_result = mdb_get(*txn_ptr, m_block_hashes, &key, &result); + if (get_result == MDB_NOTFOUND) + { + throw0(BLOCK_DNE(std::string("Attempt to get hash from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- hash not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR(std::string("Error attempting to retrieve a block hash from the db: "). + append(mdb_strerror(get_result)).c_str())); + + if (! m_batch_active) + txn.commit(); + return *(crypto::hash*)result.mv_data; +} + +std::vector<block> BlockchainLMDB::get_blocks_range(const uint64_t& h1, const uint64_t& h2) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + std::vector<block> v; + + for (uint64_t height = h1; height <= h2; ++height) + { + v.push_back(get_block_from_height(height)); + } + + return v; +} + +std::vector<crypto::hash> BlockchainLMDB::get_hashes_range(const uint64_t& h1, const uint64_t& h2) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + std::vector<crypto::hash> v; + + for (uint64_t height = h1; height <= h2; ++height) + { + v.push_back(get_block_hash_from_height(height)); + } + + return v; +} + +crypto::hash BlockchainLMDB::top_block_hash() const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + if (m_height != 0) + { + return get_block_hash_from_height(m_height - 1); + } + + return null_hash; +} + +block BlockchainLMDB::get_top_block() const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + if (m_height != 0) + { + return get_block_from_height(m_height - 1); + } + + block b; + return b; +} + +uint64_t BlockchainLMDB::height() const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + return m_height; +} + +bool BlockchainLMDB::tx_exists(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + + MDB_val_copy<crypto::hash> key(h); + MDB_val result; + + TIME_MEASURE_START(time1); + auto get_result = mdb_get(*txn_ptr, m_txs, &key, &result); + TIME_MEASURE_FINISH(time1); + time_tx_exists += time1; + if (get_result == MDB_NOTFOUND) + { + if (! m_batch_active) + txn.commit(); + LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); + return false; + } + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch transaction from hash")); + + return true; +} + +uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + MDB_val_copy<crypto::hash> key(h); + MDB_val result; + auto get_result = mdb_get(txn, m_tx_unlocks, &key, &result); + if (get_result == MDB_NOTFOUND) + throw1(TX_DNE(std::string("tx unlock time with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch tx unlock time from hash")); + + return *(const uint64_t*)result.mv_data; +} + +transaction BlockchainLMDB::get_tx(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + + MDB_val_copy<crypto::hash> key(h); + MDB_val result; + auto get_result = mdb_get(*txn_ptr, m_txs, &key, &result); + if (get_result == MDB_NOTFOUND) + throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch tx from hash")); + + blobdata bd; + bd.assign(reinterpret_cast<char*>(result.mv_data), result.mv_size); + + transaction tx; + if (!parse_and_validate_tx_from_blob(bd, tx)) + throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); + if (! m_batch_active) + txn.commit(); + + return tx; +} + +uint64_t BlockchainLMDB::get_tx_count() const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + MDB_stat db_stats; + if (mdb_stat(txn, m_txs, &db_stats)) + throw0(DB_ERROR("Failed to query m_txs")); + + txn.commit(); + + return db_stats.ms_entries; +} + +std::vector<transaction> BlockchainLMDB::get_tx_list(const std::vector<crypto::hash>& hlist) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + std::vector<transaction> v; + + for (auto& h : hlist) + { + v.push_back(get_tx(h)); + } + + return v; +} + +uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + + MDB_val_copy<crypto::hash> key(h); + MDB_val result; + auto get_result = mdb_get(*txn_ptr, m_tx_heights, &key, &result); + if (get_result == MDB_NOTFOUND) + { + throw1(TX_DNE(std::string("tx height with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch tx height from hash")); + + if (! m_batch_active) + txn.commit(); + + return *(const uint64_t*)result.mv_data; +} + +//FIXME: make sure the random method used here is appropriate +uint64_t BlockchainLMDB::get_random_output(const uint64_t& amount) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + uint64_t num_outputs = get_num_outputs(amount); + if (num_outputs == 0) + throw1(OUTPUT_DNE("Attempting to get a random output for an amount, but none exist")); + + return crypto::rand<uint64_t>() % num_outputs; +} + +uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + lmdb_cur cur(txn, m_output_amounts); + + MDB_val_copy<uint64_t> k(amount); + MDB_val v; + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + { + return 0; + } + else if (result) + throw0(DB_ERROR("DB error attempting to get number of outputs of an amount")); + + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + + txn.commit(); + + return num_elems; +} + +crypto::public_key BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + uint64_t glob_index = get_output_global_index(amount, index); + + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + MDB_val_copy<uint64_t> k(glob_index); + MDB_val v; + auto get_result = mdb_get(txn, m_output_keys, &k, &v); + if (get_result == MDB_NOTFOUND) + throw0(DB_ERROR("Attempting to get output pubkey by global index, but key does not exist")); + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db")); + + return *(crypto::public_key*)v.mv_data; +} + +tx_out BlockchainLMDB::get_output(const crypto::hash& h, const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + lmdb_cur cur(txn, m_tx_outputs); + + MDB_val_copy<crypto::hash> k(h); + MDB_val v; + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + if (num_elems <= index) + throw1(OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found")); + + mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); + + for (uint64_t i = 0; i < index; ++i) + { + mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); + } + + mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); + + blobdata b; + b = *(blobdata*)v.mv_data; + + cur.close(); + txn.commit(); + + return output_from_blob(b); +} + +// As this is not used, its return is now a blank output. +// This will save on space in the db. +tx_out BlockchainLMDB::get_output(const uint64_t& index) const +{ + return tx_out(); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + MDB_val_copy<uint64_t> k(index); + MDB_val v; + auto get_result = mdb_get(txn, m_outputs, &k, &v); + if (get_result == MDB_NOTFOUND) + { + throw OUTPUT_DNE("Attempting to get output by global index, but output does not exist"); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve an output from the db")); + + blobdata b = *(blobdata*)v.mv_data; + + return output_from_blob(b); +} + +tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + MDB_val_copy<uint64_t> k(index); + MDB_val v; + + auto get_result = mdb_get(*txn_ptr, m_output_txs, &k, &v); + if (get_result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("output with given index not in db")); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch output tx hash")); + + crypto::hash tx_hash = *(crypto::hash*)v.mv_data; + + get_result = mdb_get(*txn_ptr, m_output_indices, &k, &v); + if (get_result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("output with given index not in db")); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch output tx index")); + if (! m_batch_active) + txn.commit(); + + return tx_out_index(tx_hash, *(const uint64_t *)v.mv_data); +} + +tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + lmdb_cur cur(*txn_ptr, m_output_amounts); + + MDB_val_copy<uint64_t> k(amount); + MDB_val v; + + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + if (num_elems <= index) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); + + mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); + + for (uint64_t i = 0; i < index; ++i) + { + mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); + } + + mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); + + uint64_t glob_index = *(const uint64_t*)v.mv_data; + + cur.close(); + + if (! m_batch_active) + txn.commit(); + + return get_output_tx_and_index_from_global(glob_index); +} + +std::vector<uint64_t> BlockchainLMDB::get_tx_output_indices(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + std::vector<uint64_t> index_vec; + + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + lmdb_cur cur(txn, m_tx_outputs); + + MDB_val_copy<crypto::hash> k(h); + MDB_val v; + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + + mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); + + for (uint64_t i = 0; i < num_elems; ++i) + { + mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); + index_vec.push_back(*(const uint64_t *)v.mv_data); + mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); + } + + cur.close(); + txn.commit(); + + return index_vec; +} + +std::vector<uint64_t> BlockchainLMDB::get_tx_amount_output_indices(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + std::vector<uint64_t> index_vec; + std::vector<uint64_t> index_vec2; + + // get the transaction's global output indices first + index_vec = get_tx_output_indices(h); + // these are next used to obtain the amount output indices + + transaction tx = get_tx(h); + + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + uint64_t i = 0; + uint64_t global_index; + BOOST_FOREACH(const auto& vout, tx.vout) + { + uint64_t amount = vout.amount; + + global_index = index_vec[i]; + + lmdb_cur cur(txn, m_output_amounts); + + MDB_val_copy<uint64_t> k(amount); + MDB_val v; + + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + + mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); + + uint64_t amount_output_index = 0; + uint64_t output_index = 0; + bool found_index = false; + for (uint64_t j = 0; j < num_elems; ++j) + { + mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); + output_index = *(const uint64_t *)v.mv_data; + if (output_index == global_index) + { + amount_output_index = j; + found_index = true; + break; + } + mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); + } + if (found_index) + { + index_vec2.push_back(amount_output_index); + } + else + { + // not found + cur.close(); + txn.commit(); + throw1(OUTPUT_DNE("specified output not found in db")); + } + + cur.close(); + ++i; + } + + txn.commit(); + + return index_vec2; +} + + + +bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + MDB_val_copy<crypto::key_image> val_key(img); + MDB_val unused; + if (mdb_get(txn, m_spent_keys, &val_key, &unused) == 0) + { + txn.commit(); + return true; + } + + txn.commit(); + return false; +} + +void BlockchainLMDB::batch_start() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (! m_batch_transactions) + throw0(DB_ERROR("batch transactions not enabled")); + if (m_batch_active) + throw0(DB_ERROR("batch transaction already in progress")); + if (m_write_txn) + throw0(DB_ERROR("batch transaction attempted, but m_write_txn already in use")); + check_open(); + // NOTE: need to make sure it's destroyed properly when done + if (mdb_txn_begin(m_env, NULL, 0, m_write_batch_txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + // indicates this transaction is for batch transactions, but not whether it's + // active + m_write_batch_txn.m_batch_txn = true; + m_write_txn = &m_write_batch_txn; + m_batch_active = true; + LOG_PRINT_L3("batch transaction: begin"); +} + +void BlockchainLMDB::batch_commit() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (! m_batch_transactions) + throw0(DB_ERROR("batch transactions not enabled")); + if (! m_batch_active) + throw0(DB_ERROR("batch transaction not in progress")); + check_open(); + LOG_PRINT_L3("batch transaction: committing..."); + TIME_MEASURE_START(time1); + m_write_txn->commit(); + TIME_MEASURE_FINISH(time1); + time_commit1 += time1; + LOG_PRINT_L3("batch transaction: committed"); + + if (mdb_txn_begin(m_env, NULL, 0, m_write_batch_txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + if (! m_write_batch_txn.m_batch_txn) + throw0(DB_ERROR("m_write_batch_txn not marked as a batch transaction")); + m_write_txn = &m_write_batch_txn; +} + +void BlockchainLMDB::batch_stop() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (! m_batch_transactions) + throw0(DB_ERROR("batch transactions not enabled")); + if (! m_batch_active) + throw0(DB_ERROR("batch transaction not in progress")); + check_open(); + LOG_PRINT_L3("batch transaction: committing..."); + TIME_MEASURE_START(time1); + m_write_txn->commit(); + TIME_MEASURE_FINISH(time1); + time_commit1 += time1; + // for destruction of batch transaction + m_write_txn = nullptr; + m_batch_active = false; + LOG_PRINT_L3("batch transaction: end"); +} + +void BlockchainLMDB::batch_abort() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (! m_batch_transactions) + throw0(DB_ERROR("batch transactions not enabled")); + if (! m_batch_active) + throw0(DB_ERROR("batch transaction not in progress")); + check_open(); + // for destruction of batch transaction + m_write_txn = nullptr; + // explicitly call in case mdb_env_close() (BlockchainLMDB::close()) called before BlockchainLMDB destructor called. + m_write_batch_txn.abort(); + m_batch_active = false; + LOG_PRINT_L3("batch transaction: aborted"); +} + +void BlockchainLMDB::set_batch_transactions(bool batch_transactions) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + m_batch_transactions = batch_transactions; + LOG_PRINT_L3("batch transactions " << (m_batch_transactions ? "enabled" : "disabled")); +} + +uint64_t BlockchainLMDB::add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector<transaction>& txs + ) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + if (! m_batch_active) + { + if (mdb_txn_begin(m_env, NULL, 0, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + m_write_txn = &txn; + } + + uint64_t num_outputs = m_num_outputs; + try + { + BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); + if (! m_batch_active) + { + m_write_txn = NULL; + + TIME_MEASURE_START(time1); + txn.commit(); + TIME_MEASURE_FINISH(time1); + time_commit1 += time1; + } + } + catch (...) + { + m_num_outputs = num_outputs; + if (! m_batch_active) + m_write_txn = NULL; + throw; + } + + return ++m_height; +} + +void BlockchainLMDB::pop_block(block& blk, std::vector<transaction>& txs) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + mdb_txn_safe txn; + if (! m_batch_active) + { + if (mdb_txn_begin(m_env, NULL, 0, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + m_write_txn = &txn; + } + + uint64_t num_outputs = m_num_outputs; + try + { + BlockchainDB::pop_block(blk, txs); + if (! m_batch_active) + { + m_write_txn = NULL; + + txn.commit(); + } + } + catch (...) + { + m_num_outputs = num_outputs; + m_write_txn = NULL; + throw; + } + + --m_height; +} + +} // namespace cryptonote diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h new file mode 100644 index 000000000..8f1e07e0d --- /dev/null +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -0,0 +1,314 @@ +// Copyright (c) 2014, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include "blockchain_db/blockchain_db.h" +#include "cryptonote_protocol/blobdatatype.h" // for type blobdata + +#include <lmdb.h> + +namespace cryptonote +{ + +struct mdb_txn_safe +{ + mdb_txn_safe() : m_txn(NULL) { } + ~mdb_txn_safe() + { + LOG_PRINT_L3("mdb_txn_safe: destructor"); + if (m_txn != NULL) + { + if (m_batch_txn) // this is a batch txn and should have been handled before this point for safety + { + LOG_PRINT_L0("WARNING: mdb_txn_safe: m_txn is a batch txn and it's not NULL in destructor - calling mdb_txn_abort()"); + } + else + { + // Example of when this occurs: a lookup fails, so a read-only txn is + // aborted through this destructor. However, successful read-only txns + // ideally should have been committed when done and not end up here. + // + // NOTE: not sure if this is ever reached for a non-batch write + // transaction, but it's probably not ideal if it did. + LOG_PRINT_L3("mdb_txn_safe: m_txn not NULL in destructor - calling mdb_txn_abort()"); + } + mdb_txn_abort(m_txn); + } + } + + void commit(std::string message = "") + { + if (message.size() == 0) + { + message = "Failed to commit a transaction to the db"; + } + + if (mdb_txn_commit(m_txn)) + { + m_txn = NULL; + LOG_PRINT_L0(message); + throw DB_ERROR(message.c_str()); + } + m_txn = NULL; + } + + // This should only be needed for batch transaction which must be ensured to + // be aborted before mdb_env_close, not after. So we can't rely on + // BlockchainLMDB destructor to call mdb_txn_safe destructor, as that's too late + // to properly abort, since mdb_env_close would have been called earlier. + void abort() + { + LOG_PRINT_L3("mdb_txn_safe: abort()"); + if(m_txn != NULL) + { + mdb_txn_abort(m_txn); + m_txn = NULL; + } + else + { + LOG_PRINT_L0("WARNING: mdb_txn_safe: abort() called, but m_txn is NULL"); + } + } + + operator MDB_txn*() + { + return m_txn; + } + + operator MDB_txn**() + { + return &m_txn; + } + + MDB_txn* m_txn; + bool m_batch_txn = false; +}; + + +class BlockchainLMDB : public BlockchainDB +{ +public: + BlockchainLMDB(bool batch_transactions=false); + ~BlockchainLMDB(); + + virtual void open(const std::string& filename, const int mdb_flags=0); + + virtual void close(); + + virtual void sync(); + + virtual void reset(); + + virtual std::vector<std::string> get_filenames() const; + + virtual std::string get_db_name() const; + + virtual bool lock(); + + virtual void unlock(); + + virtual bool block_exists(const crypto::hash& h) const; + + virtual block get_block(const crypto::hash& h) const; + + virtual uint64_t get_block_height(const crypto::hash& h) const; + + virtual block_header get_block_header(const crypto::hash& h) const; + + virtual block get_block_from_height(const uint64_t& height) const; + + virtual uint64_t get_block_timestamp(const uint64_t& height) const; + + virtual uint64_t get_top_block_timestamp() const; + + virtual size_t get_block_size(const uint64_t& height) const; + + virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const; + + virtual difficulty_type get_block_difficulty(const uint64_t& height) const; + + virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const; + + virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const; + + virtual std::vector<block> get_blocks_range(const uint64_t& h1, const uint64_t& h2) const; + + virtual std::vector<crypto::hash> get_hashes_range(const uint64_t& h1, const uint64_t& h2) const; + + virtual crypto::hash top_block_hash() const; + + virtual block get_top_block() const; + + virtual uint64_t height() const; + + virtual bool tx_exists(const crypto::hash& h) const; + + virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const; + + virtual transaction get_tx(const crypto::hash& h) const; + + virtual uint64_t get_tx_count() const; + + virtual std::vector<transaction> get_tx_list(const std::vector<crypto::hash>& hlist) const; + + virtual uint64_t get_tx_block_height(const crypto::hash& h) const; + + virtual uint64_t get_random_output(const uint64_t& amount) const; + + virtual uint64_t get_num_outputs(const uint64_t& amount) const; + + virtual crypto::public_key get_output_key(const uint64_t& amount, const uint64_t& index) const; + + virtual tx_out get_output(const crypto::hash& h, const uint64_t& index) const; + + /** + * @brief get an output from its global index + * + * @param index global index of the output desired + * + * @return the output associated with the index. + * Will throw OUTPUT_DNE if not output has that global index. + * Will throw DB_ERROR if there is a non-specific LMDB error in fetching + */ + tx_out get_output(const uint64_t& index) const; + + virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const; + + virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const; + + virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const; + virtual std::vector<uint64_t> get_tx_amount_output_indices(const crypto::hash& h) const; + + virtual bool has_key_image(const crypto::key_image& img) const; + + virtual uint64_t add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector<transaction>& txs + ); + + virtual void set_batch_transactions(bool batch_transactions); + virtual void batch_start(); + virtual void batch_commit(); + virtual void batch_stop(); + virtual void batch_abort(); + + virtual void pop_block(block& blk, std::vector<transaction>& txs); + +private: + virtual void add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const crypto::hash& block_hash + ); + + virtual void remove_block(); + + virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash); + + virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx); + + virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index); + + virtual void remove_output(const tx_out& tx_output); + + void remove_tx_outputs(const crypto::hash& tx_hash, const transaction& tx); + + void remove_output(const uint64_t& out_index, const uint64_t amount); + void remove_amount_output_index(const uint64_t amount, const uint64_t global_output_index); + + virtual void add_spent_key(const crypto::key_image& k_image); + + virtual void remove_spent_key(const crypto::key_image& k_image); + + /** + * @brief convert a tx output to a blob for storage + * + * @param output the output to convert + * + * @return the resultant blob + */ + blobdata output_to_blob(const tx_out& output); + + /** + * @brief convert a tx output blob to a tx output + * + * @param blob the blob to convert + * + * @return the resultant tx output + */ + tx_out output_from_blob(const blobdata& blob) const; + + /** + * @brief get the global index of the index-th output of the given amount + * + * @param amount the output amount + * @param index the index into the set of outputs of that amount + * + * @return the global index of the desired output + */ + uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index) const; + + void check_open() const; + + MDB_env* m_env; + + MDB_dbi m_blocks; + MDB_dbi m_block_heights; + MDB_dbi m_block_hashes; + MDB_dbi m_block_timestamps; + MDB_dbi m_block_sizes; + MDB_dbi m_block_diffs; + MDB_dbi m_block_coins; + + MDB_dbi m_txs; + MDB_dbi m_tx_unlocks; + MDB_dbi m_tx_heights; + MDB_dbi m_tx_outputs; + + MDB_dbi m_output_txs; + MDB_dbi m_output_indices; + MDB_dbi m_output_gindices; + MDB_dbi m_output_amounts; + MDB_dbi m_output_keys; + MDB_dbi m_outputs; + + MDB_dbi m_spent_keys; + + uint64_t m_height; + uint64_t m_num_outputs; + std::string m_folder; + mdb_txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn + mdb_txn_safe m_write_batch_txn; // persist batch txn outside of BlockchainLMDB + + bool m_batch_transactions; // support for batch transactions + bool m_batch_active; // whether batch transaction is in progress +}; + +} // namespace cryptonote diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 739c0adee..620e1add9 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # @@ -39,8 +39,11 @@ set(common_private_headers boost_serialization_helper.h command_line.h dns_utils.h + http_connection.h int-util.h pod-class.h + rpc_client.h + scoped_message_writer.h unordered_containers_boost_serialization.h util.h varint.h) diff --git a/src/common/base58.cpp b/src/common/base58.cpp index 9ea6ba1ba..6adc46ca7 100644 --- a/src/common/base58.cpp +++ b/src/common/base58.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/common/base58.h b/src/common/base58.h index ad4f146a4..1a9ad1a77 100644 --- a/src/common/base58.h +++ b/src/common/base58.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/common/boost_serialization_helper.h b/src/common/boost_serialization_helper.h index 7c5529cc2..c108c52e1 100644 --- a/src/common/boost_serialization_helper.h +++ b/src/common/boost_serialization_helper.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/common/command_line.cpp b/src/common/command_line.cpp index a2e7faf9f..d2cd75e5b 100644 --- a/src/common/command_line.cpp +++ b/src/common/command_line.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -48,4 +48,7 @@ namespace command_line const arg_descriptor<bool> arg_version = {"version", "Output version information"}; const arg_descriptor<std::string> arg_data_dir = {"data-dir", "Specify data directory"}; const arg_descriptor<std::string> arg_testnet_data_dir = {"testnet-data-dir", "Specify testnet data directory"}; + const arg_descriptor<bool> arg_test_drop_download = {"test-drop-download", "For net tests: in download, discard ALL blocks instead checking/saving them (very fast)"}; + const arg_descriptor<uint64_t> arg_test_drop_download_height = {"test-drop-download-height", "Like test-drop-download but disards only after around certain height", 0}; + const arg_descriptor<int> arg_test_dbg_lock_sleep = {"test-dbg-lock-sleep", "Sleep time in ms, defaults to 0 (off), used to debug before/after locking mutex. Values 100 to 1000 are good for tests."}; } diff --git a/src/common/command_line.h b/src/common/command_line.h index 09ae877d3..ae79f0a05 100644 --- a/src/common/command_line.h +++ b/src/common/command_line.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -204,4 +204,7 @@ namespace command_line extern const arg_descriptor<bool> arg_version; extern const arg_descriptor<std::string> arg_data_dir; extern const arg_descriptor<std::string> arg_testnet_data_dir; + extern const arg_descriptor<bool> arg_test_drop_download; + extern const arg_descriptor<uint64_t> arg_test_drop_download_height; + extern const arg_descriptor<int> arg_test_dbg_lock_sleep; } diff --git a/src/common/dns_utils.cpp b/src/common/dns_utils.cpp index d21b538a6..ea7f1078b 100644 --- a/src/common/dns_utils.cpp +++ b/src/common/dns_utils.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -34,7 +34,65 @@ #include <stdlib.h> #include "include_base_utils.h" +#include <boost/filesystem/fstream.hpp> using namespace epee; +namespace bf = boost::filesystem; + +namespace +{ + +/* + * The following two functions were taken from unbound-anchor.c, from + * the unbound library packaged with this source. The license and source + * can be found in $PROJECT_ROOT/external/unbound + */ + +/* Cert builtin commented out until it's used, as the compiler complains + +// return the built in root update certificate +static const char* +get_builtin_cert(void) +{ + return +// The ICANN CA fetched at 24 Sep 2010. Valid to 2028 +"-----BEGIN CERTIFICATE-----\n" +"MIIDdzCCAl+gAwIBAgIBATANBgkqhkiG9w0BAQsFADBdMQ4wDAYDVQQKEwVJQ0FO\n" +"TjEmMCQGA1UECxMdSUNBTk4gQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkxFjAUBgNV\n" +"BAMTDUlDQU5OIFJvb3QgQ0ExCzAJBgNVBAYTAlVTMB4XDTA5MTIyMzA0MTkxMloX\n" +"DTI5MTIxODA0MTkxMlowXTEOMAwGA1UEChMFSUNBTk4xJjAkBgNVBAsTHUlDQU5O\n" +"IENlcnRpZmljYXRpb24gQXV0aG9yaXR5MRYwFAYDVQQDEw1JQ0FOTiBSb290IENB\n" +"MQswCQYDVQQGEwJVUzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKDb\n" +"cLhPNNqc1NB+u+oVvOnJESofYS9qub0/PXagmgr37pNublVThIzyLPGCJ8gPms9S\n" +"G1TaKNIsMI7d+5IgMy3WyPEOECGIcfqEIktdR1YWfJufXcMReZwU4v/AdKzdOdfg\n" +"ONiwc6r70duEr1IiqPbVm5T05l1e6D+HkAvHGnf1LtOPGs4CHQdpIUcy2kauAEy2\n" +"paKcOcHASvbTHK7TbbvHGPB+7faAztABLoneErruEcumetcNfPMIjXKdv1V1E3C7\n" +"MSJKy+jAqqQJqjZoQGB0necZgUMiUv7JK1IPQRM2CXJllcyJrm9WFxY0c1KjBO29\n" +"iIKK69fcglKcBuFShUECAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8B\n" +"Af8EBAMCAf4wHQYDVR0OBBYEFLpS6UmDJIZSL8eZzfyNa2kITcBQMA0GCSqGSIb3\n" +"DQEBCwUAA4IBAQAP8emCogqHny2UYFqywEuhLys7R9UKmYY4suzGO4nkbgfPFMfH\n" +"6M+Zj6owwxlwueZt1j/IaCayoKU3QsrYYoDRolpILh+FPwx7wseUEV8ZKpWsoDoD\n" +"2JFbLg2cfB8u/OlE4RYmcxxFSmXBg0yQ8/IoQt/bxOcEEhhiQ168H2yE5rxJMt9h\n" +"15nu5JBSewrCkYqYYmaxyOC3WrVGfHZxVI7MpIFcGdvSb2a1uyuua8l0BKgk3ujF\n" +"0/wsHNeP22qNyVO+XVBzrM8fk8BSUFuiT/6tZTYXRtEt5aKQZgXbKU5dUF3jT9qg\n" +"j/Br5BZw3X/zd325TvnswzMC1+ljLzHnQGGk\n" +"-----END CERTIFICATE-----\n" + ; +} +*/ + +/** return the built in root DS trust anchor */ +static const char* +get_builtin_ds(void) +{ + return +". IN DS 19036 8 2 49AAC11D7B6F6446702E54A1607371607A1A41855200FD2CE1CDDE32F24E8FB5\n"; +} + +/************************************************************ + ************************************************************ + ***********************************************************/ + +} // anonymous namespace namespace tools { @@ -104,11 +162,28 @@ DNSResolver::DNSResolver() : m_data(new DNSResolverData()) // init libunbound context m_data->m_ub_context = ub_ctx_create(); - char empty_string = '\0'; - // look for "/etc/resolv.conf" and "/etc/hosts" or platform equivalent - ub_ctx_resolvconf(m_data->m_ub_context, &empty_string); - ub_ctx_hosts(m_data->m_ub_context, &empty_string); + ub_ctx_resolvconf(m_data->m_ub_context, NULL); + ub_ctx_hosts(m_data->m_ub_context, NULL); + + #ifdef DEVELOPER_LIBUNBOUND_OLD + #pragma message "Using the work around for old libunbound" + { // work around for bug https://www.nlnetlabs.nl/bugs-script/show_bug.cgi?id=515 needed for it to compile on e.g. Debian 7 + char * ds_copy = NULL; // this will be the writable copy of string that bugged version of libunbound requires + try { + char * ds_copy = strdup( ::get_builtin_ds() ); + ub_ctx_add_ta(m_data->m_ub_context, ds_copy); + } catch(...) { // probably not needed but to work correctly in every case... + if (ds_copy) { free(ds_copy); ds_copy=NULL; } // for the strdup + throw ; + } + if (ds_copy) { free(ds_copy); ds_copy=NULL; } // for the strdup + } + #else + // normal version for fixed libunbound + ub_ctx_add_ta(m_data->m_ub_context, ::get_builtin_ds() ); + #endif + } DNSResolver::~DNSResolver() @@ -143,6 +218,8 @@ std::vector<std::string> DNSResolver::get_ipv4(const std::string& url, bool& dns // call DNS resolver, blocking. if return value not zero, something went wrong if (!ub_resolve(m_data->m_ub_context, urlC, DNS_TYPE_A, DNS_CLASS_IN, &(result.ptr))) { + dnssec_available = (result.ptr->secure || (!result.ptr->secure && result.ptr->bogus)); + dnssec_valid = !result.ptr->bogus; if (result.ptr->havedata) { for (size_t i=0; result.ptr->data[i] != NULL; i++) @@ -175,6 +252,8 @@ std::vector<std::string> DNSResolver::get_ipv6(const std::string& url, bool& dns // call DNS resolver, blocking. if return value not zero, something went wrong if (!ub_resolve(m_data->m_ub_context, urlC, DNS_TYPE_AAAA, DNS_CLASS_IN, &(result.ptr))) { + dnssec_available = (result.ptr->secure || (!result.ptr->secure && result.ptr->bogus)); + dnssec_valid = !result.ptr->bogus; if (result.ptr->havedata) { for (size_t i=0; result.ptr->data[i] != NULL; i++) @@ -207,6 +286,8 @@ std::vector<std::string> DNSResolver::get_txt_record(const std::string& url, boo // call DNS resolver, blocking. if return value not zero, something went wrong if (!ub_resolve(m_data->m_ub_context, urlC, DNS_TYPE_TXT, DNS_CLASS_IN, &(result.ptr))) { + dnssec_available = (result.ptr->secure || (!result.ptr->secure && result.ptr->bogus)); + dnssec_valid = !result.ptr->bogus; if (result.ptr->havedata) { for (size_t i=0; result.ptr->data[i] != NULL; i++) diff --git a/src/common/dns_utils.h b/src/common/dns_utils.h index 9958e4ca2..a16c7eff7 100644 --- a/src/common/dns_utils.h +++ b/src/common/dns_utils.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/common/http_connection.h b/src/common/http_connection.h new file mode 100644 index 000000000..759145009 --- /dev/null +++ b/src/common/http_connection.h @@ -0,0 +1,42 @@ +#pragma once + +#include "string_tools.h" +#include "net/http_client.h" + +namespace tools { + +class t_http_connection { +private: + epee::net_utils::http::http_simple_client * mp_http_client; + bool m_ok; +public: + static unsigned int const TIMEOUT = 200000; + + t_http_connection( + epee::net_utils::http::http_simple_client * p_http_client + , uint32_t ip + , uint16_t port + ) + : mp_http_client(p_http_client) + { + // TODO fix http client so that it accepts properly typed arguments + std::string ip_str = epee::string_tools::get_ip_string_from_int32(ip); + std::string port_str = boost::lexical_cast<std::string>(port); + m_ok = mp_http_client->connect(ip_str, port_str, TIMEOUT); + } + + ~t_http_connection() + { + if (m_ok) + { + mp_http_client->disconnect(); + } + } + + bool is_open() + { + return m_ok; + } +}; // class t_http_connection + +} // namespace tools diff --git a/src/common/int-util.h b/src/common/int-util.h index 40ac2eb64..cd6e65460 100644 --- a/src/common/int-util.h +++ b/src/common/int-util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/common/pod-class.h b/src/common/pod-class.h index 8c7e4c2e9..ebe919203 100644 --- a/src/common/pod-class.h +++ b/src/common/pod-class.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/common/rpc_client.h b/src/common/rpc_client.h new file mode 100644 index 000000000..a6d4b6cc1 --- /dev/null +++ b/src/common/rpc_client.h @@ -0,0 +1,132 @@ +#pragma once + +#include "common/http_connection.h" +#include "common/scoped_message_writer.h" +#include "rpc/core_rpc_server_commands_defs.h" +#include "storages/http_abstract_invoke.h" +#include "net/http_client.h" +#include "string_tools.h" +#include <boost/lexical_cast.hpp> + +namespace tools +{ + class t_rpc_client final + { + private: + epee::net_utils::http::http_simple_client m_http_client; + uint32_t m_ip; + uint16_t m_port; + public: + t_rpc_client( + uint32_t ip + , uint16_t port + ) + : m_http_client{} + , m_ip{ip} + , m_port{port} + {} + + std::string build_url(std::string const & relative_url) + { + std::string result = + "http://" + + epee::string_tools::get_ip_string_from_int32(m_ip) + + ":" + + boost::lexical_cast<std::string>(m_port) + + relative_url; + return result; + } + + template <typename T_req, typename T_res> + bool basic_json_rpc_request( + T_req & req + , T_res & res + , std::string const & method_name + ) + { + std::string rpc_url = build_url("/json_rpc"); + t_http_connection connection(&m_http_client, m_ip, m_port); + + bool ok = connection.is_open(); + if (!ok) + { + fail_msg_writer() << "Couldn't connect to daemon"; + return false; + } + ok = ok && epee::net_utils::invoke_http_json_rpc(rpc_url, method_name, req, res, m_http_client); + if (!ok) + { + fail_msg_writer() << "Daemon request failed"; + return false; + } + else + { + return true; + } + } + + template <typename T_req, typename T_res> + bool json_rpc_request( + T_req & req + , T_res & res + , std::string const & method_name + , std::string const & fail_msg + ) + { + std::string rpc_url = build_url("/json_rpc"); + t_http_connection connection(&m_http_client, m_ip, m_port); + + bool ok = connection.is_open(); + ok = ok && epee::net_utils::invoke_http_json_rpc(rpc_url, method_name, req, res, m_http_client); + if (!ok) + { + fail_msg_writer() << "Couldn't connect to daemon"; + return false; + } + else if (res.status != CORE_RPC_STATUS_OK) // TODO - handle CORE_RPC_STATUS_BUSY ? + { + fail_msg_writer() << fail_msg << " -- " << res.status; + return false; + } + else + { + return true; + } + } + + template <typename T_req, typename T_res> + bool rpc_request( + T_req & req + , T_res & res + , std::string const & relative_url + , std::string const & fail_msg + ) + { + std::string rpc_url = build_url(relative_url); + t_http_connection connection(&m_http_client, m_ip, m_port); + + bool ok = connection.is_open(); + ok = ok && epee::net_utils::invoke_http_json_remote_command2(rpc_url, req, res, m_http_client); + if (!ok) + { + fail_msg_writer() << "Couldn't connect to daemon"; + return false; + } + else if (res.status != CORE_RPC_STATUS_OK) // TODO - handle CORE_RPC_STATUS_BUSY ? + { + fail_msg_writer() << fail_msg << " -- " << res.status; + return false; + } + else + { + return true; + } + } + + bool check_connection() + { + t_http_connection connection(&m_http_client, m_ip, m_port); + return connection.is_open(); + } + }; +} diff --git a/src/common/scoped_message_writer.h b/src/common/scoped_message_writer.h new file mode 100644 index 000000000..70db54eae --- /dev/null +++ b/src/common/scoped_message_writer.h @@ -0,0 +1,95 @@ +#pragma once + +#include "misc_log_ex.h" +#include <iostream> + +namespace tools +{ + +class scoped_message_writer +{ +private: + bool m_flush; + std::stringstream m_oss; + epee::log_space::console_colors m_color; + bool m_bright; + int m_log_level; +public: + scoped_message_writer( + epee::log_space::console_colors color = epee::log_space::console_color_default + , bool bright = false + , std::string&& prefix = std::string() + , int log_level = LOG_LEVEL_2 + ) + : m_flush(true) + , m_color(color) + , m_bright(bright) + , m_log_level(log_level) + { + m_oss << prefix; + } + + scoped_message_writer(scoped_message_writer&& rhs) + : m_flush(std::move(rhs.m_flush)) +#if defined(_MSC_VER) + , m_oss(std::move(rhs.m_oss)) +#else + // GCC bug: http://gcc.gnu.org/bugzilla/show_bug.cgi?id=54316 + , m_oss(rhs.m_oss.str(), std::ios_base::out | std::ios_base::ate) +#endif + , m_color(std::move(rhs.m_color)) + , m_log_level(std::move(rhs.m_log_level)) + { + rhs.m_flush = false; + } + + scoped_message_writer(scoped_message_writer& rhs) = delete; + scoped_message_writer& operator=(scoped_message_writer& rhs) = delete; + scoped_message_writer& operator=(scoped_message_writer&& rhs) = delete; + + template<typename T> + std::ostream& operator<<(const T& val) + { + m_oss << val; + return m_oss; + } + + ~scoped_message_writer() + { + if (m_flush) + { + m_flush = false; + + LOG_PRINT(m_oss.str(), m_log_level) + + if (epee::log_space::console_color_default == m_color) + { + std::cout << m_oss.str(); + } + else + { + epee::log_space::set_console_color(m_color, m_bright); + std::cout << m_oss.str(); + epee::log_space::reset_console_color(); + } + std::cout << std::endl; + } + } +}; + +inline scoped_message_writer success_msg_writer() +{ + return scoped_message_writer(epee::log_space::console_color_green, false, std::string(), LOG_LEVEL_2); +} + +inline scoped_message_writer msg_writer(epee::log_space::console_colors color = epee::log_space::console_color_default) +{ + return scoped_message_writer(color, false, std::string(), LOG_LEVEL_2); +} + +inline scoped_message_writer fail_msg_writer() +{ + return scoped_message_writer(epee::log_space::console_color_red, true, "Error: ", LOG_LEVEL_0); +} + +} // namespace tools diff --git a/src/common/unordered_containers_boost_serialization.h b/src/common/unordered_containers_boost_serialization.h index f432a8cbd..563023624 100644 --- a/src/common/unordered_containers_boost_serialization.h +++ b/src/common/unordered_containers_boost_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/common/util.cpp b/src/common/util.cpp index 0ad1f0791..7d39bc4f4 100644 --- a/src/common/util.cpp +++ b/src/common/util.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -326,7 +326,7 @@ std::string get_nix_version_display_string() std::string config_folder; #ifdef WIN32 - config_folder = get_special_folder_path(CSIDL_APPDATA, true) + "/" + CRYPTONOTE_NAME; + config_folder = get_special_folder_path(CSIDL_COMMON_APPDATA, true) + "\\" + CRYPTONOTE_NAME; #else std::string pathRet; char* pszHome = getenv("HOME"); diff --git a/src/common/util.h b/src/common/util.h index 51f0dad90..883fe1e0f 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -58,6 +58,18 @@ namespace tools */ std::string get_default_data_dir(); +#ifdef WIN32 + /** + * @brief + * + * @param nfolder + * @param iscreate + * + * @return + */ + std::string get_special_folder_path(int nfolder, bool iscreate); +#endif + /*! \brief Returns the OS version string * * \details This is a wrapper around the primitives diff --git a/src/common/varint.h b/src/common/varint.h index 5a73c2746..cd88e12a7 100644 --- a/src/common/varint.h +++ b/src/common/varint.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/connectivity_tool/CMakeLists.txt b/src/connectivity_tool/CMakeLists.txt index b0178c70a..f36963d23 100644 --- a/src/connectivity_tool/CMakeLists.txt +++ b/src/connectivity_tool/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # diff --git a/src/connectivity_tool/conn_tool.cpp b/src/connectivity_tool/conn_tool.cpp index abddd5245..7506dba6f 100644 --- a/src/connectivity_tool/conn_tool.cpp +++ b/src/connectivity_tool/conn_tool.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -49,6 +49,8 @@ namespace po = boost::program_options; using namespace cryptonote; using namespace nodetool; +unsigned int epee::g_test_dbg_lock_sleep = 0; + namespace { const command_line::arg_descriptor<std::string, true> arg_ip = {"ip", "set ip"}; diff --git a/src/crypto/CMakeLists.txt b/src/crypto/CMakeLists.txt index 4afcab9c8..54dd99e26 100644 --- a/src/crypto/CMakeLists.txt +++ b/src/crypto/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # diff --git a/src/crypto/blake256.c b/src/crypto/blake256.c index b6c1bd2f5..63163b51b 100644 --- a/src/crypto/blake256.c +++ b/src/crypto/blake256.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/blake256.h b/src/crypto/blake256.h index b6edbad9c..1545b2a9d 100644 --- a/src/crypto/blake256.h +++ b/src/crypto/blake256.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/chacha8.h b/src/crypto/chacha8.h index 0ce130834..9527e4016 100644 --- a/src/crypto/chacha8.h +++ b/src/crypto/chacha8.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/crypto-ops-data.c b/src/crypto/crypto-ops-data.c index 298399308..c26abacca 100644 --- a/src/crypto/crypto-ops-data.c +++ b/src/crypto/crypto-ops-data.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/crypto-ops.c b/src/crypto/crypto-ops.c index 1a7bc8f45..e3b48fbb2 100644 --- a/src/crypto/crypto-ops.c +++ b/src/crypto/crypto-ops.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/crypto-ops.h b/src/crypto/crypto-ops.h index 8ee75d2fe..0bd1f9f9b 100644 --- a/src/crypto/crypto-ops.h +++ b/src/crypto/crypto-ops.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/crypto.cpp b/src/crypto/crypto.cpp index fa0199f20..01e35f399 100644 --- a/src/crypto/crypto.cpp +++ b/src/crypto/crypto.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/crypto.h b/src/crypto/crypto.h index f72a03f98..afbc9732e 100644 --- a/src/crypto/crypto.h +++ b/src/crypto/crypto.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/generic-ops.h b/src/crypto/generic-ops.h index a08f60dd4..a860a2076 100644 --- a/src/crypto/generic-ops.h +++ b/src/crypto/generic-ops.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/groestl.c b/src/crypto/groestl.c index 00bf987c9..e1c89cc3b 100644 --- a/src/crypto/groestl.c +++ b/src/crypto/groestl.c @@ -357,4 +357,4 @@ static int crypto_hash(unsigned char *out, return 0; } -*/
\ No newline at end of file +*/ diff --git a/src/crypto/groestl.h b/src/crypto/groestl.h index e2c6d2174..a9c907a19 100644 --- a/src/crypto/groestl.h +++ b/src/crypto/groestl.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/groestl_tables.h b/src/crypto/groestl_tables.h index 79e350bf3..d080a53db 100644 --- a/src/crypto/groestl_tables.h +++ b/src/crypto/groestl_tables.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/hash-extra-blake.c b/src/crypto/hash-extra-blake.c index e9007b5d8..d8a7b7e22 100644 --- a/src/crypto/hash-extra-blake.c +++ b/src/crypto/hash-extra-blake.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/hash-extra-groestl.c b/src/crypto/hash-extra-groestl.c index 2b8a77965..6503a0624 100644 --- a/src/crypto/hash-extra-groestl.c +++ b/src/crypto/hash-extra-groestl.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/hash-extra-jh.c b/src/crypto/hash-extra-jh.c index d18ef3993..aaa80ca0f 100644 --- a/src/crypto/hash-extra-jh.c +++ b/src/crypto/hash-extra-jh.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/hash-extra-skein.c b/src/crypto/hash-extra-skein.c index 8818815af..6f18a300e 100644 --- a/src/crypto/hash-extra-skein.c +++ b/src/crypto/hash-extra-skein.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/hash-ops.h b/src/crypto/hash-ops.h index ad5a3407a..5e2790a0c 100644 --- a/src/crypto/hash-ops.h +++ b/src/crypto/hash-ops.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/hash.c b/src/crypto/hash.c index 9a628d0bd..d357ae93f 100644 --- a/src/crypto/hash.c +++ b/src/crypto/hash.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/hash.h b/src/crypto/hash.h index 7b380c29d..2f91a5358 100644 --- a/src/crypto/hash.h +++ b/src/crypto/hash.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/initializer.h b/src/crypto/initializer.h index 61f75cecd..1d4b77be5 100644 --- a/src/crypto/initializer.h +++ b/src/crypto/initializer.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/random.c b/src/crypto/random.c index 727659e82..6cffda2c0 100644 --- a/src/crypto/random.c +++ b/src/crypto/random.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/random.h b/src/crypto/random.h index 85e55fd7d..0f5d8c9b1 100644 --- a/src/crypto/random.h +++ b/src/crypto/random.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/skein_port.h b/src/crypto/skein_port.h index a2a8c45aa..6d25c0a93 100644 --- a/src/crypto/skein_port.h +++ b/src/crypto/skein_port.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/crypto/slow-hash.c b/src/crypto/slow-hash.c index 787545830..425737984 100644 --- a/src/crypto/slow-hash.c +++ b/src/crypto/slow-hash.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -37,6 +37,10 @@ #include "hash-ops.h" #include "oaes_lib.h" +#if defined(__x86_64__) +// Optimised code below, uses x86-specific intrinsics, SSE2, AES-NI +// Fall back to more portable code is down at the bottom + #include <emmintrin.h> #if defined(_MSC_VER) @@ -619,3 +623,163 @@ void cn_slow_hash(const void *data, size_t length, char *hash) hash_permutation(&state.hs); extra_hashes[state.hs.b[0] & 3](&state, 200, hash); } + +#else +// Portable implementation as a fallback + +void slow_hash_allocate_state(void) +{ + // Do nothing, this is just to maintain compatibility with the upgraded slow-hash.c + return; +} + +void slow_hash_free_state(void) +{ + // As above + return; +} + +static void (*const extra_hashes[4])(const void *, size_t, char *) = { + hash_extra_blake, hash_extra_groestl, hash_extra_jh, hash_extra_skein +}; + +#define MEMORY (1 << 21) /* 2 MiB */ +#define ITER (1 << 20) +#define AES_BLOCK_SIZE 16 +#define AES_KEY_SIZE 32 /*16*/ +#define INIT_SIZE_BLK 8 +#define INIT_SIZE_BYTE (INIT_SIZE_BLK * AES_BLOCK_SIZE) + +extern int aesb_single_round(const uint8_t *in, uint8_t*out, const uint8_t *expandedKey); +extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *expandedKey); + +static size_t e2i(const uint8_t* a, size_t count) { return (*((uint64_t*)a) / AES_BLOCK_SIZE) & (count - 1); } + +static void mul(const uint8_t* a, const uint8_t* b, uint8_t* res) { + uint64_t a0, b0; + uint64_t hi, lo; + + a0 = SWAP64LE(((uint64_t*)a)[0]); + b0 = SWAP64LE(((uint64_t*)b)[0]); + lo = mul128(a0, b0, &hi); + ((uint64_t*)res)[0] = SWAP64LE(hi); + ((uint64_t*)res)[1] = SWAP64LE(lo); +} + +static void sum_half_blocks(uint8_t* a, const uint8_t* b) { + uint64_t a0, a1, b0, b1; + + a0 = SWAP64LE(((uint64_t*)a)[0]); + a1 = SWAP64LE(((uint64_t*)a)[1]); + b0 = SWAP64LE(((uint64_t*)b)[0]); + b1 = SWAP64LE(((uint64_t*)b)[1]); + a0 += b0; + a1 += b1; + ((uint64_t*)a)[0] = SWAP64LE(a0); + ((uint64_t*)a)[1] = SWAP64LE(a1); +} +#define U64(x) ((uint64_t *) (x)) + +static void copy_block(uint8_t* dst, const uint8_t* src) { + memcpy(dst, src, AES_BLOCK_SIZE); +} + +static void swap_blocks(uint8_t *a, uint8_t *b){ + uint64_t t[2]; + U64(t)[0] = U64(a)[0]; + U64(t)[1] = U64(a)[1]; + U64(a)[0] = U64(b)[0]; + U64(a)[1] = U64(b)[1]; + U64(b)[0] = U64(t)[0]; + U64(b)[1] = U64(t)[1]; +} + +static void xor_blocks(uint8_t* a, const uint8_t* b) { + size_t i; + for (i = 0; i < AES_BLOCK_SIZE; i++) { + a[i] ^= b[i]; + } +} + +#pragma pack(push, 1) +union cn_slow_hash_state { + union hash_state hs; + struct { + uint8_t k[64]; + uint8_t init[INIT_SIZE_BYTE]; + }; +}; +#pragma pack(pop) + +void cn_slow_hash(const void *data, size_t length, char *hash) { + uint8_t long_state[MEMORY]; + union cn_slow_hash_state state; + uint8_t text[INIT_SIZE_BYTE]; + uint8_t a[AES_BLOCK_SIZE]; + uint8_t b[AES_BLOCK_SIZE]; + uint8_t c[AES_BLOCK_SIZE]; + uint8_t d[AES_BLOCK_SIZE]; + size_t i, j; + uint8_t aes_key[AES_KEY_SIZE]; + oaes_ctx *aes_ctx; + + hash_process(&state.hs, data, length); + memcpy(text, state.init, INIT_SIZE_BYTE); + memcpy(aes_key, state.hs.b, AES_KEY_SIZE); + aes_ctx = (oaes_ctx *) oaes_alloc(); + + oaes_key_import_data(aes_ctx, aes_key, AES_KEY_SIZE); + for (i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) { + for (j = 0; j < INIT_SIZE_BLK; j++) { + aesb_pseudo_round(&text[AES_BLOCK_SIZE * j], &text[AES_BLOCK_SIZE * j], aes_ctx->key->exp_data); + } + memcpy(&long_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); + } + + for (i = 0; i < 16; i++) { + a[i] = state.k[ i] ^ state.k[32 + i]; + b[i] = state.k[16 + i] ^ state.k[48 + i]; + } + + for (i = 0; i < ITER / 2; i++) { + /* Dependency chain: address -> read value ------+ + * written value <-+ hard function (AES or MUL) <+ + * next address <-+ + */ + /* Iteration 1 */ + j = e2i(a, MEMORY / AES_BLOCK_SIZE); + copy_block(c, &long_state[j * AES_BLOCK_SIZE]); + aesb_single_round(c, c, a); + xor_blocks(b, c); + swap_blocks(b, c); + copy_block(&long_state[j * AES_BLOCK_SIZE], c); + assert(j == e2i(a, MEMORY / AES_BLOCK_SIZE)); + swap_blocks(a, b); + /* Iteration 2 */ + j = e2i(a, MEMORY / AES_BLOCK_SIZE); + copy_block(c, &long_state[j * AES_BLOCK_SIZE]); + mul(a, c, d); + sum_half_blocks(b, d); + swap_blocks(b, c); + xor_blocks(b, c); + copy_block(&long_state[j * AES_BLOCK_SIZE], c); + assert(j == e2i(a, MEMORY / AES_BLOCK_SIZE)); + swap_blocks(a, b); + } + + memcpy(text, state.init, INIT_SIZE_BYTE); + oaes_key_import_data(aes_ctx, &state.hs.b[32], AES_KEY_SIZE); + for (i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) { + for (j = 0; j < INIT_SIZE_BLK; j++) { + xor_blocks(&text[j * AES_BLOCK_SIZE], &long_state[i * INIT_SIZE_BYTE + j * AES_BLOCK_SIZE]); + aesb_pseudo_round(&text[AES_BLOCK_SIZE * j], &text[AES_BLOCK_SIZE * j], aes_ctx->key->exp_data); + } + } + memcpy(state.init, text, INIT_SIZE_BYTE); + hash_permutation(&state.hs); + /*memcpy(hash, &state, 32);*/ + extra_hashes[state.hs.b[0] & 3](&state, 200, hash); + oaes_free((OAES_CTX **) &aes_ctx); +} + +#endif diff --git a/src/crypto/tree-hash.c b/src/crypto/tree-hash.c index 5a84c8688..a75f91b80 100644 --- a/src/crypto/tree-hash.c +++ b/src/crypto/tree-hash.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index 7864b974e..e5df844e7 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -33,6 +33,8 @@ #include <string> #include <boost/uuid/uuid.hpp> +#define CRYPTONOTE_DNS_TIMEOUT_MS 20000 + #define CRYPTONOTE_MAX_BLOCK_NUMBER 500000000 #define CRYPTONOTE_MAX_BLOCK_SIZE 500000000 // block header blob limit, never used! #define CRYPTONOTE_GETBLOCKTEMPLATE_MAX_BLOCK_SIZE 196608 //size of block (bytes) that is the maximum that miners will produce @@ -49,6 +51,7 @@ // MONEY_SUPPLY - total number coins to be generated #define MONEY_SUPPLY ((uint64_t)(-1)) #define EMISSION_SPEED_FACTOR (20) +#define FINAL_SUBSIDY_PER_MINUTE ((uint64_t)300000000000) // 3 * pow(10, 11) #define CRYPTONOTE_REWARD_BLOCKS_WINDOW 100 #define CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE 20000 //size of block (bytes) after which reward for block calculated using block size @@ -59,9 +62,6 @@ #define FEE_PER_KB ((uint64_t)10000000000) // pow(10, 10) -// temporarily to allow backward compatibility during the switch to per-kb -//#define MINING_ALLOWED_LEGACY_FEE ((uint64_t)100000000000) // pow(10, 11) - #define ORPHANED_BLOCKS_MAX_COUNT 100 diff --git a/src/cryptonote_core/CMakeLists.txt b/src/cryptonote_core/CMakeLists.txt index 3c2e097c1..e144a93a2 100644 --- a/src/cryptonote_core/CMakeLists.txt +++ b/src/cryptonote_core/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # @@ -29,6 +29,7 @@ set(cryptonote_core_sources account.cpp blockchain_storage.cpp + blockchain.cpp checkpoints.cpp checkpoints_create.cpp cryptonote_basic_impl.cpp @@ -45,6 +46,7 @@ set(cryptonote_core_private_headers account_boost_serialization.h blockchain_storage.h blockchain_storage_boost_serialization.h + blockchain.h checkpoints.h checkpoints_create.h connection_context.h @@ -70,6 +72,8 @@ target_link_libraries(cryptonote_core LINK_PUBLIC common crypto + otshell_utils + blockchain_db ${Boost_DATE_TIME_LIBRARY} ${Boost_PROGRAM_OPTIONS_LIBRARY} ${Boost_SERIALIZATION_LIBRARY} diff --git a/src/cryptonote_core/account.cpp b/src/cryptonote_core/account.cpp index 9f6c91026..a4de8fb25 100644 --- a/src/cryptonote_core/account.cpp +++ b/src/cryptonote_core/account.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/account.h b/src/cryptonote_core/account.h index dd6618542..46f14fcbb 100644 --- a/src/cryptonote_core/account.h +++ b/src/cryptonote_core/account.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/account_boost_serialization.h b/src/cryptonote_core/account_boost_serialization.h index f9ecea895..cddc05e36 100644 --- a/src/cryptonote_core/account_boost_serialization.h +++ b/src/cryptonote_core/account_boost_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp new file mode 100644 index 000000000..0261a5614 --- /dev/null +++ b/src/cryptonote_core/blockchain.cpp @@ -0,0 +1,2342 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#include <algorithm> +#include <cstdio> +#include <boost/archive/binary_oarchive.hpp> +#include <boost/archive/binary_iarchive.hpp> +#include <boost/filesystem.hpp> + +#include "include_base_utils.h" +#include "cryptonote_basic_impl.h" +#include "tx_pool.h" +#include "blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "cryptonote_format_utils.h" +#include "cryptonote_boost_serialization.h" +#include "cryptonote_config.h" +#include "miner.h" +#include "misc_language.h" +#include "profile_tools.h" +#include "file_io_utils.h" +#include "common/boost_serialization_helper.h" +#include "warnings.h" +#include "crypto/hash.h" +#include "cryptonote_core/checkpoints_create.h" +//#include "serialization/json_archive.h" + +/* TODO: + * Clean up code: + * Possibly change how outputs are referred to/indexed in blockchain and wallets + * + */ + +using namespace cryptonote; +using epee::string_tools::pod_to_hex; + +DISABLE_VS_WARNINGS(4267) + +//------------------------------------------------------------------ +Blockchain::Blockchain(tx_memory_pool& tx_pool):m_db(), m_tx_pool(tx_pool), m_current_block_cumul_sz_limit(0), m_is_in_checkpoint_zone(false), m_is_blockchain_storing(false), m_enforce_dns_checkpoints(false) +{ + LOG_PRINT_L3("Blockchain::" << __func__); +} +//------------------------------------------------------------------ +//TODO: is this still needed? I don't think so - tewinget +template<class archive_t> +void Blockchain::serialize(archive_t & ar, const unsigned int version) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + if(version < 11) + return; + CRITICAL_REGION_LOCAL(m_blockchain_lock); + ar & m_blocks; + ar & m_blocks_index; + ar & m_transactions; + ar & m_spent_keys; + ar & m_alternative_chains; + ar & m_outputs; + ar & m_invalid_blocks; + ar & m_current_block_cumul_sz_limit; + /*serialization bug workaround*/ + if(version > 11) + { + uint64_t total_check_count = m_db->height() + m_blocks_index.size() + m_transactions.size() + m_spent_keys.size() + m_alternative_chains.size() + m_outputs.size() + m_invalid_blocks.size() + m_current_block_cumul_sz_limit; + if(archive_t::is_saving::value) + { + ar & total_check_count; + }else + { + uint64_t total_check_count_loaded = 0; + ar & total_check_count_loaded; + if(total_check_count != total_check_count_loaded) + { + LOG_ERROR("Blockchain storage data corruption detected. total_count loaded from file = " << total_check_count_loaded << ", expected = " << total_check_count); + + LOG_PRINT_L0("Blockchain storage:" << std::endl << + "m_blocks: " << m_db->height() << std::endl << + "m_blocks_index: " << m_blocks_index.size() << std::endl << + "m_transactions: " << m_transactions.size() << std::endl << + "m_spent_keys: " << m_spent_keys.size() << std::endl << + "m_alternative_chains: " << m_alternative_chains.size() << std::endl << + "m_outputs: " << m_outputs.size() << std::endl << + "m_invalid_blocks: " << m_invalid_blocks.size() << std::endl << + "m_current_block_cumul_sz_limit: " << m_current_block_cumul_sz_limit); + + throw std::runtime_error("Blockchain data corruption"); + } + } + } + + + LOG_PRINT_L3("Blockchain storage:" << std::endl << + "m_blocks: " << m_db->height() << std::endl << + "m_blocks_index: " << m_blocks_index.size() << std::endl << + "m_transactions: " << m_transactions.size() << std::endl << + "m_spent_keys: " << m_spent_keys.size() << std::endl << + "m_alternative_chains: " << m_alternative_chains.size() << std::endl << + "m_outputs: " << m_outputs.size() << std::endl << + "m_invalid_blocks: " << m_invalid_blocks.size() << std::endl << + "m_current_block_cumul_sz_limit: " << m_current_block_cumul_sz_limit); +} +//------------------------------------------------------------------ +bool Blockchain::have_tx(const crypto::hash &id) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + return m_db->tx_exists(id); +} +//------------------------------------------------------------------ +bool Blockchain::have_tx_keyimg_as_spent(const crypto::key_image &key_im) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + return m_db->has_key_image(key_im); +} +//------------------------------------------------------------------ +// This function makes sure that each "input" in an input (mixins) exists +// and collects the public key for each from the transaction it was included in +// via the visitor passed to it. +template<class visitor_t> +bool Blockchain::scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, visitor_t& vis, uint64_t* pmax_related_block_height) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + // verify that the input has key offsets (that it exists properly, really) + if(!tx_in_to_key.key_offsets.size()) + return false; + + // cryptonote_format_utils uses relative offsets for indexing to the global + // outputs list. that is to say that absolute offset #2 is absolute offset + // #1 plus relative offset #2. + // TODO: Investigate if this is necessary / why this is done. + std::vector<uint64_t> absolute_offsets = relative_output_offsets_to_absolute(tx_in_to_key.key_offsets); + + + //std::vector<std::pair<crypto::hash, size_t> >& amount_outs_vec = it->second; + size_t count = 0; + for (const uint64_t& i : absolute_offsets) + { + try + { + // get tx hash and output index for output + auto output_index = m_db->get_output_tx_and_index(tx_in_to_key.amount, i); + + // get tx that output is from + auto tx = m_db->get_tx(output_index.first); + + // make sure output index is within range for the given transaction + if (output_index.second >= tx.vout.size()) + { + LOG_PRINT_L0("Output does not exist. tx = " << output_index.first << ", index = " << output_index.second); + return false; + } + + // call to the passed boost visitor to grab the public key for the output + if(!vis.handle_output(tx, tx.vout[output_index.second])) + { + LOG_PRINT_L0("Failed to handle_output for output no = " << count << ", with absolute offset " << i); + return false; + } + + // if on last output and pmax_related_block_height not null pointer + if(++count == absolute_offsets.size() && pmax_related_block_height) + { + // set *pmax_related_block_height to tx block height for this output + auto h = m_db->get_tx_block_height(output_index.first); + if(*pmax_related_block_height < h) + { + *pmax_related_block_height = h; + } + } + + } + catch (const OUTPUT_DNE& e) + { + LOG_PRINT_L0("Output does not exist: " << e.what()); + return false; + } + catch (const TX_DNE& e) + { + LOG_PRINT_L0("Transaction does not exist: " << e.what()); + return false; + } + + } + + return true; +} +//------------------------------------------------------------------ +uint64_t Blockchain::get_current_blockchain_height() const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + return m_db->height(); +} +//------------------------------------------------------------------ +//FIXME: possibly move this into the constructor, to avoid accidentally +// dereferencing a null BlockchainDB pointer +bool Blockchain::init(BlockchainDB* db, const bool testnet) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + if (db == nullptr) + { + LOG_ERROR("Attempted to init Blockchain with null DB"); + return false; + } + if (!db->is_open()) + { + LOG_ERROR("Attempted to init Blockchain with unopened DB"); + return false; + } + + m_db = db; + + // if the blockchain is new, add the genesis block + // this feels kinda kludgy to do it this way, but can be looked at later. + // TODO: add function to create and store genesis block, + // taking testnet into account + if(!m_db->height()) + { + LOG_PRINT_L0("Blockchain not loaded, generating genesis block."); + block bl = boost::value_initialized<block>(); + block_verification_context bvc = boost::value_initialized<block_verification_context>(); + if (testnet) + { + generate_genesis_block(bl, config::testnet::GENESIS_TX, config::testnet::GENESIS_NONCE); + } + else + { + generate_genesis_block(bl, config::GENESIS_TX, config::GENESIS_NONCE); + } + add_new_block(bl, bvc); + CHECK_AND_ASSERT_MES(!bvc.m_verifivation_failed, false, "Failed to add genesis block to blockchain"); + } + // TODO: if blockchain load successful, verify blockchain against both + // hard-coded and runtime-loaded (and enforced) checkpoints. + else + { + } + + // check how far behind we are + uint64_t top_block_timestamp = m_db->get_top_block_timestamp(); + uint64_t timestamp_diff = time(NULL) - top_block_timestamp; + + // genesis block has no timestamp, could probably change it to have timestamp of 1341378000... + if(!top_block_timestamp) + timestamp_diff = time(NULL) - 1341378000; + LOG_PRINT_GREEN("Blockchain initialized. last block: " << m_db->height() - 1 << ", " << epee::misc_utils::get_time_interval_string(timestamp_diff) << " time ago, current difficulty: " << get_difficulty_for_next_block(), LOG_LEVEL_0); + + return true; +} +//------------------------------------------------------------------ +bool Blockchain::store_blockchain() +{ + LOG_PRINT_L3("Blockchain::" << __func__); + // TODO: make sure if this throws that it is not simply ignored higher + // up the call stack + try + { + m_db->sync(); + } + catch (const std::exception& e) + { + LOG_PRINT_L0(std::string("Error syncing blockchain db: ") + e.what() + "-- shutting down now to prevent issues!"); + throw; + } + catch (...) + { + LOG_PRINT_L0("There was an issue storing the blockchain, shutting down now to prevent issues!"); + throw; + } + LOG_PRINT_L0("Blockchain stored OK."); + return true; +} +//------------------------------------------------------------------ +bool Blockchain::deinit() +{ + LOG_PRINT_L3("Blockchain::" << __func__); + // as this should be called if handling a SIGSEGV, need to check + // if m_db is a NULL pointer (and thus may have caused the illegal + // memory operation), otherwise we may cause a loop. + if (m_db == NULL) + { + throw new DB_ERROR("The db pointer is null in Blockchain, the blockchain may be corrupt!"); + } + + try + { + m_db->close(); + } + catch (const std::exception& e) + { + LOG_PRINT_L0(std::string("Error closing blockchain db: ") + e.what()); + } + catch (...) + { + LOG_PRINT_L0("There was an issue closing/storing the blockchain, shutting down now to prevent issues!"); + } + + delete m_db; + return true; +} +//------------------------------------------------------------------ +// This function tells BlockchainDB to remove the top block from the +// blockchain and then returns all transactions (except the miner tx, of course) +// from it to the tx_pool +block Blockchain::pop_block_from_blockchain() +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + block popped_block; + std::vector<transaction> popped_txs; + + try + { + m_db->pop_block(popped_block, popped_txs); + } + // anything that could cause this to throw is likely catastrophic, + // so we re-throw + catch (const std::exception& e) + { + LOG_ERROR("Error popping block from blockchain: " << e.what()); + throw; + } + catch (...) + { + LOG_ERROR("Error popping block from blockchain, throwing!"); + throw; + } + + // return transactions from popped block to the tx_pool + for (transaction& tx : popped_txs) + { + if (!is_coinbase(tx)) + { + cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); + bool r = m_tx_pool.add_tx(tx, tvc, true); + if (!r) + { + LOG_ERROR("Error returning transaction to tx_pool"); + } + } + } + + return popped_block; +} +//------------------------------------------------------------------ +bool Blockchain::reset_and_set_genesis_block(const block& b) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + m_transactions.clear(); + m_spent_keys.clear(); + m_blocks.clear(); + m_blocks_index.clear(); + m_alternative_chains.clear(); + m_outputs.clear(); + m_db->reset(); + + block_verification_context bvc = boost::value_initialized<block_verification_context>(); + add_new_block(b, bvc); + return bvc.m_added_to_main_chain && !bvc.m_verifivation_failed; +} +//------------------------------------------------------------------ +//TODO: move to BlockchainDB subclass +bool Blockchain::purge_transaction_keyimages_from_blockchain(const transaction& tx, bool strict_check) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + struct purge_transaction_visitor: public boost::static_visitor<bool> + { + key_images_container& m_spent_keys; + bool m_strict_check; + purge_transaction_visitor(key_images_container& spent_keys, bool strict_check):m_spent_keys(spent_keys), m_strict_check(strict_check){} + + bool operator()(const txin_to_key& inp) const + { + //const crypto::key_image& ki = inp.k_image; + auto r = m_spent_keys.find(inp.k_image); + if(r != m_spent_keys.end()) + { + m_spent_keys.erase(r); + }else + { + CHECK_AND_ASSERT_MES(!m_strict_check, false, "purge_block_data_from_blockchain: key image in transaction not found"); + } + return true; + } + bool operator()(const txin_gen& inp) const + { + return true; + } + bool operator()(const txin_to_script& tx) const + { + return false; + } + + bool operator()(const txin_to_scripthash& tx) const + { + return false; + } + }; + + BOOST_FOREACH(const txin_v& in, tx.vin) + { + bool r = boost::apply_visitor(purge_transaction_visitor(m_spent_keys, strict_check), in); + CHECK_AND_ASSERT_MES(!strict_check || r, false, "failed to process purge_transaction_visitor"); + } + return true; +} +//------------------------------------------------------------------ +crypto::hash Blockchain::get_tail_id(uint64_t& height) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + height = m_db->height() - 1; + return get_tail_id(); +} +//------------------------------------------------------------------ +crypto::hash Blockchain::get_tail_id() const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + return m_db->top_block_hash(); +} +//------------------------------------------------------------------ +/*TODO: this function was...poorly written. As such, I'm not entirely + * certain on what it was supposed to be doing. Need to look into this, + * but it doesn't seem terribly important just yet. + * + * puts into list <ids> a list of hashes representing certain blocks + * from the blockchain in reverse chronological order + * + * the blocks chosen, at the time of this writing, are: + * the most recent 11 + * powers of 2 less recent from there, so 13, 17, 25, etc... + * + */ +bool Blockchain::get_short_chain_history(std::list<crypto::hash>& ids) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + uint64_t i = 0; + uint64_t current_multiplier = 1; + uint64_t sz = m_db->height(); + + if(!sz) + return true; + + bool genesis_included = false; + uint64_t current_back_offset = 1; + while(current_back_offset < sz) + { + ids.push_back(m_db->get_block_hash_from_height(sz - current_back_offset)); + + if(sz-current_back_offset == 0) + { + genesis_included = true; + } + if(i < 10) + { + ++current_back_offset; + } + else + { + current_multiplier *= 2; + current_back_offset += current_multiplier; + } + ++i; + } + + if (!genesis_included) + { + ids.push_back(m_db->get_block_hash_from_height(0)); + } + + return true; +} +//------------------------------------------------------------------ +crypto::hash Blockchain::get_block_id_by_height(uint64_t height) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + try + { + return m_db->get_block_hash_from_height(height); + } + catch (const BLOCK_DNE& e) + { + } + catch (const std::exception& e) + { + LOG_PRINT_L0(std::string("Something went wrong fetching block hash by height: ") + e.what()); + throw; + } + catch (...) + { + LOG_PRINT_L0(std::string("Something went wrong fetching block hash by height")); + throw; + } + return null_hash; +} +//------------------------------------------------------------------ +bool Blockchain::get_block_by_hash(const crypto::hash &h, block &blk) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + // try to find block in main chain + try + { + blk = m_db->get_block(h); + return true; + } + // try to find block in alternative chain + catch (const BLOCK_DNE& e) + { + blocks_ext_by_hash::const_iterator it_alt = m_alternative_chains.find(h); + if (m_alternative_chains.end() != it_alt) { + blk = it_alt->second.bl; + return true; + } + } + catch (const std::exception& e) + { + LOG_PRINT_L0(std::string("Something went wrong fetching block by hash: ") + e.what()); + throw; + } + catch (...) + { + LOG_PRINT_L0(std::string("Something went wrong fetching block hash by hash")); + throw; + } + + return false; +} +//------------------------------------------------------------------ +//FIXME: this function does not seem to be called from anywhere, but +// if it ever is, should probably change std::list for std::vector +void Blockchain::get_all_known_block_ids(std::list<crypto::hash> &main, std::list<crypto::hash> &alt, std::list<crypto::hash> &invalid) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + for (auto& a : m_db->get_hashes_range(0, m_db->height() - 1)) + { + main.push_back(a); + } + + BOOST_FOREACH(const blocks_ext_by_hash::value_type &v, m_alternative_chains) + alt.push_back(v.first); + + BOOST_FOREACH(const blocks_ext_by_hash::value_type &v, m_invalid_blocks) + invalid.push_back(v.first); +} +//------------------------------------------------------------------ +// This function aggregates the cumulative difficulties and timestamps of the +// last DIFFICULTY_BLOCKS_COUNT blocks and passes them to next_difficulty, +// returning the result of that call. Ignores the genesis block, and can use +// less blocks than desired if there aren't enough. +difficulty_type Blockchain::get_difficulty_for_next_block() const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + std::vector<uint64_t> timestamps; + std::vector<difficulty_type> cumulative_difficulties; + auto h = m_db->height(); + + size_t offset = h - std::min<size_t>(h, static_cast<size_t>(DIFFICULTY_BLOCKS_COUNT)); + + if (offset == 0) + { + ++offset; + } + + for(; offset < h; offset++) + { + timestamps.push_back(m_db->get_block_timestamp(offset)); + cumulative_difficulties.push_back(m_db->get_block_cumulative_difficulty(offset)); + } + return next_difficulty(timestamps, cumulative_difficulties); +} +//------------------------------------------------------------------ +// This function removes blocks from the blockchain until it gets to the +// position where the blockchain switch started and then re-adds the blocks +// that had been removed. +bool Blockchain::rollback_blockchain_switching(std::list<block>& original_chain, uint64_t rollback_height) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + // remove blocks from blockchain until we get back to where we should be. + while (m_db->height() != rollback_height) + { + pop_block_from_blockchain(); + } + + //return back original chain + for (auto& bl : original_chain) + { + block_verification_context bvc = boost::value_initialized<block_verification_context>(); + bool r = handle_block_to_main_chain(bl, bvc); + CHECK_AND_ASSERT_MES(r && bvc.m_added_to_main_chain, false, "PANIC! failed to add (again) block while chain switching during the rollback!"); + } + + LOG_PRINT_L1("Rollback to height " << rollback_height << " was successful."); + if (original_chain.size()) + { + LOG_PRINT_L1("Restoration to previous blockchain successful as well."); + } + return true; +} +//------------------------------------------------------------------ +// This function attempts to switch to an alternate chain, returning +// boolean based on success therein. +bool Blockchain::switch_to_alternative_blockchain(std::list<blocks_ext_by_hash::iterator>& alt_chain, bool discard_disconnected_chain) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + // if empty alt chain passed (not sure how that could happen), return false + CHECK_AND_ASSERT_MES(alt_chain.size(), false, "switch_to_alternative_blockchain: empty chain passed"); + + // verify that main chain has front of alt chain's parent block + if (!m_db->block_exists(alt_chain.front()->second.bl.prev_id)) + { + LOG_ERROR("Attempting to move to an alternate chain, but it doesn't appear to connect to the main chain!"); + return false; + } + + // pop blocks from the blockchain until the top block is the parent + // of the front block of the alt chain. + std::list<block> disconnected_chain; + while (m_db->top_block_hash() != alt_chain.front()->second.bl.prev_id) + { + block b = pop_block_from_blockchain(); + disconnected_chain.push_front(b); + } + + auto split_height = m_db->height(); + + //connecting new alternative chain + for(auto alt_ch_iter = alt_chain.begin(); alt_ch_iter != alt_chain.end(); alt_ch_iter++) + { + auto ch_ent = *alt_ch_iter; + block_verification_context bvc = boost::value_initialized<block_verification_context>(); + + // add block to main chain + bool r = handle_block_to_main_chain(ch_ent->second.bl, bvc); + + // if adding block to main chain failed, rollback to previous state and + // return false + if(!r || !bvc.m_added_to_main_chain) + { + LOG_PRINT_L1("Failed to switch to alternative blockchain"); + + // rollback_blockchain_switching should be moved to two different + // functions: rollback and apply_chain, but for now we pretend it is + // just the latter (because the rollback was done above). + rollback_blockchain_switching(disconnected_chain, m_db->height()); + + // FIXME: Why do we keep invalid blocks around? Possibly in case we hear + // about them again so we can immediately dismiss them, but needs some + // looking into. + add_block_as_invalid(ch_ent->second, get_block_hash(ch_ent->second.bl)); + LOG_PRINT_L1("The block was inserted as invalid while connecting new alternative chain, block_id: " << get_block_hash(ch_ent->second.bl)); + m_alternative_chains.erase(ch_ent); + + for(auto alt_ch_to_orph_iter = ++alt_ch_iter; alt_ch_to_orph_iter != alt_chain.end(); alt_ch_to_orph_iter++) + { + add_block_as_invalid((*alt_ch_iter)->second, (*alt_ch_iter)->first); + m_alternative_chains.erase(*alt_ch_to_orph_iter); + } + return false; + } + } + + // if we're to keep the disconnected blocks, add them as alternates + if(!discard_disconnected_chain) + { + //pushing old chain as alternative chain + for (auto& old_ch_ent : disconnected_chain) + { + block_verification_context bvc = boost::value_initialized<block_verification_context>(); + bool r = handle_alternative_block(old_ch_ent, get_block_hash(old_ch_ent), bvc); + if(!r) + { + LOG_PRINT_L1("Failed to push ex-main chain blocks to alternative chain "); + // previously this would fail the blockchain switching, but I don't + // think this is bad enough to warrant that. + } + } + } + + //removing alt_chain entries from alternative chain + BOOST_FOREACH(auto ch_ent, alt_chain) + { + m_alternative_chains.erase(ch_ent); + } + + LOG_PRINT_GREEN("REORGANIZE SUCCESS! on height: " << split_height << ", new blockchain size: " << m_db->height(), LOG_LEVEL_0); + return true; +} +//------------------------------------------------------------------ +// This function calculates the difficulty target for the block being added to +// an alternate chain. +difficulty_type Blockchain::get_next_difficulty_for_alternative_chain(const std::list<blocks_ext_by_hash::iterator>& alt_chain, block_extended_info& bei) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + std::vector<uint64_t> timestamps; + std::vector<difficulty_type> cumulative_difficulties; + + // if the alt chain isn't long enough to calculate the difficulty target + // based on its blocks alone, need to get more blocks from the main chain + if(alt_chain.size()< DIFFICULTY_BLOCKS_COUNT) + { + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + // Figure out start and stop offsets for main chain blocks + size_t main_chain_stop_offset = alt_chain.size() ? alt_chain.front()->second.height : bei.height; + size_t main_chain_count = DIFFICULTY_BLOCKS_COUNT - std::min(static_cast<size_t>(DIFFICULTY_BLOCKS_COUNT), alt_chain.size()); + main_chain_count = std::min(main_chain_count, main_chain_stop_offset); + size_t main_chain_start_offset = main_chain_stop_offset - main_chain_count; + + if(!main_chain_start_offset) + ++main_chain_start_offset; //skip genesis block + + // get difficulties and timestamps from relevant main chain blocks + for(; main_chain_start_offset < main_chain_stop_offset; ++main_chain_start_offset) + { + timestamps.push_back(m_db->get_block_timestamp(main_chain_start_offset)); + cumulative_difficulties.push_back(m_db->get_block_cumulative_difficulty(main_chain_start_offset)); + } + + // make sure we haven't accidentally grabbed too many blocks...maybe don't need this check? + CHECK_AND_ASSERT_MES((alt_chain.size() + timestamps.size()) <= DIFFICULTY_BLOCKS_COUNT, false, + "Internal error, alt_chain.size()[" << alt_chain.size() + << "] + vtimestampsec.size()[" << timestamps.size() + << "] NOT <= DIFFICULTY_WINDOW[]" << DIFFICULTY_BLOCKS_COUNT + ); + + for (auto it : alt_chain) + { + timestamps.push_back(it->second.bl.timestamp); + cumulative_difficulties.push_back(it->second.cumulative_difficulty); + } + } + // if the alt chain is long enough for the difficulty calc, grab difficulties + // and timestamps from it alone + else + { + timestamps.resize(static_cast<size_t>(DIFFICULTY_BLOCKS_COUNT)); + cumulative_difficulties.resize(static_cast<size_t>(DIFFICULTY_BLOCKS_COUNT)); + size_t count = 0; + size_t max_i = timestamps.size()-1; + // get difficulties and timestamps from most recent blocks in alt chain + BOOST_REVERSE_FOREACH(auto it, alt_chain) + { + timestamps[max_i - count] = it->second.bl.timestamp; + cumulative_difficulties[max_i - count] = it->second.cumulative_difficulty; + count++; + if(count >= DIFFICULTY_BLOCKS_COUNT) + break; + } + } + + // calculate the difficulty target for the block and return it + return next_difficulty(timestamps, cumulative_difficulties); +} +//------------------------------------------------------------------ +// This function does a sanity check on basic things that all miner +// transactions have in common, such as: +// one input, of type txin_gen, with height set to the block's height +// correct miner tx unlock time +// a non-overflowing tx amount (dubious necessity on this check) +bool Blockchain::prevalidate_miner_transaction(const block& b, uint64_t height) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CHECK_AND_ASSERT_MES(b.miner_tx.vin.size() == 1, false, "coinbase transaction in the block has no inputs"); + CHECK_AND_ASSERT_MES(b.miner_tx.vin[0].type() == typeid(txin_gen), false, "coinbase transaction in the block has the wrong type"); + if(boost::get<txin_gen>(b.miner_tx.vin[0]).height != height) + { + LOG_PRINT_RED_L1("The miner transaction in block has invalid height: " << boost::get<txin_gen>(b.miner_tx.vin[0]).height << ", expected: " << height); + return false; + } + CHECK_AND_ASSERT_MES(b.miner_tx.unlock_time == height + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW, + false, + "coinbase transaction transaction has the wrong unlock time=" << b.miner_tx.unlock_time << ", expected " << height + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW); + + + //check outs overflow + //NOTE: not entirely sure this is necessary, given that this function is + // designed simply to make sure the total amount for a transaction + // does not overflow a uint64_t, and this transaction *is* a uint64_t... + if(!check_outs_overflow(b.miner_tx)) + { + LOG_PRINT_RED_L1("miner transaction has money overflow in block " << get_block_hash(b)); + return false; + } + + return true; +} +//------------------------------------------------------------------ +// This function validates the miner transaction reward +bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + //validate reward + uint64_t money_in_use = 0; + BOOST_FOREACH(auto& o, b.miner_tx.vout) + money_in_use += o.amount; + + std::vector<size_t> last_blocks_sizes; + get_last_n_blocks_sizes(last_blocks_sizes, CRYPTONOTE_REWARD_BLOCKS_WINDOW); + if (!get_block_reward(epee::misc_utils::median(last_blocks_sizes), cumulative_block_size, already_generated_coins, base_reward)) { + LOG_PRINT_L1("block size " << cumulative_block_size << " is bigger than allowed for this blockchain"); + return false; + } + if(base_reward + fee < money_in_use) + { + LOG_PRINT_L1("coinbase transaction spend too much money (" << print_money(money_in_use) << "). Block reward is " << print_money(base_reward + fee) << "(" << print_money(base_reward) << "+" << print_money(fee) << ")"); + return false; + } + if(base_reward + fee != money_in_use) + { + LOG_PRINT_L1("coinbase transaction doesn't use full amount of block reward: spent: " + << money_in_use << ", block reward " << base_reward + fee << "(" << base_reward << "+" << fee << ")"); + return false; + } + return true; +} +//------------------------------------------------------------------ +// get the block sizes of the last <count> blocks, starting at <from_height> +// and return by reference <sz>. +void Blockchain::get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t count) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + auto h = m_db->height(); + + // this function is meaningless for an empty blockchain...granted it should never be empty + if(h == 0) + return; + + // add size of last <count> blocks to vector <sz> (or less, if blockchain size < count) + size_t start_offset = h - std::min<size_t>(h, count); + for(size_t i = start_offset; i < h; i++) + { + sz.push_back(m_db->get_block_size(i)); + } +} +//------------------------------------------------------------------ +uint64_t Blockchain::get_current_cumulative_blocksize_limit() const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + return m_current_block_cumul_sz_limit; +} +//------------------------------------------------------------------ +//TODO: This function only needed minor modification to work with BlockchainDB, +// and *works*. As such, to reduce the number of things that might break +// in moving to BlockchainDB, this function will remain otherwise +// unchanged for the time being. +// +// This function makes a new block for a miner to mine the hash for +// +// FIXME: this codebase references #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) +// in a lot of places. That flag is not referenced in any of the code +// nor any of the makefiles, howeve. Need to look into whether or not it's +// necessary at all. +bool Blockchain::create_block_template(block& b, const account_public_address& miner_address, difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + size_t median_size; + uint64_t already_generated_coins; + + CRITICAL_REGION_BEGIN(m_blockchain_lock); + b.major_version = CURRENT_BLOCK_MAJOR_VERSION; + b.minor_version = CURRENT_BLOCK_MINOR_VERSION; + b.prev_id = get_tail_id(); + b.timestamp = time(NULL); + + height = m_db->height(); + diffic = get_difficulty_for_next_block(); + CHECK_AND_ASSERT_MES(diffic, false, "difficulty owverhead."); + + median_size = m_current_block_cumul_sz_limit / 2; + already_generated_coins = m_db->get_block_already_generated_coins(height - 1); + + CRITICAL_REGION_END(); + + size_t txs_size; + uint64_t fee; + if (!m_tx_pool.fill_block_template(b, median_size, already_generated_coins, txs_size, fee)) { + return false; + } +#if defined(DEBUG_CREATE_BLOCK_TEMPLATE) + size_t real_txs_size = 0; + uint64_t real_fee = 0; + CRITICAL_REGION_BEGIN(m_tx_pool.m_transactions_lock); + BOOST_FOREACH(crypto::hash &cur_hash, b.tx_hashes) { + auto cur_res = m_tx_pool.m_transactions.find(cur_hash); + if (cur_res == m_tx_pool.m_transactions.end()) { + LOG_ERROR("Creating block template: error: transaction not found"); + continue; + } + tx_memory_pool::tx_details &cur_tx = cur_res->second; + real_txs_size += cur_tx.blob_size; + real_fee += cur_tx.fee; + if (cur_tx.blob_size != get_object_blobsize(cur_tx.tx)) { + LOG_ERROR("Creating block template: error: invalid transaction size"); + } + uint64_t inputs_amount; + if (!get_inputs_money_amount(cur_tx.tx, inputs_amount)) { + LOG_ERROR("Creating block template: error: cannot get inputs amount"); + } else if (cur_tx.fee != inputs_amount - get_outs_money_amount(cur_tx.tx)) { + LOG_ERROR("Creating block template: error: invalid fee"); + } + } + if (txs_size != real_txs_size) { + LOG_ERROR("Creating block template: error: wrongly calculated transaction size"); + } + if (fee != real_fee) { + LOG_ERROR("Creating block template: error: wrongly calculated fee"); + } + CRITICAL_REGION_END(); + LOG_PRINT_L1("Creating block template: height " << height << + ", median size " << median_size << + ", already generated coins " << already_generated_coins << + ", transaction size " << txs_size << + ", fee " << fee); +#endif + + /* + two-phase miner transaction generation: we don't know exact block size until we prepare block, but we don't know reward until we know + block size, so first miner transaction generated with fake amount of money, and with phase we know think we know expected block size + */ + //make blocks coin-base tx looks close to real coinbase tx to get truthful blob size + bool r = construct_miner_tx(height, median_size, already_generated_coins, txs_size, fee, miner_address, b.miner_tx, ex_nonce, 11); + CHECK_AND_ASSERT_MES(r, false, "Failed to construc miner tx, first chance"); + size_t cumulative_size = txs_size + get_object_blobsize(b.miner_tx); +#if defined(DEBUG_CREATE_BLOCK_TEMPLATE) + LOG_PRINT_L1("Creating block template: miner tx size " << get_object_blobsize(b.miner_tx) << + ", cumulative size " << cumulative_size); +#endif + for (size_t try_count = 0; try_count != 10; ++try_count) { + r = construct_miner_tx(height, median_size, already_generated_coins, cumulative_size, fee, miner_address, b.miner_tx, ex_nonce, 11); + + CHECK_AND_ASSERT_MES(r, false, "Failed to construc miner tx, second chance"); + size_t coinbase_blob_size = get_object_blobsize(b.miner_tx); + if (coinbase_blob_size > cumulative_size - txs_size) { + cumulative_size = txs_size + coinbase_blob_size; +#if defined(DEBUG_CREATE_BLOCK_TEMPLATE) + LOG_PRINT_L1("Creating block template: miner tx size " << coinbase_blob_size << + ", cumulative size " << cumulative_size << " is greater then before"); +#endif + continue; + } + + if (coinbase_blob_size < cumulative_size - txs_size) { + size_t delta = cumulative_size - txs_size - coinbase_blob_size; +#if defined(DEBUG_CREATE_BLOCK_TEMPLATE) + LOG_PRINT_L1("Creating block template: miner tx size " << coinbase_blob_size << + ", cumulative size " << txs_size + coinbase_blob_size << + " is less then before, adding " << delta << " zero bytes"); +#endif + b.miner_tx.extra.insert(b.miner_tx.extra.end(), delta, 0); + //here could be 1 byte difference, because of extra field counter is varint, and it can become from 1-byte len to 2-bytes len. + if (cumulative_size != txs_size + get_object_blobsize(b.miner_tx)) { + CHECK_AND_ASSERT_MES(cumulative_size + 1 == txs_size + get_object_blobsize(b.miner_tx), false, "unexpected case: cumulative_size=" << cumulative_size << " + 1 is not equal txs_cumulative_size=" << txs_size << " + get_object_blobsize(b.miner_tx)=" << get_object_blobsize(b.miner_tx)); + b.miner_tx.extra.resize(b.miner_tx.extra.size() - 1); + if (cumulative_size != txs_size + get_object_blobsize(b.miner_tx)) { + //fuck, not lucky, -1 makes varint-counter size smaller, in that case we continue to grow with cumulative_size + LOG_PRINT_RED("Miner tx creation has no luck with delta_extra size = " << delta << " and " << delta - 1 , LOG_LEVEL_2); + cumulative_size += delta - 1; + continue; + } + LOG_PRINT_GREEN("Setting extra for block: " << b.miner_tx.extra.size() << ", try_count=" << try_count, LOG_LEVEL_1); + } + } + CHECK_AND_ASSERT_MES(cumulative_size == txs_size + get_object_blobsize(b.miner_tx), false, "unexpected case: cumulative_size=" << cumulative_size << " is not equal txs_cumulative_size=" << txs_size << " + get_object_blobsize(b.miner_tx)=" << get_object_blobsize(b.miner_tx)); +#if defined(DEBUG_CREATE_BLOCK_TEMPLATE) + LOG_PRINT_L1("Creating block template: miner tx size " << coinbase_blob_size << + ", cumulative size " << cumulative_size << " is now good"); +#endif + return true; + } + LOG_ERROR("Failed to create_block_template with " << 10 << " tries"); + return false; +} +//------------------------------------------------------------------ +// for an alternate chain, get the timestamps from the main chain to complete +// the needed number of timestamps for the BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW. +bool Blockchain::complete_timestamps_vector(uint64_t start_top_height, std::vector<uint64_t>& timestamps) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + + if(timestamps.size() >= BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW) + return true; + + CRITICAL_REGION_LOCAL(m_blockchain_lock); + size_t need_elements = BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW - timestamps.size(); + CHECK_AND_ASSERT_MES(start_top_height < m_db->height(), false, "internal error: passed start_height not < " << " m_db->height() -- " << start_top_height << " >= " << m_db->height()); + size_t stop_offset = start_top_height > need_elements ? start_top_height - need_elements : 0; + while (start_top_height != stop_offset) + { + timestamps.push_back(m_db->get_block_timestamp(start_top_height)); + --start_top_height; + } + return true; +} +//------------------------------------------------------------------ +// If a block is to be added and its parent block is not the current +// main chain top block, then we need to see if we know about its parent block. +// If its parent block is part of a known forked chain, then we need to see +// if that chain is long enough to become the main chain and re-org accordingly +// if so. If not, we need to hang on to the block in case it becomes part of +// a long forked chain eventually. +bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id, block_verification_context& bvc) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + uint64_t block_height = get_block_height(b); + if(0 == block_height) + { + LOG_PRINT_L1("Block with id: " << epee::string_tools::pod_to_hex(id) << " (as alternative), but miner tx says height is 0."); + bvc.m_verifivation_failed = true; + return false; + } + // this basically says if the blockchain is smaller than the first + // checkpoint then alternate blocks are allowed. Alternatively, if the + // last checkpoint *before* the end of the current chain is also before + // the block to be added, then this is fine. + if (!m_checkpoints.is_alternative_block_allowed(get_current_blockchain_height(), block_height)) + { + LOG_PRINT_RED_L1("Block with id: " << id + << std::endl << " can't be accepted for alternative chain, block height: " << block_height + << std::endl << " blockchain height: " << get_current_blockchain_height()); + bvc.m_verifivation_failed = true; + return false; + } + + //block is not related with head of main chain + //first of all - look in alternative chains container + auto it_prev = m_alternative_chains.find(b.prev_id); + bool parent_in_main = m_db->block_exists(b.prev_id); + if(it_prev != m_alternative_chains.end() || parent_in_main) + { + //we have new block in alternative chain + + //build alternative subchain, front -> mainchain, back -> alternative head + blocks_ext_by_hash::iterator alt_it = it_prev; //m_alternative_chains.find() + std::list<blocks_ext_by_hash::iterator> alt_chain; + std::vector<uint64_t> timestamps; + while(alt_it != m_alternative_chains.end()) + { + alt_chain.push_front(alt_it); + timestamps.push_back(alt_it->second.bl.timestamp); + alt_it = m_alternative_chains.find(alt_it->second.bl.prev_id); + } + + // if block to be added connects to known blocks that aren't part of the + // main chain -- that is, if we're adding on to an alternate chain + if(alt_chain.size()) + { + // make sure alt chain doesn't somehow start past the end of the main chain + CHECK_AND_ASSERT_MES(m_db->height() > alt_chain.front()->second.height, false, "main blockchain wrong height"); + + // make sure that the blockchain contains the block that should connect + // this alternate chain with it. + if (!m_db->block_exists(alt_chain.front()->second.bl.prev_id)) + { + LOG_PRINT_L1("alternate chain does not appear to connect to main chain..."); + return false; + } + + // make sure block connects correctly to the main chain + auto h = m_db->get_block_hash_from_height(alt_chain.front()->second.height - 1); + CHECK_AND_ASSERT_MES(h == alt_chain.front()->second.bl.prev_id, false, "alternative chain has wrong connection to main chain"); + complete_timestamps_vector(m_db->get_block_height(alt_chain.front()->second.bl.prev_id), timestamps); + } + // if block not associated with known alternate chain + else + { + // if block parent is not part of main chain or an alternate chain, + // we ignore it + CHECK_AND_ASSERT_MES(parent_in_main, false, "internal error: broken imperative condition it_main_prev != m_blocks_index.end()"); + + complete_timestamps_vector(m_db->get_block_height(b.prev_id), timestamps); + } + + // verify that the block's timestamp is within the acceptable range + // (not earlier than the median of the last X blocks) + if(!check_block_timestamp(timestamps, b)) + { + LOG_PRINT_RED_L1("Block with id: " << id + << std::endl << " for alternative chain, has invalid timestamp: " << b.timestamp); + bvc.m_verifivation_failed = true; + return false; + } + + // FIXME: consider moving away from block_extended_info at some point + block_extended_info bei = boost::value_initialized<block_extended_info>(); + bei.bl = b; + bei.height = alt_chain.size() ? it_prev->second.height + 1 : m_db->get_block_height(b.prev_id) + 1; + + bool is_a_checkpoint; + if(!m_checkpoints.check_block(bei.height, id, is_a_checkpoint)) + { + LOG_ERROR("CHECKPOINT VALIDATION FAILED"); + bvc.m_verifivation_failed = true; + return false; + } + + // Check the block's hash against the difficulty target for its alt chain + m_is_in_checkpoint_zone = false; + difficulty_type current_diff = get_next_difficulty_for_alternative_chain(alt_chain, bei); + CHECK_AND_ASSERT_MES(current_diff, false, "!!!!!!! DIFFICULTY OVERHEAD !!!!!!!"); + crypto::hash proof_of_work = null_hash; + get_block_longhash(bei.bl, proof_of_work, bei.height); + if(!check_hash(proof_of_work, current_diff)) + { + LOG_PRINT_RED_L1("Block with id: " << id + << std::endl << " for alternative chain, does not have enough proof of work: " << proof_of_work + << std::endl << " expected difficulty: " << current_diff); + bvc.m_verifivation_failed = true; + return false; + } + + if(!prevalidate_miner_transaction(b, bei.height)) + { + LOG_PRINT_RED_L1("Block with id: " << epee::string_tools::pod_to_hex(id) + << " (as alternative) has incorrect miner transaction."); + bvc.m_verifivation_failed = true; + return false; + + } + + // FIXME: + // this brings up an interesting point: consider allowing to get block + // difficulty both by height OR by hash, not just height. + difficulty_type main_chain_cumulative_difficulty = m_db->get_block_cumulative_difficulty(m_db->height() - 1); + if (alt_chain.size()) + { + bei.cumulative_difficulty = it_prev->second.cumulative_difficulty; + } + else + { + // passed-in block's previous block's cumulative difficulty, found on the main chain + bei.cumulative_difficulty = m_db->get_block_cumulative_difficulty(m_db->get_block_height(b.prev_id)); + } + bei.cumulative_difficulty += current_diff; + + // add block to alternate blocks storage, + // as well as the current "alt chain" container + auto i_res = m_alternative_chains.insert(blocks_ext_by_hash::value_type(id, bei)); + CHECK_AND_ASSERT_MES(i_res.second, false, "insertion of new alternative block returned as it already exist"); + alt_chain.push_back(i_res.first); + + // FIXME: is it even possible for a checkpoint to show up not on the main chain? + if(is_a_checkpoint) + { + //do reorganize! + LOG_PRINT_GREEN("###### REORGANIZE on height: " << alt_chain.front()->second.height << " of " << m_db->height() - 1 << + ", checkpoint is found in alternative chain on height " << bei.height, LOG_LEVEL_0); + + bool r = switch_to_alternative_blockchain(alt_chain, true); + + bvc.m_added_to_main_chain = r; + bvc.m_verifivation_failed = !r; + + return r; + } + else if(main_chain_cumulative_difficulty < bei.cumulative_difficulty) //check if difficulty bigger then in main chain + { + //do reorganize! + LOG_PRINT_GREEN("###### REORGANIZE on height: " + << alt_chain.front()->second.height << " of " << m_db->height() - 1 + << " with cum_difficulty " << m_db->get_block_cumulative_difficulty(m_db->height() - 1) + << std::endl << " alternative blockchain size: " << alt_chain.size() + << " with cum_difficulty " << bei.cumulative_difficulty, LOG_LEVEL_0 + ); + + bool r = switch_to_alternative_blockchain(alt_chain, false); + if(r) bvc.m_added_to_main_chain = true; + else bvc.m_verifivation_failed = true; + return r; + } + else + { + LOG_PRINT_BLUE("----- BLOCK ADDED AS ALTERNATIVE ON HEIGHT " << bei.height + << std::endl << "id:\t" << id + << std::endl << "PoW:\t" << proof_of_work + << std::endl << "difficulty:\t" << current_diff, LOG_LEVEL_0); + return true; + } + } + else + { + //block orphaned + bvc.m_marked_as_orphaned = true; + LOG_PRINT_RED_L1("Block recognized as orphaned and rejected, id = " << id); + } + + return true; +} +//------------------------------------------------------------------ +bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks, std::list<transaction>& txs) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + if(start_offset > m_db->height()) + return false; + + if (!get_blocks(start_offset, count, blocks)) + { + return false; + } + + for(const block& blk : blocks) + { + std::list<crypto::hash> missed_ids; + get_transactions(blk.tx_hashes, txs, missed_ids); + CHECK_AND_ASSERT_MES(!missed_ids.size(), false, "has missed transactions in own block in main blockchain"); + } + + return true; +} +//------------------------------------------------------------------ +bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + if(start_offset > m_db->height()) + return false; + + for(size_t i = start_offset; i < start_offset + count && i <= m_db->height();i++) + { + blocks.push_back(m_db->get_block_from_height(i)); + } + return true; +} +//------------------------------------------------------------------ +//TODO: This function *looks* like it won't need to be rewritten +// to use BlockchainDB, as it calls other functions that were, +// but it warrants some looking into later. +bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NOTIFY_RESPONSE_GET_OBJECTS::request& rsp) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + rsp.current_blockchain_height = get_current_blockchain_height(); + std::list<block> blocks; + get_blocks(arg.blocks, blocks, rsp.missed_ids); + + BOOST_FOREACH(const auto& bl, blocks) + { + std::list<crypto::hash> missed_tx_id; + std::list<transaction> txs; + get_transactions(bl.tx_hashes, txs, rsp.missed_ids); + CHECK_AND_ASSERT_MES(!missed_tx_id.size(), false, "Internal error: has missed missed_tx_id.size()=" << missed_tx_id.size() + << std::endl << "for block id = " << get_block_hash(bl)); + rsp.blocks.push_back(block_complete_entry()); + block_complete_entry& e = rsp.blocks.back(); + //pack block + e.block = t_serializable_object_to_blob(bl); + //pack transactions + BOOST_FOREACH(transaction& tx, txs) + e.txs.push_back(t_serializable_object_to_blob(tx)); + + } + //get another transactions, if need + std::list<transaction> txs; + get_transactions(arg.txs, txs, rsp.missed_ids); + //pack aside transactions + BOOST_FOREACH(const auto& tx, txs) + rsp.txs.push_back(t_serializable_object_to_blob(tx)); + + return true; +} +//------------------------------------------------------------------ +bool Blockchain::get_alternative_blocks(std::list<block>& blocks) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + BOOST_FOREACH(const auto& alt_bl, m_alternative_chains) + { + blocks.push_back(alt_bl.second.bl); + } + return true; +} +//------------------------------------------------------------------ +size_t Blockchain::get_alternative_blocks_count() const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + return m_alternative_chains.size(); +} +//------------------------------------------------------------------ +// This function adds the output specified by <amount, i> to the result_outs container +// unlocked and other such checks should be done by here. +void Blockchain::add_out_to_get_random_outs(COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry& oen = *result_outs.outs.insert(result_outs.outs.end(), COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry()); + oen.global_amount_index = i; + oen.out_key = m_db->get_output_key(amount, i); +} +//------------------------------------------------------------------ +// This function takes an RPC request for mixins and creates an RPC response +// with the requested mixins. +// TODO: figure out why this returns boolean / if we should be returning false +// in some cases +bool Blockchain::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + srand(static_cast<unsigned int>(time(NULL))); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + // for each amount that we need to get mixins for, get <n> random outputs + // from BlockchainDB where <n> is req.outs_count (number of mixins). + for (uint64_t amount : req.amounts) + { + // create outs_for_amount struct and populate amount field + COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs = *res.outs.insert(res.outs.end(), COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount()); + result_outs.amount = amount; + + std::unordered_set<uint64_t> seen_indices; + + // if there aren't enough outputs to mix with (or just enough), + // use all of them. Eventually this should become impossible. + if (m_db->get_num_outputs(amount) <= req.outs_count) + { + for (uint64_t i = 0; i < m_db->get_num_outputs(amount); i++) + { + // get tx_hash, tx_out_index from DB + tx_out_index toi = m_db->get_output_tx_and_index(amount, i); + + // if tx is unlocked, add output to result_outs + if (is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first))) + { + add_out_to_get_random_outs(result_outs, amount, i); + } + + } + } + else + { + // while we still need more mixins + auto num_outs = m_db->get_num_outputs(amount); + while (result_outs.outs.size() < req.outs_count) + { + // if we've gone through every possible output, we've gotten all we can + if (seen_indices.size() == num_outs) + { + break; + } + + // get a random output index from the DB. If we've already seen it, + // return to the top of the loop and try again, otherwise add it to the + // list of output indices we've seen. + uint64_t i = m_db->get_random_output(amount); + if (seen_indices.count(i)) + { + continue; + } + seen_indices.emplace(i); + + // get tx_hash, tx_out_index from DB + tx_out_index toi = m_db->get_output_tx_and_index(amount, i); + + // if the output's transaction is unlocked, add the output's index to + // our list. + if (is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first))) + { + add_out_to_get_random_outs(result_outs, amount, i); + } + } + } + } + return true; +} +//------------------------------------------------------------------ +// This function takes a list of block hashes from another node +// on the network to find where the split point is between us and them. +// This is used to see what to send another node that needs to sync. +bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, uint64_t& starter_offset) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + // make sure the request includes at least the genesis block, otherwise + // how can we expect to sync from the client that the block list came from? + if(!qblock_ids.size() /*|| !req.m_total_height*/) + { + LOG_PRINT_L1("Client sent wrong NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << qblock_ids.size() << /*", m_height=" << req.m_total_height <<*/ ", dropping connection"); + return false; + } + + // make sure that the last block in the request's block list matches + // the genesis block + auto gen_hash = m_db->get_block_hash_from_height(0); + if(qblock_ids.back() != gen_hash) + { + LOG_PRINT_L1("Client sent wrong NOTIFY_REQUEST_CHAIN: genesis block missmatch: " << std::endl << "id: " + << qblock_ids.back() << ", " << std::endl << "expected: " << gen_hash + << "," << std::endl << " dropping connection"); + return false; + } + + // Find the first block the foreign chain has that we also have. + // Assume qblock_ids is in reverse-chronological order. + auto bl_it = qblock_ids.begin(); + uint64_t split_height = 0; + for(; bl_it != qblock_ids.end(); bl_it++) + { + try + { + split_height = m_db->get_block_height(*bl_it); + break; + } + catch (const BLOCK_DNE& e) + { + continue; + } + catch (const std::exception& e) + { + LOG_PRINT_L1("Non-critical error trying to find block by hash in BlockchainDB, hash: " << *bl_it); + return false; + } + } + + // this should be impossible, as we checked that we share the genesis block, + // but just in case... + if(bl_it == qblock_ids.end()) + { + LOG_PRINT_L1("Internal error handling connection, can't find split point"); + return false; + } + + // if split_height remains 0, we didn't have any but the genesis block in common + // which is only fine if the blocks just have the genesis block + if(split_height == 0 && qblock_ids.size() > 1) + { + LOG_ERROR("Ours and foreign blockchain have only genesis block in common... o.O"); + return false; + } + + //we start to put block ids INCLUDING last known id, just to make other side be sure + starter_offset = split_height; + return true; +} +//------------------------------------------------------------------ +uint64_t Blockchain::block_difficulty(uint64_t i) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + try + { + return m_db->get_block_difficulty(i); + } + catch (const BLOCK_DNE& e) + { + LOG_PRINT_L0("Attempted to get block difficulty for height above blockchain height"); + } + return 0; +} +//------------------------------------------------------------------ +template<class t_ids_container, class t_blocks_container, class t_missed_container> +bool Blockchain::get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + for (const auto& block_hash : block_ids) + { + try + { + blocks.push_back(m_db->get_block(block_hash)); + } + catch (const BLOCK_DNE& e) + { + missed_bs.push_back(block_hash); + } + catch (const std::exception& e) + { + return false; + } + } + return true; +} +//------------------------------------------------------------------ +template<class t_ids_container, class t_tx_container, class t_missed_container> +bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + for (const auto& tx_hash : txs_ids) + { + try + { + txs.push_back(m_db->get_tx(tx_hash)); + } + catch (const TX_DNE& e) + { + missed_txs.push_back(tx_hash); + } + //FIXME: is this the correct way to handle this? + catch (const std::exception& e) + { + return false; + } + } + return true; +} +//------------------------------------------------------------------ +void Blockchain::print_blockchain(uint64_t start_index, uint64_t end_index) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + std::stringstream ss; + CRITICAL_REGION_LOCAL(m_blockchain_lock); + auto h = m_db->height(); + if(start_index > h) + { + LOG_PRINT_L1("Wrong starter index set: " << start_index << ", expected max index " << h); + return; + } + + for(size_t i = start_index; i <= h && i != end_index; i++) + { + ss << "height " << i + << ", timestamp " << m_db->get_block_timestamp(i) + << ", cumul_dif " << m_db->get_block_cumulative_difficulty(i) + << ", size " << m_db->get_block_size(i) + << "\nid\t\t" << m_db->get_block_hash_from_height(i) + << "\ndifficulty\t\t" << m_db->get_block_difficulty(i) + << ", nonce " << m_db->get_block_from_height(i).nonce + << ", tx_count " << m_db->get_block_from_height(i).tx_hashes.size() + << std::endl; + } + LOG_PRINT_L1("Current blockchain:" << std::endl << ss.str()); + LOG_PRINT_L0("Blockchain printed with log level 1"); +} +//------------------------------------------------------------------ +void Blockchain::print_blockchain_index() +{ + LOG_PRINT_L3("Blockchain::" << __func__); + std::stringstream ss; + CRITICAL_REGION_LOCAL(m_blockchain_lock); + auto height = m_db->height(); + if (height != 0) + { + for(uint64_t i = 0; i <= height; i++) + { + ss << "height: " << i << ", hash: " << m_db->get_block_hash_from_height(i); + } + } + + LOG_PRINT_L0("Current blockchain index:" << std::endl + << ss.str() + ); +} +//------------------------------------------------------------------ +//TODO: remove this function and references to it +void Blockchain::print_blockchain_outs(const std::string& file) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + return; +} +//------------------------------------------------------------------ +// Find the split point between us and foreign blockchain and return +// (by reference) the most recent common block hash along with up to +// BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT additional (more recent) hashes. +bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + // if we can't find the split point, return false + if(!find_blockchain_supplement(qblock_ids, resp.start_height)) + { + return false; + } + + resp.total_height = get_current_blockchain_height(); + size_t count = 0; + for(size_t i = resp.start_height; i < resp.total_height && count < BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT; i++, count++) + { + resp.m_block_ids.push_back(m_db->get_block_hash_from_height(i)); + } + return true; +} +//------------------------------------------------------------------ +//FIXME: change argument to std::vector, low priority +// find split point between ours and foreign blockchain (or start at +// blockchain height <req_start_block>), and return up to max_count FULL +// blocks by reference. +bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<block, std::list<transaction> > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + // if a specific start height has been requested + if(req_start_block > 0) + { + // if requested height is higher than our chain, return false -- we can't help + if (req_start_block >= m_db->height()) + { + return false; + } + start_height = req_start_block; + } + else + { + if(!find_blockchain_supplement(qblock_ids, start_height)) + { + return false; + } + } + + total_height = get_current_blockchain_height(); + size_t count = 0; + for(size_t i = start_height; i < total_height && count < max_count; i++, count++) + { + blocks.resize(blocks.size()+1); + blocks.back().first = m_db->get_block_from_height(i); + std::list<crypto::hash> mis; + get_transactions(blocks.back().first.tx_hashes, blocks.back().second, mis); + CHECK_AND_ASSERT_MES(!mis.size(), false, "internal error, transaction from block not found"); + } + return true; +} +//------------------------------------------------------------------ +bool Blockchain::add_block_as_invalid(const block& bl, const crypto::hash& h) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + block_extended_info bei = AUTO_VAL_INIT(bei); + bei.bl = bl; + return add_block_as_invalid(bei, h); +} +//------------------------------------------------------------------ +bool Blockchain::add_block_as_invalid(const block_extended_info& bei, const crypto::hash& h) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + auto i_res = m_invalid_blocks.insert(std::map<crypto::hash, block_extended_info>::value_type(h, bei)); + CHECK_AND_ASSERT_MES(i_res.second, false, "at insertion invalid by tx returned status existed"); + LOG_PRINT_L1("BLOCK ADDED AS INVALID: " << h << std::endl << ", prev_id=" << bei.bl.prev_id << ", m_invalid_blocks count=" << m_invalid_blocks.size()); + return true; +} +//------------------------------------------------------------------ +bool Blockchain::have_block(const crypto::hash& id) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + if(m_db->block_exists(id)) + { + LOG_PRINT_L3("block exists in main chain"); + return true; + } + + if(m_alternative_chains.count(id)) + { + LOG_PRINT_L3("block found in m_alternative_chains"); + return true; + } + + if(m_invalid_blocks.count(id)) + { + LOG_PRINT_L3("block found in m_invalid_blocks"); + return true; + } + + return false; +} +//------------------------------------------------------------------ +bool Blockchain::handle_block_to_main_chain(const block& bl, block_verification_context& bvc) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + crypto::hash id = get_block_hash(bl); + return handle_block_to_main_chain(bl, id, bvc); +} +//------------------------------------------------------------------ +size_t Blockchain::get_total_transactions() const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + return m_db->get_tx_count(); +} +//------------------------------------------------------------------ +// This function checks each input in the transaction <tx> to make sure it +// has not been used already, and adds its key to the container <keys_this_block>. +// +// This container should be managed by the code that validates blocks so we don't +// have to store the used keys in a given block in the permanent storage only to +// remove them later if the block fails validation. +bool Blockchain::check_for_double_spend(const transaction& tx, key_images_container& keys_this_block) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + struct add_transaction_input_visitor: public boost::static_visitor<bool> + { + key_images_container& m_spent_keys; + BlockchainDB* m_db; + add_transaction_input_visitor(key_images_container& spent_keys, BlockchainDB* db):m_spent_keys(spent_keys), m_db(db) + {} + bool operator()(const txin_to_key& in) const + { + const crypto::key_image& ki = in.k_image; + + // attempt to insert the newly-spent key into the container of + // keys spent this block. If this fails, the key was spent already + // in this block, return false to flag that a double spend was detected. + // + // if the insert into the block-wide spent keys container succeeds, + // check the blockchain-wide spent keys container and make sure the + // key wasn't used in another block already. + auto r = m_spent_keys.insert(ki); + if(!r.second || m_db->has_key_image(ki)) + { + //double spend detected + return false; + } + + // if no double-spend detected, return true + return true; + } + + bool operator()(const txin_gen& tx) const{return true;} + bool operator()(const txin_to_script& tx) const{return false;} + bool operator()(const txin_to_scripthash& tx) const{return false;} + }; + + for (const txin_v& in : tx.vin) + { + if(!boost::apply_visitor(add_transaction_input_visitor(keys_this_block, m_db), in)) + { + LOG_ERROR("Double spend detected!"); + return false; + } + } + + return true; +} +//------------------------------------------------------------------ +bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + if (!m_db->tx_exists(tx_id)) + { + LOG_PRINT_RED_L1("warning: get_tx_outputs_gindexs failed to find transaction with id = " << tx_id); + return false; + } + + // get amount output indexes, currently referred to in parts as "output global indices", but they are actually specific to amounts + indexs = m_db->get_tx_amount_output_indices(tx_id); + CHECK_AND_ASSERT_MES(indexs.size(), false, "internal error: global indexes for transaction " << tx_id << " is empty"); + + return true; +} +//------------------------------------------------------------------ +// This function overloads its sister function with +// an extra value (hash of highest block that holds an output used as input) +// as a return-by-reference. +bool Blockchain::check_tx_inputs(const transaction& tx, uint64_t& max_used_block_height, crypto::hash& max_used_block_id) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + bool res = check_tx_inputs(tx, &max_used_block_height); + if(!res) return false; + CHECK_AND_ASSERT_MES(max_used_block_height < m_db->height(), false, "internal error: max used block index=" << max_used_block_height << " is not less then blockchain size = " << m_db->height()); + max_used_block_id = m_db->get_block_hash_from_height(max_used_block_height); + return true; +} +//------------------------------------------------------------------ +bool Blockchain::have_tx_keyimges_as_spent(const transaction &tx) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + BOOST_FOREACH(const txin_v& in, tx.vin) + { + CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, in_to_key, true); + if(have_tx_keyimg_as_spent(in_to_key.k_image)) + return true; + } + return false; +} +//------------------------------------------------------------------ +// This function validates transaction inputs and their keys. Previously +// it also performed double spend checking, but that has been moved to its +// own function. +bool Blockchain::check_tx_inputs(const transaction& tx, uint64_t* pmax_used_block_height) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + size_t sig_index = 0; + if(pmax_used_block_height) + *pmax_used_block_height = 0; + + crypto::hash tx_prefix_hash = get_transaction_prefix_hash(tx); + + for (const auto& txin : tx.vin) + { + // make sure output being spent is of type txin_to_key, rather than + // e.g. txin_gen, which is only used for miner transactions + CHECK_AND_ASSERT_MES(txin.type() == typeid(txin_to_key), false, "wrong type id in tx input at Blockchain::check_tx_inputs"); + const txin_to_key& in_to_key = boost::get<txin_to_key>(txin); + + // make sure tx output has key offset(s) (is signed to be used) + CHECK_AND_ASSERT_MES(in_to_key.key_offsets.size(), false, "empty in_to_key.key_offsets in transaction with id " << get_transaction_hash(tx)); + + // basically, make sure number of inputs == number of signatures + CHECK_AND_ASSERT_MES(sig_index < tx.signatures.size(), false, "wrong transaction: not signature entry for input with index= " << sig_index); + + // make sure that output being spent matches up correctly with the + // signature spending it. + if(!check_tx_input(in_to_key, tx_prefix_hash, tx.signatures[sig_index], pmax_used_block_height)) + { + LOG_PRINT_L1("Failed to check ring signature for tx " << get_transaction_hash(tx) << " vin key with k_image: " << in_to_key.k_image << " sig_index: " << sig_index); + if (pmax_used_block_height) // a default value of NULL is used when called from Blockchain::handle_block_to_main_chain() + { + LOG_PRINT_L1(" *pmax_used_block_height: " << *pmax_used_block_height); + } + return false; + } + + sig_index++; + } + + return true; +} +//------------------------------------------------------------------ +// This function checks to see if a tx is unlocked. unlock_time is either +// a block index or a unix time. +bool Blockchain::is_tx_spendtime_unlocked(uint64_t unlock_time) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + if(unlock_time < CRYPTONOTE_MAX_BLOCK_NUMBER) + { + //interpret as block index + if(get_current_blockchain_height() + CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_BLOCKS >= unlock_time) + return true; + else + return false; + }else + { + //interpret as time + uint64_t current_time = static_cast<uint64_t>(time(NULL)); + if(current_time + CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_SECONDS >= unlock_time) + return true; + else + return false; + } + return false; +} +//------------------------------------------------------------------ +// This function locates all outputs associated with a given input (mixins) +// and validates that they exist and are usable. It also checks the ring +// signature for each input. +bool Blockchain::check_tx_input(const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector<crypto::signature>& sig, uint64_t* pmax_related_block_height) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + struct outputs_visitor + { + std::vector<const crypto::public_key *>& m_p_output_keys; + std::vector<crypto::public_key >& m_output_keys; + const Blockchain& m_bch; + outputs_visitor(std::vector<crypto::public_key >& output_keys, std::vector<const crypto::public_key *>& p_output_keys, const Blockchain& bch) : m_output_keys(output_keys), m_p_output_keys(p_output_keys), m_bch(bch) + {} + bool handle_output(const transaction& tx, const tx_out& out) + { + //check tx unlock time + if(!m_bch.is_tx_spendtime_unlocked(tx.unlock_time)) + { + LOG_PRINT_L1("One of outputs for one of inputs has wrong tx.unlock_time = " << tx.unlock_time); + return false; + } + + if(out.target.type() != typeid(txout_to_key)) + { + LOG_PRINT_L1("Output has wrong type id, which=" << out.target.which()); + return false; + } + + m_output_keys.push_back(boost::get<txout_to_key>(out.target).key); + return true; + } + }; + + //check ring signature + std::vector<crypto::public_key> output_keys; + std::vector<const crypto::public_key *> p_output_keys; + outputs_visitor vi(output_keys, p_output_keys, *this); + if(!scan_outputkeys_for_indexes(txin, vi, pmax_related_block_height)) + { + LOG_PRINT_L1("Failed to get output keys for tx with amount = " << print_money(txin.amount) << " and count indexes " << txin.key_offsets.size()); + return false; + } + + for (auto& k : output_keys) + { + p_output_keys.push_back(&k); + } + + if(txin.key_offsets.size() != output_keys.size()) + { + LOG_PRINT_L1("Output keys for tx with amount = " << txin.amount << " and count indexes " << txin.key_offsets.size() << " returned wrong keys count " << output_keys.size()); + return false; + } + CHECK_AND_ASSERT_MES(sig.size() == output_keys.size(), false, "internal error: tx signatures count=" << sig.size() << " mismatch with outputs keys count for inputs=" << output_keys.size()); + if(m_is_in_checkpoint_zone) + return true; + return crypto::check_ring_signature(tx_prefix_hash, txin.k_image, p_output_keys, sig.data()); +} +//------------------------------------------------------------------ +//TODO: Is this intended to do something else? Need to look into the todo there. +uint64_t Blockchain::get_adjusted_time() const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + //TODO: add collecting median time + return time(NULL); +} +//------------------------------------------------------------------ +//TODO: revisit, has changed a bit on upstream +bool Blockchain::check_block_timestamp(std::vector<uint64_t>& timestamps, const block& b) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + uint64_t median_ts = epee::misc_utils::median(timestamps); + + if(b.timestamp < median_ts) + { + LOG_PRINT_L1("Timestamp of block with id: " << get_block_hash(b) << ", " << b.timestamp << ", less than median of last " << BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW << " blocks, " << median_ts); + return false; + } + + return true; +} +//------------------------------------------------------------------ +// This function grabs the timestamps from the most recent <n> blocks, +// where n = BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW. If there are not those many +// blocks in the blockchain, the timestap is assumed to be valid. If there +// are, this function returns: +// true if the block's timestamp is not less than the timestamp of the +// median of the selected blocks +// false otherwise +bool Blockchain::check_block_timestamp(const block& b) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + if(b.timestamp > get_adjusted_time() + CRYPTONOTE_BLOCK_FUTURE_TIME_LIMIT) + { + LOG_PRINT_L1("Timestamp of block with id: " << get_block_hash(b) << ", " << b.timestamp << ", bigger than adjusted time + 2 hours"); + return false; + } + + // if not enough blocks, no proper median yet, return true + if(m_db->height() < BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW + 1) + { + return true; + } + + std::vector<uint64_t> timestamps; + auto h = m_db->height(); + + // need most recent 60 blocks, get index of first of those + // using +1 because BlockchainDB::height() returns the index of the top block, + // not the size of the blockchain (0-indexed) + size_t offset = h - BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW - 1; + for(;offset < h; ++offset) + { + timestamps.push_back(m_db->get_block_timestamp(offset)); + } + + return check_block_timestamp(timestamps, b); +} +//------------------------------------------------------------------ +// Needs to validate the block and acquire each transaction from the +// transaction mem_pool, then pass the block and transactions to +// m_db->add_block() +bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& id, block_verification_context& bvc) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + + TIME_MEASURE_START(block_processing_time); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + if(bl.prev_id != get_tail_id()) + { + LOG_PRINT_L1("Block with id: " << id << std::endl + << "has wrong prev_id: " << bl.prev_id << std::endl + << "expected: " << get_tail_id()); + return false; + } + + // make sure block timestamp is not less than the median timestamp + // of a set number of the most recent blocks. + if(!check_block_timestamp(bl)) + { + LOG_PRINT_L1("Block with id: " << id << std::endl + << "has invalid timestamp: " << bl.timestamp); + bvc.m_verifivation_failed = true; + return false; + } + + //check proof of work + TIME_MEASURE_START(target_calculating_time); + + // get the target difficulty for the block. + // the calculation can overflow, among other failure cases, + // so we need to check the return type. + // FIXME: get_difficulty_for_next_block can also assert, look into + // changing this to throwing exceptions instead so we can clean up. + difficulty_type current_diffic = get_difficulty_for_next_block(); + CHECK_AND_ASSERT_MES(current_diffic, false, "!!!!!!!!! difficulty overhead !!!!!!!!!"); + + TIME_MEASURE_FINISH(target_calculating_time); + + TIME_MEASURE_START(longhash_calculating_time); + + crypto::hash proof_of_work = null_hash; + + // Formerly the code below contained an if loop with the following condition + // !m_checkpoints.is_in_checkpoint_zone(get_current_blockchain_height()) + // however, this caused the daemon to not bother checking PoW for blocks + // before checkpoints, which is very dangerous behaviour. We moved the PoW + // validation out of the next chunk of code to make sure that we correctly + // check PoW now. + // FIXME: height parameter is not used...should it be used or should it not + // be a parameter? + proof_of_work = get_block_longhash(bl, m_db->height()); + + // validate proof_of_work versus difficulty target + if(!check_hash(proof_of_work, current_diffic)) + { + LOG_PRINT_L1("Block with id: " << id << std::endl + << "does not have enough proof of work: " << proof_of_work << std::endl + << "unexpected difficulty: " << current_diffic ); + bvc.m_verifivation_failed = true; + return false; + } + + // If we're at a checkpoint, ensure that our hardcoded checkpoint hash + // is correct. + if(m_checkpoints.is_in_checkpoint_zone(get_current_blockchain_height())) + { + if(!m_checkpoints.check_block(get_current_blockchain_height(), id)) + { + LOG_ERROR("CHECKPOINT VALIDATION FAILED"); + bvc.m_verifivation_failed = true; + return false; + } + } + + TIME_MEASURE_FINISH(longhash_calculating_time); + + // sanity check basic miner tx properties + if(!prevalidate_miner_transaction(bl, m_db->height())) + { + LOG_PRINT_L1("Block with id: " << id + << " failed to pass prevalidation"); + bvc.m_verifivation_failed = true; + return false; + } + + size_t coinbase_blob_size = get_object_blobsize(bl.miner_tx); + size_t cumulative_block_size = coinbase_blob_size; + + std::vector<transaction> txs; + key_images_container keys; + + uint64_t fee_summary = 0; + + // Iterate over the block's transaction hashes, grabbing each + // from the tx_pool and validating them. Each is then added + // to txs. Keys spent in each are added to <keys> by the double spend check. + for (const crypto::hash& tx_id : bl.tx_hashes) + { + transaction tx; + size_t blob_size = 0; + uint64_t fee = 0; + + if (m_db->tx_exists(tx_id)) + { + LOG_PRINT_L1("Block with id: " << id << " attempting to add transaction already in blockchain with id: " << tx_id); + bvc.m_verifivation_failed = true; + break; + } + + // get transaction with hash <tx_id> from tx_pool + if(!m_tx_pool.take_tx(tx_id, tx, blob_size, fee)) + { + LOG_PRINT_L1("Block with id: " << id << " has at least one unknown transaction with id: " << tx_id); + bvc.m_verifivation_failed = true; + break; + } + + // add the transaction to the temp list of transactions, so we can either + // store the list of transactions all at once or return the ones we've + // taken from the tx_pool back to it if the block fails verification. + txs.push_back(tx); + + // validate that transaction inputs and the keys spending them are correct. + if(!check_tx_inputs(tx)) + { + LOG_PRINT_L1("Block with id: " << id << " has at least one transaction (id: " << tx_id << ") with wrong inputs."); + + //TODO: why is this done? make sure that keeping invalid blocks makes sense. + add_block_as_invalid(bl, id); + LOG_PRINT_L1("Block with id " << id << " added as invalid becouse of wrong inputs in transactions"); + bvc.m_verifivation_failed = true; + break; + } + + if (!check_for_double_spend(tx, keys)) + { + LOG_PRINT_L0("Double spend detected in transaction (id: " << tx_id); + bvc.m_verifivation_failed = true; + break; + } + + fee_summary += fee; + cumulative_block_size += blob_size; + } + + uint64_t base_reward = 0; + uint64_t already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height() - 1) : 0; + if(!validate_miner_transaction(bl, cumulative_block_size, fee_summary, base_reward, already_generated_coins)) + { + LOG_PRINT_L1("Block with id: " << id + << " has incorrect miner transaction"); + bvc.m_verifivation_failed = true; + } + + + block_extended_info bei = boost::value_initialized<block_extended_info>(); + size_t block_size; + difficulty_type cumulative_difficulty; + + // populate various metadata about the block to be stored alongside it. + block_size = cumulative_block_size; + cumulative_difficulty = current_diffic; + already_generated_coins = already_generated_coins + base_reward; + if(m_db->height()) + cumulative_difficulty += m_db->get_block_cumulative_difficulty(m_db->height() - 1); + + update_next_cumulative_size_limit(); + + TIME_MEASURE_FINISH(block_processing_time); + + uint64_t new_height = 0; + bool add_success = true; + if (!bvc.m_verifivation_failed) + { + try + { + new_height = m_db->add_block(bl, block_size, cumulative_difficulty, already_generated_coins, txs); + } + catch (const std::exception& e) + { + //TODO: figure out the best way to deal with this failure + LOG_ERROR("Error adding block with hash: " << id << " to blockchain, what = " << e.what()); + add_success = false; + } + } + + // if we failed for any reason to verify the block, return taken + // transactions to the tx_pool. + if (bvc.m_verifivation_failed || !add_success) + { + // return taken transactions to transaction pool + for (auto& tx : txs) + { + cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); + if (!m_tx_pool.add_tx(tx, tvc, true)) + { + LOG_PRINT_L0("Failed to return taken transaction with hash: " << get_transaction_hash(tx) << " to tx_pool"); + } + } + return false; + } + + LOG_PRINT_L1("+++++ BLOCK SUCCESSFULLY ADDED" << std::endl << "id:\t" << id + << std::endl << "PoW:\t" << proof_of_work + << std::endl << "HEIGHT " << new_height << ", difficulty:\t" << current_diffic + << std::endl << "block reward: " << print_money(fee_summary + base_reward) << "(" << print_money(base_reward) << " + " << print_money(fee_summary) + << "), coinbase_blob_size: " << coinbase_blob_size << ", cumulative size: " << cumulative_block_size + << ", " << block_processing_time << "("<< target_calculating_time << "/" << longhash_calculating_time << ")ms"); + + bvc.m_added_to_main_chain = true; + + // appears to be a NOP *and* is called elsewhere. wat? + m_tx_pool.on_blockchain_inc(new_height, id); + + return true; +} +//------------------------------------------------------------------ +bool Blockchain::update_next_cumulative_size_limit() +{ + LOG_PRINT_L3("Blockchain::" << __func__); + std::vector<size_t> sz; + get_last_n_blocks_sizes(sz, CRYPTONOTE_REWARD_BLOCKS_WINDOW); + + uint64_t median = epee::misc_utils::median(sz); + if(median <= CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE) + median = CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE; + + m_current_block_cumul_sz_limit = median*2; + return true; +} +//------------------------------------------------------------------ +bool Blockchain::add_new_block(const block& bl_, block_verification_context& bvc) +{ + LOG_PRINT_L3("Blockchain::" << __func__); + //copy block here to let modify block.target + block bl = bl_; + crypto::hash id = get_block_hash(bl); + CRITICAL_REGION_LOCAL(m_tx_pool);//to avoid deadlock lets lock tx_pool for whole add/reorganize process + CRITICAL_REGION_LOCAL1(m_blockchain_lock); + if(have_block(id)) + { + LOG_PRINT_L3("block with id = " << id << " already exists"); + bvc.m_already_exists = true; + return false; + } + + //check that block refers to chain tail + if(!(bl.prev_id == get_tail_id())) + { + //chain switching or wrong block + bvc.m_added_to_main_chain = false; + return handle_alternative_block(bl, id, bvc); + //never relay alternative blocks + } + + return handle_block_to_main_chain(bl, id, bvc); +} +//------------------------------------------------------------------ +void Blockchain::check_against_checkpoints(const checkpoints& points, bool enforce) +{ + const auto& pts = points.get_points(); + + for (const auto& pt : pts) + { + // if the checkpoint is for a block we don't have yet, move on + if (pt.first >= m_db->height()) + { + continue; + } + + if (!points.check_block(pt.first, m_db->get_block_hash_from_height(pt.first))) + { + // if asked to enforce checkpoints, roll back to a couple of blocks before the checkpoint + if (enforce) + { + LOG_ERROR("Local blockchain failed to pass a checkpoint, rolling back!"); + std::list<block> empty; + rollback_blockchain_switching(empty, pt.first - 2); + } + else + { + LOG_ERROR("WARNING: local blockchain failed to pass a MoneroPulse checkpoint, and you could be on a fork. You should either sync up from scratch, OR download a fresh blockchain bootstrap, OR enable checkpoint enforcing with the --enforce-dns-checkpointing command-line option"); + } + } + } +} +//------------------------------------------------------------------ +// returns false if any of the checkpoints loading returns false. +// That should happen only if a checkpoint is added that conflicts +// with an existing checkpoint. +bool Blockchain::update_checkpoints(const std::string& file_path, bool check_dns) +{ + if (!cryptonote::load_checkpoints_from_json(m_checkpoints, file_path)) + { + return false; + } + + // if we're checking both dns and json, load checkpoints from dns. + // if we're not hard-enforcing dns checkpoints, handle accordingly + if (m_enforce_dns_checkpoints && check_dns) + { + if (!cryptonote::load_checkpoints_from_dns(m_checkpoints)) + { + return false; + } + } + else if (check_dns) + { + checkpoints dns_points; + cryptonote::load_checkpoints_from_dns(dns_points); + if (m_checkpoints.check_for_conflicts(dns_points)) + { + check_against_checkpoints(dns_points, false); + } + else + { + LOG_PRINT_L0("One or more checkpoints fetched from DNS conflicted with existing checkpoints!"); + } + } + + check_against_checkpoints(m_checkpoints, true); + + return true; +} +//------------------------------------------------------------------ +void Blockchain::set_enforce_dns_checkpoints(bool enforce_checkpoints) +{ + m_enforce_dns_checkpoints = enforce_checkpoints; +} diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h new file mode 100644 index 000000000..bc13901d2 --- /dev/null +++ b/src/cryptonote_core/blockchain.h @@ -0,0 +1,229 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#pragma once +#include <boost/serialization/serialization.hpp> +#include <boost/serialization/version.hpp> +#include <boost/serialization/list.hpp> +#include <boost/multi_index_container.hpp> +#include <boost/multi_index/global_fun.hpp> +#include <boost/multi_index/hashed_index.hpp> +#include <boost/multi_index/member.hpp> +#include <boost/foreach.hpp> +#include <atomic> + +#include "syncobj.h" +#include "string_tools.h" +#include "cryptonote_basic.h" +#include "common/util.h" +#include "cryptonote_protocol/cryptonote_protocol_defs.h" +#include "rpc/core_rpc_server_commands_defs.h" +#include "difficulty.h" +#include "cryptonote_core/cryptonote_format_utils.h" +#include "verification_context.h" +#include "crypto/hash.h" +#include "checkpoints.h" +#include "blockchain_db/blockchain_db.h" + +namespace cryptonote +{ + class tx_memory_pool; + + /************************************************************************/ + /* */ + /************************************************************************/ + class Blockchain + { + public: + struct transaction_chain_entry + { + transaction tx; + uint64_t m_keeper_block_height; + size_t m_blob_size; + std::vector<uint64_t> m_global_output_indexes; + }; + + struct block_extended_info + { + block bl; + uint64_t height; + size_t block_cumulative_size; + difficulty_type cumulative_difficulty; + uint64_t already_generated_coins; + }; + + Blockchain(tx_memory_pool& tx_pool); + + bool init(BlockchainDB* db, const bool testnet = false); + bool deinit(); + + void set_checkpoints(checkpoints&& chk_pts) { m_checkpoints = chk_pts; } + + //bool push_new_block(); + bool get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks, std::list<transaction>& txs) const; + bool get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks) const; + bool get_alternative_blocks(std::list<block>& blocks) const; + size_t get_alternative_blocks_count() const; + crypto::hash get_block_id_by_height(uint64_t height) const; + bool get_block_by_hash(const crypto::hash &h, block &blk) const; + void get_all_known_block_ids(std::list<crypto::hash> &main, std::list<crypto::hash> &alt, std::list<crypto::hash> &invalid) const; + + template<class archive_t> + void serialize(archive_t & ar, const unsigned int version); + + bool have_tx(const crypto::hash &id) const; + bool have_tx_keyimges_as_spent(const transaction &tx) const; + bool have_tx_keyimg_as_spent(const crypto::key_image &key_im) const; + + template<class visitor_t> + bool scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, visitor_t& vis, uint64_t* pmax_related_block_height = NULL) const; + + uint64_t get_current_blockchain_height() const; + crypto::hash get_tail_id() const; + crypto::hash get_tail_id(uint64_t& height) const; + difficulty_type get_difficulty_for_next_block() const; + bool add_new_block(const block& bl_, block_verification_context& bvc); + bool reset_and_set_genesis_block(const block& b); + bool create_block_template(block& b, const account_public_address& miner_address, difficulty_type& di, uint64_t& height, const blobdata& ex_nonce) const; + bool have_block(const crypto::hash& id) const; + size_t get_total_transactions() const; + bool get_short_chain_history(std::list<crypto::hash>& ids) const; + bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const; + bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, uint64_t& starter_offset) const; + bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<block, std::list<transaction> > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count) const; + bool handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NOTIFY_RESPONSE_GET_OBJECTS::request& rsp); + bool handle_get_objects(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res); + bool get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const; + bool get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) const; + bool store_blockchain(); + bool check_tx_input(const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector<crypto::signature>& sig, uint64_t* pmax_related_block_height = NULL) const; + bool check_tx_inputs(const transaction& tx, uint64_t* pmax_used_block_height = NULL) const; + bool check_tx_inputs(const transaction& tx, uint64_t& pmax_used_block_height, crypto::hash& max_used_block_id) const; + uint64_t get_current_cumulative_blocksize_limit() const; + bool is_storing_blockchain()const{return m_is_blockchain_storing;} + uint64_t block_difficulty(uint64_t i) const; + + template<class t_ids_container, class t_blocks_container, class t_missed_container> + bool get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs) const; + + template<class t_ids_container, class t_tx_container, class t_missed_container> + bool get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const; + + //debug functions + void print_blockchain(uint64_t start_index, uint64_t end_index); + void print_blockchain_index(); + void print_blockchain_outs(const std::string& file); + + void check_against_checkpoints(const checkpoints& points, bool enforce); + void set_enforce_dns_checkpoints(bool enforce); + bool update_checkpoints(const std::string& file_path, bool check_dns); + + BlockchainDB& get_db() + { + return *m_db; + } + + private: + typedef std::unordered_map<crypto::hash, size_t> blocks_by_id_index; + typedef std::unordered_map<crypto::hash, transaction_chain_entry> transactions_container; + typedef std::unordered_set<crypto::key_image> key_images_container; + typedef std::vector<block_extended_info> blocks_container; + typedef std::unordered_map<crypto::hash, block_extended_info> blocks_ext_by_hash; + typedef std::unordered_map<crypto::hash, block> blocks_by_hash; + typedef std::map<uint64_t, std::vector<std::pair<crypto::hash, size_t>>> outputs_container; //crypto::hash - tx hash, size_t - index of out in transaction + + BlockchainDB* m_db; + + tx_memory_pool& m_tx_pool; + mutable epee::critical_section m_blockchain_lock; // TODO: add here reader/writer lock + + // main chain + blocks_container m_blocks; // height -> block_extended_info + blocks_by_id_index m_blocks_index; // crypto::hash -> height + transactions_container m_transactions; + key_images_container m_spent_keys; + size_t m_current_block_cumul_sz_limit; + + + // all alternative chains + blocks_ext_by_hash m_alternative_chains; // crypto::hash -> block_extended_info + + // some invalid blocks + blocks_ext_by_hash m_invalid_blocks; // crypto::hash -> block_extended_info + outputs_container m_outputs; + + + checkpoints m_checkpoints; + std::atomic<bool> m_is_in_checkpoint_zone; + std::atomic<bool> m_is_blockchain_storing; + bool m_enforce_dns_checkpoints; + + bool switch_to_alternative_blockchain(std::list<blocks_ext_by_hash::iterator>& alt_chain, bool discard_disconnected_chain); + block pop_block_from_blockchain(); + bool purge_transaction_from_blockchain(const crypto::hash& tx_id); + bool purge_transaction_keyimages_from_blockchain(const transaction& tx, bool strict_check); + + bool handle_block_to_main_chain(const block& bl, block_verification_context& bvc); + bool handle_block_to_main_chain(const block& bl, const crypto::hash& id, block_verification_context& bvc); + bool handle_alternative_block(const block& b, const crypto::hash& id, block_verification_context& bvc); + difficulty_type get_next_difficulty_for_alternative_chain(const std::list<blocks_ext_by_hash::iterator>& alt_chain, block_extended_info& bei) const; + bool prevalidate_miner_transaction(const block& b, uint64_t height); + bool validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins); + bool validate_transaction(const block& b, uint64_t height, const transaction& tx); + bool rollback_blockchain_switching(std::list<block>& original_chain, uint64_t rollback_height); + bool add_transaction_from_block(const transaction& tx, const crypto::hash& tx_id, const crypto::hash& bl_id, uint64_t bl_height); + bool push_transaction_to_global_outs_index(const transaction& tx, const crypto::hash& tx_id, std::vector<uint64_t>& global_indexes); + bool pop_transaction_from_global_index(const transaction& tx, const crypto::hash& tx_id); + void get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t count) const; + void add_out_to_get_random_outs(COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) const; + bool is_tx_spendtime_unlocked(uint64_t unlock_time) const; + bool add_block_as_invalid(const block& bl, const crypto::hash& h); + bool add_block_as_invalid(const block_extended_info& bei, const crypto::hash& h); + bool check_block_timestamp(const block& b) const; + bool check_block_timestamp(std::vector<uint64_t>& timestamps, const block& b) const; + uint64_t get_adjusted_time() const; + bool complete_timestamps_vector(uint64_t start_height, std::vector<uint64_t>& timestamps); + bool update_next_cumulative_size_limit(); + + bool check_for_double_spend(const transaction& tx, key_images_container& keys_this_block) const; + }; + + + /************************************************************************/ + /* */ + /************************************************************************/ + + #define CURRENT_BLOCKCHAIN_STORAGE_ARCHIVE_VER 12 + + //------------------------------------------------------------------ + +} // namespace cryptonote + +BOOST_CLASS_VERSION(cryptonote::Blockchain, CURRENT_BLOCKCHAIN_STORAGE_ARCHIVE_VER) diff --git a/src/cryptonote_core/blockchain_storage.cpp b/src/cryptonote_core/blockchain_storage.cpp index 4cca878db..ee4263687 100644 --- a/src/cryptonote_core/blockchain_storage.cpp +++ b/src/cryptonote_core/blockchain_storage.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -30,6 +30,7 @@ #include <algorithm> #include <cstdio> +#include <cmath> #include <boost/archive/binary_oarchive.hpp> #include <boost/archive/binary_iarchive.hpp> @@ -49,25 +50,27 @@ #include "crypto/hash.h" #include "cryptonote_core/checkpoints_create.h" //#include "serialization/json_archive.h" +#include "../../contrib/otshell_utils/utils.hpp" +#include "../../src/p2p/data_logger.hpp" using namespace cryptonote; DISABLE_VS_WARNINGS(4267) //------------------------------------------------------------------ -bool blockchain_storage::have_tx(const crypto::hash &id) +bool blockchain_storage::have_tx(const crypto::hash &id) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); return m_transactions.find(id) != m_transactions.end(); } //------------------------------------------------------------------ -bool blockchain_storage::have_tx_keyimg_as_spent(const crypto::key_image &key_im) +bool blockchain_storage::have_tx_keyimg_as_spent(const crypto::key_image &key_im) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); return m_spent_keys.find(key_im) != m_spent_keys.end(); } //------------------------------------------------------------------ -transaction *blockchain_storage::get_tx(const crypto::hash &id) +const transaction *blockchain_storage::get_tx(const crypto::hash &id) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); auto it = m_transactions.find(id); @@ -77,7 +80,7 @@ transaction *blockchain_storage::get_tx(const crypto::hash &id) return &it->second.tx; } //------------------------------------------------------------------ -uint64_t blockchain_storage::get_current_blockchain_height() +uint64_t blockchain_storage::get_current_blockchain_height() const { CRITICAL_REGION_LOCAL(m_blockchain_lock); return m_blocks.size(); @@ -87,6 +90,7 @@ bool blockchain_storage::init(const std::string& config_folder, bool testnet) { CRITICAL_REGION_LOCAL(m_blockchain_lock); m_config_folder = config_folder; + m_testnet = testnet; LOG_PRINT_L0("Loading blockchain..."); const std::string filename = m_config_folder + "/" CRYPTONOTE_BLOCKCHAINDATA_FILENAME; if(tools::unserialize_obj_from_file(*this, filename)) @@ -113,24 +117,14 @@ bool blockchain_storage::init(const std::string& config_folder, bool testnet) else { LOG_PRINT_L0("Can't load blockchain storage from file, generating genesis block."); - block bl = boost::value_initialized<block>(); - block_verification_context bvc = boost::value_initialized<block_verification_context>(); - if (testnet) - { - generate_genesis_block(bl, config::testnet::GENESIS_TX, config::testnet::GENESIS_NONCE); - } - else - { - generate_genesis_block(bl, config::GENESIS_TX, config::GENESIS_NONCE); - } - add_new_block(bl, bvc); - CHECK_AND_ASSERT_MES(!bvc.m_verifivation_failed && bvc.m_added_to_main_chain, false, "Failed to add genesis block to blockchain"); + if (!store_genesis_block(testnet, true)) + return false; } if(!m_blocks.size()) { LOG_PRINT_L0("Blockchain not loaded, generating genesis block."); - if (!store_genesis_block(testnet)) { + if (!store_genesis_block(testnet, false)) { return false; } } else { @@ -159,7 +153,7 @@ bool blockchain_storage::init(const std::string& config_folder, bool testnet) return true; } //------------------------------------------------------------------ -bool blockchain_storage::store_genesis_block(bool testnet) { +bool blockchain_storage::store_genesis_block(bool testnet, bool check_added) { block bl = ::boost::value_initialized<block>(); block_verification_context bvc = boost::value_initialized<block_verification_context>(); @@ -173,7 +167,7 @@ bool blockchain_storage::store_genesis_block(bool testnet) { } add_new_block(bl, bvc); - CHECK_AND_ASSERT_MES(!bvc.m_verifivation_failed, false, "Failed to add genesis block to blockchain"); + CHECK_AND_ASSERT_MES(!bvc.m_verifivation_failed && (bvc.m_added_to_main_chain || !check_added), false, "Failed to add genesis block to blockchain"); return true; } //------------------------------------------------------------------ @@ -231,7 +225,7 @@ bool blockchain_storage::pop_block_from_blockchain() m_blocks_index.erase(bl_ind); //pop block from core m_blocks.pop_back(); - m_tx_pool.on_blockchain_dec(m_blocks.size()-1, get_tail_id()); + m_tx_pool->on_blockchain_dec(m_blocks.size()-1, get_tail_id()); return true; } //------------------------------------------------------------------ @@ -307,7 +301,7 @@ bool blockchain_storage::purge_transaction_from_blockchain(const crypto::hash& t if(!is_coinbase(tx)) { cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); - bool r = m_tx_pool.add_tx(tx, tvc, true); + bool r = m_tx_pool->add_tx(tx, tvc, true); CHECK_AND_ASSERT_MES(r, false, "purge_block_data_from_blockchain: failed to add transaction to transaction pool"); } @@ -333,14 +327,14 @@ bool blockchain_storage::purge_block_data_from_blockchain(const block& bl, size_ return res; } //------------------------------------------------------------------ -crypto::hash blockchain_storage::get_tail_id(uint64_t& height) +crypto::hash blockchain_storage::get_tail_id(uint64_t& height) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); height = get_current_blockchain_height()-1; return get_tail_id(); } //------------------------------------------------------------------ -crypto::hash blockchain_storage::get_tail_id() +crypto::hash blockchain_storage::get_tail_id() const { CRITICAL_REGION_LOCAL(m_blockchain_lock); crypto::hash id = null_hash; @@ -351,7 +345,7 @@ crypto::hash blockchain_storage::get_tail_id() return id; } //------------------------------------------------------------------ -bool blockchain_storage::get_short_chain_history(std::list<crypto::hash>& ids) +bool blockchain_storage::get_short_chain_history(std::list<crypto::hash>& ids) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); size_t i = 0; @@ -381,7 +375,7 @@ bool blockchain_storage::get_short_chain_history(std::list<crypto::hash>& ids) return true; } //------------------------------------------------------------------ -crypto::hash blockchain_storage::get_block_id_by_height(uint64_t height) +crypto::hash blockchain_storage::get_block_id_by_height(uint64_t height) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); if(height >= m_blocks.size()) @@ -390,7 +384,7 @@ crypto::hash blockchain_storage::get_block_id_by_height(uint64_t height) return get_block_hash(m_blocks[height].bl); } //------------------------------------------------------------------ -bool blockchain_storage::get_block_by_hash(const crypto::hash &h, block &blk) { +bool blockchain_storage::get_block_by_hash(const crypto::hash &h, block &blk) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); // try to find block in main chain @@ -410,20 +404,20 @@ bool blockchain_storage::get_block_by_hash(const crypto::hash &h, block &blk) { return false; } //------------------------------------------------------------------ -void blockchain_storage::get_all_known_block_ids(std::list<crypto::hash> &main, std::list<crypto::hash> &alt, std::list<crypto::hash> &invalid) { +void blockchain_storage::get_all_known_block_ids(std::list<crypto::hash> &main, std::list<crypto::hash> &alt, std::list<crypto::hash> &invalid) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); - BOOST_FOREACH(blocks_by_id_index::value_type &v, m_blocks_index) + BOOST_FOREACH(const blocks_by_id_index::value_type &v, m_blocks_index) main.push_back(v.first); - BOOST_FOREACH(blocks_ext_by_hash::value_type &v, m_alternative_chains) + BOOST_FOREACH(const blocks_ext_by_hash::value_type &v, m_alternative_chains) alt.push_back(v.first); - BOOST_FOREACH(blocks_ext_by_hash::value_type &v, m_invalid_blocks) + BOOST_FOREACH(const blocks_ext_by_hash::value_type &v, m_invalid_blocks) invalid.push_back(v.first); } //------------------------------------------------------------------ -difficulty_type blockchain_storage::get_difficulty_for_next_block() +difficulty_type blockchain_storage::get_difficulty_for_next_block() const { CRITICAL_REGION_LOCAL(m_blockchain_lock); std::vector<uint64_t> timestamps; @@ -535,7 +529,7 @@ bool blockchain_storage::switch_to_alternative_blockchain(std::list<blocks_ext_b return true; } //------------------------------------------------------------------ -difficulty_type blockchain_storage::get_next_difficulty_for_alternative_chain(const std::list<blocks_ext_by_hash::iterator>& alt_chain, block_extended_info& bei) +difficulty_type blockchain_storage::get_next_difficulty_for_alternative_chain(const std::list<blocks_ext_by_hash::iterator>& alt_chain, block_extended_info& bei) const { std::vector<uint64_t> timestamps; std::vector<difficulty_type> commulative_difficulties; @@ -580,7 +574,7 @@ difficulty_type blockchain_storage::get_next_difficulty_for_alternative_chain(co return next_difficulty(timestamps, commulative_difficulties); } //------------------------------------------------------------------ -bool blockchain_storage::prevalidate_miner_transaction(const block& b, uint64_t height) +bool blockchain_storage::prevalidate_miner_transaction(const block& b, uint64_t height) const { CHECK_AND_ASSERT_MES(b.miner_tx.vin.size() == 1, false, "coinbase transaction in the block has no inputs"); CHECK_AND_ASSERT_MES(b.miner_tx.vin[0].type() == typeid(txin_gen), false, "coinbase transaction in the block has the wrong type"); @@ -603,7 +597,7 @@ bool blockchain_storage::prevalidate_miner_transaction(const block& b, uint64_t return true; } //------------------------------------------------------------------ -bool blockchain_storage::validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins) +bool blockchain_storage::validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins) const { //validate reward uint64_t money_in_use = 0; @@ -630,7 +624,7 @@ bool blockchain_storage::validate_miner_transaction(const block& b, size_t cumul return true; } //------------------------------------------------------------------ -bool blockchain_storage::get_backward_blocks_sizes(size_t from_height, std::vector<size_t>& sz, size_t count) +bool blockchain_storage::get_backward_blocks_sizes(size_t from_height, std::vector<size_t>& sz, size_t count) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); CHECK_AND_ASSERT_MES(from_height < m_blocks.size(), false, "Internal error: get_backward_blocks_sizes called with from_height=" << from_height << ", blockchain height = " << m_blocks.size()); @@ -642,7 +636,7 @@ bool blockchain_storage::get_backward_blocks_sizes(size_t from_height, std::vect return true; } //------------------------------------------------------------------ -bool blockchain_storage::get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t count) +bool blockchain_storage::get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t count) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); if(!m_blocks.size()) @@ -650,12 +644,12 @@ bool blockchain_storage::get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t return get_backward_blocks_sizes(m_blocks.size() -1, sz, count); } //------------------------------------------------------------------ -uint64_t blockchain_storage::get_current_comulative_blocksize_limit() +uint64_t blockchain_storage::get_current_cumulative_blocksize_limit() const { return m_current_block_cumul_sz_limit; } //------------------------------------------------------------------ -bool blockchain_storage::create_block_template(block& b, const account_public_address& miner_address, difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce) +bool blockchain_storage::create_block_template(block& b, const account_public_address& miner_address, difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce) const { size_t median_size; uint64_t already_generated_coins; @@ -676,16 +670,16 @@ bool blockchain_storage::create_block_template(block& b, const account_public_ad size_t txs_size; uint64_t fee; - if (!m_tx_pool.fill_block_template(b, median_size, already_generated_coins, txs_size, fee)) { + if (!m_tx_pool->fill_block_template(b, median_size, already_generated_coins, txs_size, fee)) { return false; } #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) size_t real_txs_size = 0; uint64_t real_fee = 0; - CRITICAL_REGION_BEGIN(m_tx_pool.m_transactions_lock); + CRITICAL_REGION_BEGIN(m_tx_pool->m_transactions_lock); BOOST_FOREACH(crypto::hash &cur_hash, b.tx_hashes) { - auto cur_res = m_tx_pool.m_transactions.find(cur_hash); - if (cur_res == m_tx_pool.m_transactions.end()) { + auto cur_res = m_tx_pool->m_transactions.find(cur_hash); + if (cur_res == m_tx_pool->m_transactions.end()) { LOG_ERROR("Creating block template: error: transaction not found"); continue; } @@ -774,7 +768,7 @@ bool blockchain_storage::create_block_template(block& b, const account_public_ad return false; } //------------------------------------------------------------------ -bool blockchain_storage::complete_timestamps_vector(uint64_t start_top_height, std::vector<uint64_t>& timestamps) +bool blockchain_storage::complete_timestamps_vector(uint64_t start_top_height, std::vector<uint64_t>& timestamps) const { if(timestamps.size() >= BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW) @@ -939,7 +933,7 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto:: return true; } //------------------------------------------------------------------ -bool blockchain_storage::get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks, std::list<transaction>& txs) +bool blockchain_storage::get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks, std::list<transaction>& txs) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); if(start_offset >= m_blocks.size()) @@ -955,7 +949,7 @@ bool blockchain_storage::get_blocks(uint64_t start_offset, size_t count, std::li return true; } //------------------------------------------------------------------ -bool blockchain_storage::get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks) +bool blockchain_storage::get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); if(start_offset >= m_blocks.size()) @@ -999,7 +993,7 @@ bool blockchain_storage::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& return true; } //------------------------------------------------------------------ -bool blockchain_storage::get_alternative_blocks(std::list<block>& blocks) +bool blockchain_storage::get_alternative_blocks(std::list<block>& blocks) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1010,21 +1004,21 @@ bool blockchain_storage::get_alternative_blocks(std::list<block>& blocks) return true; } //------------------------------------------------------------------ -size_t blockchain_storage::get_alternative_blocks_count() +size_t blockchain_storage::get_alternative_blocks_count() const { CRITICAL_REGION_LOCAL(m_blockchain_lock); return m_alternative_chains.size(); } //------------------------------------------------------------------ -bool blockchain_storage::add_out_to_get_random_outs(std::vector<std::pair<crypto::hash, size_t> >& amount_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) +bool blockchain_storage::add_out_to_get_random_outs(const std::vector<std::pair<crypto::hash, size_t> >& amount_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); - transactions_container::iterator tx_it = m_transactions.find(amount_outs[i].first); + transactions_container::const_iterator tx_it = m_transactions.find(amount_outs[i].first); CHECK_AND_ASSERT_MES(tx_it != m_transactions.end(), false, "internal error: transaction with id " << amount_outs[i].first << ENDL << ", used in mounts global index for amount=" << amount << ": i=" << i << "not found in transactions index"); CHECK_AND_ASSERT_MES(tx_it->second.tx.vout.size() > amount_outs[i].second, false, "internal error: in global outs index, transaction out index=" << amount_outs[i].second << " more than transaction outputs = " << tx_it->second.tx.vout.size() << ", for tx id = " << amount_outs[i].first); - transaction& tx = tx_it->second.tx; + const transaction& tx = tx_it->second.tx; CHECK_AND_ASSERT_MES(tx.vout[amount_outs[i].second].target.type() == typeid(txout_to_key), false, "unknown tx out type"); //check if transaction is unlocked @@ -1037,7 +1031,7 @@ bool blockchain_storage::add_out_to_get_random_outs(std::vector<std::pair<crypto return true; } //------------------------------------------------------------------ -size_t blockchain_storage::find_end_of_allowed_index(const std::vector<std::pair<crypto::hash, size_t> >& amount_outs) +size_t blockchain_storage::find_end_of_allowed_index(const std::vector<std::pair<crypto::hash, size_t> >& amount_outs) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); if(!amount_outs.size()) @@ -1046,7 +1040,7 @@ size_t blockchain_storage::find_end_of_allowed_index(const std::vector<std::pair do { --i; - transactions_container::iterator it = m_transactions.find(amount_outs[i].first); + transactions_container::const_iterator it = m_transactions.find(amount_outs[i].first); CHECK_AND_ASSERT_MES(it != m_transactions.end(), 0, "internal error: failed to find transaction from outputs index with tx_id=" << amount_outs[i].first); if(it->second.m_keeper_block_height + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW <= get_current_blockchain_height() ) return i+1; @@ -1054,7 +1048,7 @@ size_t blockchain_storage::find_end_of_allowed_index(const std::vector<std::pair return 0; } //------------------------------------------------------------------ -bool blockchain_storage::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) +bool blockchain_storage::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); BOOST_FOREACH(uint64_t amount, req.amounts) @@ -1067,7 +1061,7 @@ bool blockchain_storage::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDO LOG_PRINT_L1("COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS: not outs for amount " << amount << ", wallet should use some real outs when it lookup for some mix, so, at least one out for this amount should exist"); continue;//actually this is strange situation, wallet should use some real outs when it lookup for some mix, so, at least one out for this amount should exist } - std::vector<std::pair<crypto::hash, size_t> >& amount_outs = it->second; + const std::vector<std::pair<crypto::hash, size_t> >& amount_outs = it->second; //it is not good idea to use top fresh outs, because it increases possibility of transaction canceling on split //lets find upper bound of not fresh outs size_t up_index_limit = find_end_of_allowed_index(amount_outs); @@ -1078,7 +1072,13 @@ bool blockchain_storage::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDO size_t try_count = 0; for(uint64_t j = 0; j != req.outs_count && try_count < up_index_limit;) { - size_t i = crypto::rand<size_t>()%up_index_limit; + // triangular distribution over [a,b) with a=0, mode c=b=up_index_limit + uint64_t r = crypto::rand<uint64_t>() % ((uint64_t)1 << 53); + double frac = std::sqrt((double)r / ((uint64_t)1 << 53)); + size_t i = (size_t)(frac*up_index_limit); + // just in case rounding up to 1 occurs after sqrt + if (i == up_index_limit) + --i; if(used.count(i)) continue; bool added = add_out_to_get_random_outs(amount_outs, result_outs, amount, i); @@ -1096,7 +1096,7 @@ bool blockchain_storage::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDO return true; } //------------------------------------------------------------------ -bool blockchain_storage::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, uint64_t& starter_offset) +bool blockchain_storage::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, uint64_t& starter_offset) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1143,7 +1143,7 @@ bool blockchain_storage::find_blockchain_supplement(const std::list<crypto::hash return true; } //------------------------------------------------------------------ -uint64_t blockchain_storage::block_difficulty(size_t i) +uint64_t blockchain_storage::block_difficulty(size_t i) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); CHECK_AND_ASSERT_MES(i < m_blocks.size(), false, "wrong block index i = " << i << " at blockchain_storage::block_difficulty()"); @@ -1153,6 +1153,31 @@ uint64_t blockchain_storage::block_difficulty(size_t i) return m_blocks[i].cumulative_difficulty - m_blocks[i-1].cumulative_difficulty; } //------------------------------------------------------------------ +double blockchain_storage::get_avg_block_size( size_t count) const +{ + if (count > get_current_blockchain_height()) return 500; + + double average = 0; + _dbg1_c("net/blksize", "HEIGHT: " << get_current_blockchain_height()); + _dbg1_c("net/blksize", "BLOCK ID BY HEIGHT: " << get_block_id_by_height(get_current_blockchain_height()) ); + _dbg1_c("net/blksize", "BLOCK TAIL ID: " << get_tail_id() ); + std::vector<size_t> size_vector; + + get_backward_blocks_sizes(get_current_blockchain_height() - count, size_vector, count); + + std::vector<size_t>::iterator it; + it = size_vector.begin(); + while (it != size_vector.end()) { + average += *it; + _dbg2_c("net/blksize", "VECTOR ELEMENT: " << (*it) ); + it++; + } + average = average / count; + _dbg1_c("net/blksize", "VECTOR SIZE: " << size_vector.size() << " average=" << average); + + return average; +} +//------------------------------------------------------------------ void blockchain_storage::print_blockchain(uint64_t start_index, uint64_t end_index) { std::stringstream ss; @@ -1206,7 +1231,7 @@ void blockchain_storage::print_blockchain_outs(const std::string& file) } } //------------------------------------------------------------------ -bool blockchain_storage::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) +bool blockchain_storage::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); if(!find_blockchain_supplement(qblock_ids, resp.start_height)) @@ -1219,7 +1244,7 @@ bool blockchain_storage::find_blockchain_supplement(const std::list<crypto::hash return true; } //------------------------------------------------------------------ -bool blockchain_storage::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<block, std::list<transaction> > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count) +bool blockchain_storage::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<block, std::list<transaction> > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); if(req_start_block > 0) { @@ -1258,7 +1283,7 @@ bool blockchain_storage::add_block_as_invalid(const block_extended_info& bei, co return true; } //------------------------------------------------------------------ -bool blockchain_storage::have_block(const crypto::hash& id) +bool blockchain_storage::have_block(const crypto::hash& id) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); if(m_blocks_index.count(id)) @@ -1296,13 +1321,13 @@ bool blockchain_storage::push_transaction_to_global_outs_index(const transaction return true; } //------------------------------------------------------------------ -size_t blockchain_storage::get_total_transactions() +size_t blockchain_storage::get_total_transactions() const { CRITICAL_REGION_LOCAL(m_blockchain_lock); return m_transactions.size(); } //------------------------------------------------------------------ -bool blockchain_storage::get_outs(uint64_t amount, std::list<crypto::public_key>& pkeys) +bool blockchain_storage::get_outs(uint64_t amount, std::list<crypto::public_key>& pkeys) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); auto it = m_outputs.find(amount); @@ -1338,7 +1363,7 @@ bool blockchain_storage::pop_transaction_from_global_index(const transaction& tx return true; } //------------------------------------------------------------------ -bool blockchain_storage::add_transaction_from_block(const transaction& tx, const crypto::hash& tx_id, const crypto::hash& bl_id, uint64_t bl_height) +bool blockchain_storage::add_transaction_from_block(const transaction& tx, const crypto::hash& tx_id, const crypto::hash& bl_id, uint64_t bl_height, size_t blob_size) { CRITICAL_REGION_LOCAL(m_blockchain_lock); struct add_transaction_input_visitor: public boost::static_visitor<bool> @@ -1377,6 +1402,7 @@ bool blockchain_storage::add_transaction_from_block(const transaction& tx, const } transaction_chain_entry ch_e; ch_e.m_keeper_block_height = bl_height; + ch_e.m_blob_size = blob_size; ch_e.tx = tx; auto i_r = m_transactions.insert(std::pair<crypto::hash, transaction_chain_entry>(tx_id, ch_e)); if(!i_r.second) @@ -1392,7 +1418,7 @@ bool blockchain_storage::add_transaction_from_block(const transaction& tx, const return true; } //------------------------------------------------------------------ -bool blockchain_storage::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) +bool blockchain_storage::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); auto it = m_transactions.find(tx_id); @@ -1407,7 +1433,7 @@ bool blockchain_storage::get_tx_outputs_gindexs(const crypto::hash& tx_id, std:: return true; } //------------------------------------------------------------------ -bool blockchain_storage::check_tx_inputs(const transaction& tx, uint64_t& max_used_block_height, crypto::hash& max_used_block_id) +bool blockchain_storage::check_tx_inputs(const transaction& tx, uint64_t& max_used_block_height, crypto::hash& max_used_block_id) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); bool res = check_tx_inputs(tx, &max_used_block_height); @@ -1417,7 +1443,7 @@ bool blockchain_storage::check_tx_inputs(const transaction& tx, uint64_t& max_us return true; } //------------------------------------------------------------------ -bool blockchain_storage::have_tx_keyimges_as_spent(const transaction &tx) +bool blockchain_storage::have_tx_keyimges_as_spent(const transaction &tx) const { BOOST_FOREACH(const txin_v& in, tx.vin) { @@ -1428,13 +1454,13 @@ bool blockchain_storage::have_tx_keyimges_as_spent(const transaction &tx) return false; } //------------------------------------------------------------------ -bool blockchain_storage::check_tx_inputs(const transaction& tx, uint64_t* pmax_used_block_height) +bool blockchain_storage::check_tx_inputs(const transaction& tx, uint64_t* pmax_used_block_height) const { crypto::hash tx_prefix_hash = get_transaction_prefix_hash(tx); return check_tx_inputs(tx, tx_prefix_hash, pmax_used_block_height); } //------------------------------------------------------------------ -bool blockchain_storage::check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash, uint64_t* pmax_used_block_height) +bool blockchain_storage::check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash, uint64_t* pmax_used_block_height) const { size_t sig_index = 0; if(pmax_used_block_height) @@ -1466,7 +1492,7 @@ bool blockchain_storage::check_tx_inputs(const transaction& tx, const crypto::ha return true; } //------------------------------------------------------------------ -bool blockchain_storage::is_tx_spendtime_unlocked(uint64_t unlock_time) +bool blockchain_storage::is_tx_spendtime_unlocked(uint64_t unlock_time) const { if(unlock_time < CRYPTONOTE_MAX_BLOCK_NUMBER) { @@ -1487,15 +1513,15 @@ bool blockchain_storage::is_tx_spendtime_unlocked(uint64_t unlock_time) return false; } //------------------------------------------------------------------ -bool blockchain_storage::check_tx_input(const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector<crypto::signature>& sig, uint64_t* pmax_related_block_height) +bool blockchain_storage::check_tx_input(const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector<crypto::signature>& sig, uint64_t* pmax_related_block_height) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); struct outputs_visitor { std::vector<const crypto::public_key *>& m_results_collector; - blockchain_storage& m_bch; - outputs_visitor(std::vector<const crypto::public_key *>& results_collector, blockchain_storage& bch):m_results_collector(results_collector), m_bch(bch) + const blockchain_storage& m_bch; + outputs_visitor(std::vector<const crypto::public_key *>& results_collector, const blockchain_storage& bch):m_results_collector(results_collector), m_bch(bch) {} bool handle_output(const transaction& tx, const tx_out& out) { @@ -1537,13 +1563,13 @@ bool blockchain_storage::check_tx_input(const txin_to_key& txin, const crypto::h return crypto::check_ring_signature(tx_prefix_hash, txin.k_image, output_keys, sig.data()); } //------------------------------------------------------------------ -uint64_t blockchain_storage::get_adjusted_time() +uint64_t blockchain_storage::get_adjusted_time() const { //TODO: add collecting median time return time(NULL); } //------------------------------------------------------------------ -bool blockchain_storage::check_block_timestamp_main(const block& b) +bool blockchain_storage::check_block_timestamp_main(const block& b) const { if(b.timestamp > get_adjusted_time() + CRYPTONOTE_BLOCK_FUTURE_TIME_LIMIT) { @@ -1559,7 +1585,7 @@ bool blockchain_storage::check_block_timestamp_main(const block& b) return check_block_timestamp(std::move(timestamps), b); } //------------------------------------------------------------------ -bool blockchain_storage::check_block_timestamp(std::vector<uint64_t> timestamps, const block& b) +bool blockchain_storage::check_block_timestamp(std::vector<uint64_t> timestamps, const block& b) const { if(timestamps.size() < BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW) return true; @@ -1642,10 +1668,12 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt bvc.m_verifivation_failed = true; return false; } - size_t coinbase_blob_size = get_object_blobsize(bl.miner_tx); + crypto::hash coinbase_hash = null_hash; + size_t coinbase_blob_size = 0; + get_transaction_hash(bl.miner_tx, coinbase_hash, coinbase_blob_size); size_t cumulative_block_size = coinbase_blob_size; //process transactions - if(!add_transaction_from_block(bl.miner_tx, get_transaction_hash(bl.miner_tx), id, get_current_blockchain_height())) + if(!add_transaction_from_block(bl.miner_tx, coinbase_hash, id, get_current_blockchain_height(), coinbase_blob_size)) { LOG_PRINT_L1("Block with id: " << id << " failed to add transaction to blockchain storage"); bvc.m_verifivation_failed = true; @@ -1658,7 +1686,7 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt transaction tx; size_t blob_size = 0; uint64_t fee = 0; - if(!m_tx_pool.take_tx(tx_id, tx, blob_size, fee)) + if(!m_tx_pool->take_tx(tx_id, tx, blob_size, fee)) { LOG_PRINT_L1("Block with id: " << id << "has at least one unknown transaction with id: " << tx_id); purge_block_data_from_blockchain(bl, tx_processed_count); @@ -1670,7 +1698,7 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt { LOG_PRINT_L1("Block with id: " << id << "has at least one transaction (id: " << tx_id << ") with wrong inputs."); cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); - bool add_res = m_tx_pool.add_tx(tx, tvc, true); + bool add_res = m_tx_pool->add_tx(tx, tvc, true); CHECK_AND_ASSERT_MES2(add_res, "handle_block_to_main_chain: failed to add transaction back to transaction pool"); purge_block_data_from_blockchain(bl, tx_processed_count); add_block_as_invalid(bl, id); @@ -1679,11 +1707,11 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt return false; } - if(!add_transaction_from_block(tx, tx_id, id, get_current_blockchain_height())) + if(!add_transaction_from_block(tx, tx_id, id, get_current_blockchain_height(), blob_size)) { LOG_PRINT_L1("Block with id: " << id << " failed to add transaction to blockchain storage"); cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); - bool add_res = m_tx_pool.add_tx(tx, tvc, true); + bool add_res = m_tx_pool->add_tx(tx, tvc, true); CHECK_AND_ASSERT_MES2(add_res, "handle_block_to_main_chain: failed to add transaction back to transaction pool"); purge_block_data_from_blockchain(bl, tx_processed_count); bvc.m_verifivation_failed = true; @@ -1709,7 +1737,14 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt bei.bl = bl; bei.block_cumulative_size = cumulative_block_size; bei.cumulative_difficulty = current_diffic; - bei.already_generated_coins = already_generated_coins + base_reward; + + // In the "tail" state when the minimum subsidy (implemented in get_block_reward) is in effect, the number of + // coins will eventually exceed MONEY_SUPPLY and overflow a uint64. To prevent overflow, cap already_generated_coins + // at MONEY_SUPPLY. already_generated_coins is only used to compute the block subsidy and MONEY_SUPPLY yields a + // subsidy of 0 under the base formula and therefore the minimum subsidy >0 in the tail state. + + bei.already_generated_coins = base_reward < (MONEY_SUPPLY-already_generated_coins) ? already_generated_coins + base_reward : MONEY_SUPPLY; + if(m_blocks.size()) bei.cumulative_difficulty += m_blocks.back().cumulative_difficulty; @@ -1734,11 +1769,13 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt << "), coinbase_blob_size: " << coinbase_blob_size << ", cumulative size: " << cumulative_block_size << ", " << block_processing_time << "("<< target_calculating_time << "/" << longhash_calculating_time << ")ms"); + epee::net_utils::data_logger::get_instance().add_data("blockchain_processing_time", block_processing_time); + bvc.m_added_to_main_chain = true; /*if(!m_orphanes_reorganize_in_work) review_orphaned_blocks_with_new_block_id(id, true);*/ - m_tx_pool.on_blockchain_inc(bei.height, id); + m_tx_pool->on_blockchain_inc(bei.height, id); //LOG_PRINT_L0("BLOCK: " << ENDL << "" << dump_obj_as_json(bei.bl)); return true; } @@ -1761,7 +1798,7 @@ bool blockchain_storage::add_new_block(const block& bl_, block_verification_cont //copy block here to let modify block.target block bl = bl_; crypto::hash id = get_block_hash(bl); - CRITICAL_REGION_LOCAL(m_tx_pool);//to avoid deadlock lets lock tx_pool for whole add/reorganize process + CRITICAL_REGION_LOCAL(*m_tx_pool);//to avoid deadlock lets lock tx_pool for whole add/reorganize process CRITICAL_REGION_LOCAL1(m_blockchain_lock); if(have_block(id)) { @@ -1782,7 +1819,7 @@ bool blockchain_storage::add_new_block(const block& bl_, block_verification_cont return handle_block_to_main_chain(bl, id, bvc); } //------------------------------------------------------------------ -void blockchain_storage::check_against_checkpoints(checkpoints& points, bool enforce) +void blockchain_storage::check_against_checkpoints(const checkpoints& points, bool enforce) { const auto& pts = points.get_points(); @@ -1833,7 +1870,7 @@ bool blockchain_storage::update_checkpoints(const std::string& file_path, bool c else if (check_dns) { checkpoints dns_points; - cryptonote::load_checkpoints_from_dns(dns_points); + cryptonote::load_checkpoints_from_dns(dns_points, m_testnet); if (m_checkpoints.check_for_conflicts(dns_points)) { check_against_checkpoints(dns_points, false); diff --git a/src/cryptonote_core/blockchain_storage.h b/src/cryptonote_core/blockchain_storage.h index 08c007a4a..50a62a428 100644 --- a/src/cryptonote_core/blockchain_storage.h +++ b/src/cryptonote_core/blockchain_storage.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -78,7 +78,7 @@ namespace cryptonote uint64_t already_generated_coins; }; - blockchain_storage(tx_memory_pool& tx_pool):m_tx_pool(tx_pool), m_current_block_cumul_sz_limit(0), m_is_in_checkpoint_zone(false), m_is_blockchain_storing(false), m_enforce_dns_checkpoints(false) + blockchain_storage(tx_memory_pool* tx_pool):m_tx_pool(tx_pool), m_current_block_cumul_sz_limit(0), m_is_in_checkpoint_zone(false), m_is_blockchain_storing(false), m_enforce_dns_checkpoints(false) {}; bool init() { return init(tools::get_default_data_dir(), true); } @@ -88,55 +88,56 @@ namespace cryptonote void set_checkpoints(checkpoints&& chk_pts) { m_checkpoints = chk_pts; } //bool push_new_block(); - bool get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks, std::list<transaction>& txs); - bool get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks); - bool get_alternative_blocks(std::list<block>& blocks); - size_t get_alternative_blocks_count(); - crypto::hash get_block_id_by_height(uint64_t height); - bool get_block_by_hash(const crypto::hash &h, block &blk); - void get_all_known_block_ids(std::list<crypto::hash> &main, std::list<crypto::hash> &alt, std::list<crypto::hash> &invalid); + bool get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks, std::list<transaction>& txs) const; + bool get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks) const; + bool get_alternative_blocks(std::list<block>& blocks) const; + size_t get_alternative_blocks_count() const; + crypto::hash get_block_id_by_height(uint64_t height) const; + bool get_block_by_hash(const crypto::hash &h, block &blk) const; + void get_all_known_block_ids(std::list<crypto::hash> &main, std::list<crypto::hash> &alt, std::list<crypto::hash> &invalid) const; template<class archive_t> void serialize(archive_t & ar, const unsigned int version); - bool have_tx(const crypto::hash &id); - bool have_tx_keyimges_as_spent(const transaction &tx); - bool have_tx_keyimg_as_spent(const crypto::key_image &key_im); - transaction *get_tx(const crypto::hash &id); + bool have_tx(const crypto::hash &id) const; + bool have_tx_keyimges_as_spent(const transaction &tx) const; + bool have_tx_keyimg_as_spent(const crypto::key_image &key_im) const; + const transaction *get_tx(const crypto::hash &id) const; template<class visitor_t> - bool scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, visitor_t& vis, uint64_t* pmax_related_block_height = NULL); + bool scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, visitor_t& vis, uint64_t* pmax_related_block_height = NULL) const; - uint64_t get_current_blockchain_height(); - crypto::hash get_tail_id(); - crypto::hash get_tail_id(uint64_t& height); - difficulty_type get_difficulty_for_next_block(); + uint64_t get_current_blockchain_height() const; + crypto::hash get_tail_id() const; + crypto::hash get_tail_id(uint64_t& height) const; + difficulty_type get_difficulty_for_next_block() const; bool add_new_block(const block& bl_, block_verification_context& bvc); bool reset_and_set_genesis_block(const block& b); - bool create_block_template(block& b, const account_public_address& miner_address, difficulty_type& di, uint64_t& height, const blobdata& ex_nonce); - bool have_block(const crypto::hash& id); - size_t get_total_transactions(); - bool get_outs(uint64_t amount, std::list<crypto::public_key>& pkeys); - bool get_short_chain_history(std::list<crypto::hash>& ids); - bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp); - bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, uint64_t& starter_offset); - bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<block, std::list<transaction> > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count); + bool create_block_template(block& b, const account_public_address& miner_address, difficulty_type& di, uint64_t& height, const blobdata& ex_nonce) const; + bool have_block(const crypto::hash& id) const; + size_t get_total_transactions() const; + bool get_outs(uint64_t amount, std::list<crypto::public_key>& pkeys) const; + bool get_short_chain_history(std::list<crypto::hash>& ids) const; + bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const; + bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, uint64_t& starter_offset) const; + bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<block, std::list<transaction> > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count) const; bool handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NOTIFY_RESPONSE_GET_OBJECTS::request& rsp); bool handle_get_objects(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res); - bool get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res); - bool get_backward_blocks_sizes(size_t from_height, std::vector<size_t>& sz, size_t count); - bool get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs); + bool get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const; + bool get_backward_blocks_sizes(size_t from_height, std::vector<size_t>& sz, size_t count) const; + bool get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) const; bool store_blockchain(); - bool check_tx_input(const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector<crypto::signature>& sig, uint64_t* pmax_related_block_height = NULL); - bool check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash, uint64_t* pmax_used_block_height = NULL); - bool check_tx_inputs(const transaction& tx, uint64_t* pmax_used_block_height = NULL); - bool check_tx_inputs(const transaction& tx, uint64_t& pmax_used_block_height, crypto::hash& max_used_block_id); - uint64_t get_current_comulative_blocksize_limit(); - bool is_storing_blockchain(){return m_is_blockchain_storing;} - uint64_t block_difficulty(size_t i); + bool check_tx_input(const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector<crypto::signature>& sig, uint64_t* pmax_related_block_height = NULL) const; + bool check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash, uint64_t* pmax_used_block_height = NULL) const; + bool check_tx_inputs(const transaction& tx, uint64_t* pmax_used_block_height = NULL) const; + bool check_tx_inputs(const transaction& tx, uint64_t& pmax_used_block_height, crypto::hash& max_used_block_id) const; + uint64_t get_current_cumulative_blocksize_limit() const; + bool is_storing_blockchain()const{return m_is_blockchain_storing;} + uint64_t block_difficulty(size_t i) const; + double get_avg_block_size( size_t count) const; template<class t_ids_container, class t_blocks_container, class t_missed_container> - bool get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs) + bool get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -156,7 +157,7 @@ namespace cryptonote } template<class t_ids_container, class t_tx_container, class t_missed_container> - bool get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) + bool get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -166,7 +167,7 @@ namespace cryptonote if(it == m_transactions.end()) { transaction tx; - if(!m_tx_pool.get_transaction(tx_id, tx)) + if(!m_tx_pool->get_transaction(tx_id, tx)) missed_txs.push_back(tx_id); else txs.push_back(tx); @@ -180,10 +181,15 @@ namespace cryptonote void print_blockchain(uint64_t start_index, uint64_t end_index); void print_blockchain_index(); void print_blockchain_outs(const std::string& file); - void check_against_checkpoints(checkpoints& points, bool enforce); + void check_against_checkpoints(const checkpoints& points, bool enforce); bool update_checkpoints(const std::string& file_path, bool check_dns); void set_enforce_dns_checkpoints(bool enforce_checkpoints); + block get_block(uint64_t height) const { return m_blocks[height].bl; } + size_t get_block_size(uint64_t height) const { return m_blocks[height].block_cumulative_size; } + difficulty_type get_block_cumulative_difficulty(uint64_t height) const { return m_blocks[height].cumulative_difficulty; } + uint64_t get_block_coins_generated(uint64_t height) const { return m_blocks[height].already_generated_coins; } + private: typedef std::unordered_map<crypto::hash, size_t> blocks_by_id_index; typedef std::unordered_map<crypto::hash, transaction_chain_entry> transactions_container; @@ -193,8 +199,8 @@ namespace cryptonote typedef std::unordered_map<crypto::hash, block> blocks_by_hash; typedef std::map<uint64_t, std::vector<std::pair<crypto::hash, size_t>>> outputs_container; //crypto::hash - tx hash, size_t - index of out in transaction - tx_memory_pool& m_tx_pool; - epee::critical_section m_blockchain_lock; // TODO: add here reader/writer lock + tx_memory_pool* m_tx_pool; + mutable epee::critical_section m_blockchain_lock; // TODO: add here reader/writer lock // main chain blocks_container m_blocks; // height -> block_extended_info @@ -218,6 +224,7 @@ namespace cryptonote std::atomic<bool> m_is_blockchain_storing; bool m_enforce_dns_checkpoints; + bool m_testnet; bool switch_to_alternative_blockchain(std::list<blocks_ext_by_hash::iterator>& alt_chain, bool discard_disconnected_chain); bool pop_block_from_blockchain(); @@ -228,26 +235,26 @@ namespace cryptonote bool handle_block_to_main_chain(const block& bl, block_verification_context& bvc); bool handle_block_to_main_chain(const block& bl, const crypto::hash& id, block_verification_context& bvc); bool handle_alternative_block(const block& b, const crypto::hash& id, block_verification_context& bvc); - difficulty_type get_next_difficulty_for_alternative_chain(const std::list<blocks_ext_by_hash::iterator>& alt_chain, block_extended_info& bei); - bool prevalidate_miner_transaction(const block& b, uint64_t height); - bool validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins); - bool validate_transaction(const block& b, uint64_t height, const transaction& tx); + difficulty_type get_next_difficulty_for_alternative_chain(const std::list<blocks_ext_by_hash::iterator>& alt_chain, block_extended_info& bei) const; + bool prevalidate_miner_transaction(const block& b, uint64_t height) const; + bool validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins) const; + bool validate_transaction(const block& b, uint64_t height, const transaction& tx) const; bool rollback_blockchain_switching(std::list<block>& original_chain, size_t rollback_height); - bool add_transaction_from_block(const transaction& tx, const crypto::hash& tx_id, const crypto::hash& bl_id, uint64_t bl_height); + bool add_transaction_from_block(const transaction& tx, const crypto::hash& tx_id, const crypto::hash& bl_id, uint64_t bl_height, size_t blob_size); bool push_transaction_to_global_outs_index(const transaction& tx, const crypto::hash& tx_id, std::vector<uint64_t>& global_indexes); bool pop_transaction_from_global_index(const transaction& tx, const crypto::hash& tx_id); - bool get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t count); - bool add_out_to_get_random_outs(std::vector<std::pair<crypto::hash, size_t> >& amount_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i); - bool is_tx_spendtime_unlocked(uint64_t unlock_time); + bool get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t count) const; + bool add_out_to_get_random_outs(const std::vector<std::pair<crypto::hash, size_t> >& amount_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) const; + bool is_tx_spendtime_unlocked(uint64_t unlock_time) const; bool add_block_as_invalid(const block& bl, const crypto::hash& h); bool add_block_as_invalid(const block_extended_info& bei, const crypto::hash& h); - size_t find_end_of_allowed_index(const std::vector<std::pair<crypto::hash, size_t> >& amount_outs); - bool check_block_timestamp_main(const block& b); - bool check_block_timestamp(std::vector<uint64_t> timestamps, const block& b); - uint64_t get_adjusted_time(); - bool complete_timestamps_vector(uint64_t start_height, std::vector<uint64_t>& timestamps); + size_t find_end_of_allowed_index(const std::vector<std::pair<crypto::hash, size_t> >& amount_outs) const; + bool check_block_timestamp_main(const block& b) const; + bool check_block_timestamp(std::vector<uint64_t> timestamps, const block& b) const; + uint64_t get_adjusted_time() const; + bool complete_timestamps_vector(uint64_t start_height, std::vector<uint64_t>& timestamps) const; bool update_next_comulative_size_limit(); - bool store_genesis_block(bool testnet); + bool store_genesis_block(bool testnet, bool check_added = false); }; @@ -315,7 +322,7 @@ namespace cryptonote //------------------------------------------------------------------ template<class visitor_t> - bool blockchain_storage::scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, visitor_t& vis, uint64_t* pmax_related_block_height) + bool blockchain_storage::scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, visitor_t& vis, uint64_t* pmax_related_block_height) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); auto it = m_outputs.find(tx_in_to_key.amount); @@ -325,7 +332,7 @@ namespace cryptonote std::vector<uint64_t> absolute_offsets = relative_output_offsets_to_absolute(tx_in_to_key.key_offsets); - std::vector<std::pair<crypto::hash, size_t> >& amount_outs_vec = it->second; + const std::vector<std::pair<crypto::hash, size_t> >& amount_outs_vec = it->second; size_t count = 0; BOOST_FOREACH(uint64_t i, absolute_offsets) { @@ -334,7 +341,7 @@ namespace cryptonote LOG_PRINT_L0("Wrong index in transaction inputs: " << i << ", expected maximum " << amount_outs_vec.size() - 1); return false; } - transactions_container::iterator tx_it = m_transactions.find(amount_outs_vec[i].first); + transactions_container::const_iterator tx_it = m_transactions.find(amount_outs_vec[i].first); CHECK_AND_ASSERT_MES(tx_it != m_transactions.end(), false, "Wrong transaction id in output indexes: " << epee::string_tools::pod_to_hex(amount_outs_vec[i].first)); CHECK_AND_ASSERT_MES(amount_outs_vec[i].second < tx_it->second.tx.vout.size(), false, "Wrong index in transaction outputs: " << amount_outs_vec[i].second << ", expected less then " << tx_it->second.tx.vout.size()); diff --git a/src/cryptonote_core/blockchain_storage_boost_serialization.h b/src/cryptonote_core/blockchain_storage_boost_serialization.h index c64ce8e0c..28c06ece1 100644 --- a/src/cryptonote_core/blockchain_storage_boost_serialization.h +++ b/src/cryptonote_core/blockchain_storage_boost_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/checkpoints.cpp b/src/cryptonote_core/checkpoints.cpp index 6a86302b6..e4223afb5 100644 --- a/src/cryptonote_core/checkpoints.cpp +++ b/src/cryptonote_core/checkpoints.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -84,6 +84,10 @@ namespace cryptonote return check_block(height, h, ignored); } //--------------------------------------------------------------------------- + // this basically says if the blockchain is smaller than the first + // checkpoint then alternate blocks are allowed. Alternatively, if the + // last checkpoint *before* the end of the current chain is also before + // the block to be added, then this is fine. bool checkpoints::is_alternative_block_allowed(uint64_t blockchain_height, uint64_t block_height) const { if (0 == block_height) @@ -99,7 +103,7 @@ namespace cryptonote return checkpoint_height < block_height; } //--------------------------------------------------------------------------- - uint64_t checkpoints::get_max_height() + uint64_t checkpoints::get_max_height() const { std::map< uint64_t, crypto::hash >::const_iterator highest = std::max_element( m_points.begin(), m_points.end(), @@ -108,18 +112,18 @@ namespace cryptonote return highest->first; } //--------------------------------------------------------------------------- - const std::map<uint64_t, crypto::hash>& checkpoints::get_points() + const std::map<uint64_t, crypto::hash>& checkpoints::get_points() const { return m_points; } - bool checkpoints::check_for_conflicts(checkpoints& other) + bool checkpoints::check_for_conflicts(const checkpoints& other) const { for (auto& pt : other.get_points()) { if (m_points.count(pt.first)) { - CHECK_AND_ASSERT_MES(pt.second == m_points[pt.first], false, "Checkpoint at given height already exists, and hash for new checkpoint was different!"); + CHECK_AND_ASSERT_MES(pt.second == m_points.at(pt.first), false, "Checkpoint at given height already exists, and hash for new checkpoint was different!"); } } return true; diff --git a/src/cryptonote_core/checkpoints.h b/src/cryptonote_core/checkpoints.h index 132917228..55d765b71 100644 --- a/src/cryptonote_core/checkpoints.h +++ b/src/cryptonote_core/checkpoints.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -45,9 +45,9 @@ namespace cryptonote bool check_block(uint64_t height, const crypto::hash& h) const; bool check_block(uint64_t height, const crypto::hash& h, bool& is_a_checkpoint) const; bool is_alternative_block_allowed(uint64_t blockchain_height, uint64_t block_height) const; - uint64_t get_max_height(); - const std::map<uint64_t, crypto::hash>& get_points(); - bool check_for_conflicts(checkpoints& other); + uint64_t get_max_height() const; + const std::map<uint64_t, crypto::hash>& get_points() const; + bool check_for_conflicts(const checkpoints& other) const; private: std::map<uint64_t, crypto::hash> m_points; }; diff --git a/src/cryptonote_core/checkpoints_create.cpp b/src/cryptonote_core/checkpoints_create.cpp index 9ad0a27e8..6f22e596d 100644 --- a/src/cryptonote_core/checkpoints_create.cpp +++ b/src/cryptonote_core/checkpoints_create.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -35,6 +35,30 @@ #include <random> #include "storages/portable_storage_template_helper.h" // epee json include +namespace +{ + bool dns_records_match(const std::vector<std::string>& a, const std::vector<std::string>& b) + { + if (a.size() != b.size()) return false; + + for (const auto& record_in_a : a) + { + bool ok = false; + for (const auto& record_in_b : b) + { + if (record_in_a == record_in_b) + { + ok = true; + break; + } + } + if (!ok) return false; + } + + return true; + } +} // anonymous namespace + namespace cryptonote { @@ -76,6 +100,7 @@ bool create_checkpoints(cryptonote::checkpoints& checkpoints) ADD_CHECKPOINT(231350, "b5add137199b820e1ea26640e5c3e121fd85faa86a1e39cf7e6cc097bdeb1131"); ADD_CHECKPOINT(232150, "955de8e6b6508af2c24f7334f97beeea651d78e9ade3ab18fec3763be3201aa8"); ADD_CHECKPOINT(249380, "654fb0a81ce3e5caf7e3264a70f447d4bd07586c08fa50f6638cc54da0a52b2d"); + ADD_CHECKPOINT(460000, "75037a7aed3e765db96c75bcf908f59d690a5f3390baebb9edeafd336a1c4831"); return true; } @@ -112,7 +137,7 @@ bool load_checkpoints_from_json(cryptonote::checkpoints& checkpoints, std::strin return true; } -bool load_checkpoints_from_dns(cryptonote::checkpoints& checkpoints) +bool load_checkpoints_from_dns(cryptonote::checkpoints& checkpoints, bool testnet) { // All four MoneroPulse domains have DNSSEC on and valid static const std::vector<std::string> dns_urls = { "checkpoints.moneropulse.se" @@ -120,43 +145,97 @@ bool load_checkpoints_from_dns(cryptonote::checkpoints& checkpoints) , "checkpoints.moneropulse.net" , "checkpoints.moneropulse.co" }; - bool avail, valid; - std::vector<std::string> records; + + static const std::vector<std::string> testnet_dns_urls = { "testpoints.moneropulse.se" + , "testpoints.moneropulse.org" + , "testpoints.moneropulse.net" + , "testpoints.moneropulse.co" + }; + + std::vector<std::vector<std::string> > records; + records.resize(dns_urls.size()); std::random_device rd; std::mt19937 gen(rd()); std::uniform_int_distribution<int> dis(0, dns_urls.size() - 1); size_t first_index = dis(gen); + bool avail, valid; size_t cur_index = first_index; do { - records = tools::DNSResolver::instance().get_txt_record(dns_urls[cur_index], avail, valid); - if (records.size() == 0 || (avail && !valid)) + std::string url; + if (testnet) + { + url = testnet_dns_urls[cur_index]; + } + else + { + url = dns_urls[cur_index]; + } + + records[cur_index] = tools::DNSResolver::instance().get_txt_record(url, avail, valid); + if (!avail) + { + LOG_PRINT_L2("DNSSEC not available for checkpoint update at URL: " << url << ", skipping."); + } + if (!valid) + { + LOG_PRINT_L2("DNSSEC validation failed for checkpoint update at URL: " << url << ", skipping."); + } + + if (records[cur_index].size() == 0 || !avail || !valid) { cur_index++; if (cur_index == dns_urls.size()) { cur_index = 0; } + records[cur_index].clear(); continue; } break; } while (cur_index != first_index); - if (records.size() == 0) + size_t num_valid_records = 0; + + for( const auto& record_set : records) + { + if (record_set.size() != 0) + { + num_valid_records++; + } + } + + if (num_valid_records < 2) { - LOG_PRINT_L1("Fetching MoneroPulse checkpoints failed, no TXT records available."); + LOG_PRINT_L0("WARNING: no two valid MoneroPulse DNS checkpoint records were received"); return true; } - if (avail && !valid) + int good_records_index = -1; + for (size_t i = 0; i < records.size() - 1; ++i) + { + if (records[i].size() == 0) continue; + + for (size_t j = i + 1; j < records.size(); ++j) + { + if (dns_records_match(records[i], records[j])) + { + good_records_index = i; + break; + } + } + if (good_records_index >= 0) break; + } + + if (good_records_index < 0) { - LOG_PRINT_L0("WARNING: MoneroPulse failed DNSSEC validation and/or returned no records"); + LOG_PRINT_L0("WARNING: no two MoneroPulse DNS checkpoint records matched"); return true; } - for (auto& record : records) + for (auto& record : records[good_records_index]) { auto pos = record.find(":"); if (pos != std::string::npos) diff --git a/src/cryptonote_core/checkpoints_create.h b/src/cryptonote_core/checkpoints_create.h index 92a970e9f..8422e2b33 100644 --- a/src/cryptonote_core/checkpoints_create.h +++ b/src/cryptonote_core/checkpoints_create.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -42,7 +42,7 @@ namespace cryptonote bool create_checkpoints(cryptonote::checkpoints& checkpoints); bool load_checkpoints_from_json(cryptonote::checkpoints& checkpoints, std::string json_hashfile_fullpath); - bool load_checkpoints_from_dns(cryptonote::checkpoints& checkpoints); + bool load_checkpoints_from_dns(cryptonote::checkpoints& checkpoints, bool testnet = false); bool load_new_checkpoints(cryptonote::checkpoints& checkpoints, std::string json_hashfile_fullpath); } // namespace cryptonote diff --git a/src/cryptonote_core/connection_context.h b/src/cryptonote_core/connection_context.h index 6574e92e6..7d40bc2e9 100644 --- a/src/cryptonote_core/connection_context.h +++ b/src/cryptonote_core/connection_context.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/cryptonote_basic.h b/src/cryptonote_core/cryptonote_basic.h index dfdfa03e5..2be76c0de 100644 --- a/src/cryptonote_core/cryptonote_basic.h +++ b/src/cryptonote_core/cryptonote_basic.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -50,6 +50,8 @@ #include "misc_language.h" #include "tx_extra.h" +#define DB_MEMORY 1 +#define DB_LMDB 2 namespace cryptonote { diff --git a/src/cryptonote_core/cryptonote_basic_impl.cpp b/src/cryptonote_core/cryptonote_basic_impl.cpp index 9cec666f2..f2d862e57 100644 --- a/src/cryptonote_core/cryptonote_basic_impl.cpp +++ b/src/cryptonote_core/cryptonote_basic_impl.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -60,6 +60,10 @@ namespace cryptonote { //----------------------------------------------------------------------------------------------- bool get_block_reward(size_t median_size, size_t current_block_size, uint64_t already_generated_coins, uint64_t &reward) { uint64_t base_reward = (MONEY_SUPPLY - already_generated_coins) >> EMISSION_SPEED_FACTOR; + if (base_reward < FINAL_SUBSIDY_PER_MINUTE) + { + base_reward = FINAL_SUBSIDY_PER_MINUTE; + } //make it soft if (median_size < CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE) { diff --git a/src/cryptonote_core/cryptonote_basic_impl.h b/src/cryptonote_core/cryptonote_basic_impl.h index ddef677cb..5c773c4e1 100644 --- a/src/cryptonote_core/cryptonote_basic_impl.h +++ b/src/cryptonote_core/cryptonote_basic_impl.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/cryptonote_boost_serialization.h b/src/cryptonote_core/cryptonote_boost_serialization.h index e3ccb7c32..17face4bd 100644 --- a/src/cryptonote_core/cryptonote_boost_serialization.h +++ b/src/cryptonote_core/cryptonote_boost_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index d9d8a7faf..a5517e011 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -42,6 +42,13 @@ using namespace epee; #include "cryptonote_format_utils.h" #include "misc_language.h" #include <csignal> +#include "daemon/command_line_args.h" +#include "cryptonote_core/checkpoints_create.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/lmdb/db_lmdb.h" +#ifndef STATICLIB +#include "blockchain_db/berkeleydb/db_bdb.h" +#endif DISABLE_VS_WARNINGS(4355) @@ -51,7 +58,11 @@ namespace cryptonote //----------------------------------------------------------------------------------------------- core::core(i_cryptonote_protocol* pprotocol): m_mempool(m_blockchain_storage), +#if BLOCKCHAIN_DB == DB_LMDB m_blockchain_storage(m_mempool), +#else + m_blockchain_storage(&m_mempool), +#endif m_miner(this), m_miner_address(boost::value_initialized<account_public_address>()), m_starter_message_showed(false), @@ -87,6 +98,10 @@ namespace cryptonote //----------------------------------------------------------------------------------------------- bool core::update_checkpoints() { + if (m_testnet) return true; + + if (m_checkpoints_updating.test_and_set()) return true; + bool res = true; if (time(NULL) - m_last_dns_checkpoints_update >= 3600) { @@ -100,6 +115,8 @@ namespace cryptonote m_last_json_checkpoints_update = time(NULL); } + m_checkpoints_updating.clear(); + // if anything fishy happened getting new checkpoints, bring down the house if (!res) { @@ -108,14 +125,46 @@ namespace cryptonote return res; } //----------------------------------------------------------------------------------- + void core::stop() + { + graceful_exit(); + } + //----------------------------------------------------------------------------------- void core::init_options(boost::program_options::options_description& /*desc*/) { } //----------------------------------------------------------------------------------------------- - bool core::handle_command_line(const boost::program_options::variables_map& vm, bool testnet) + bool core::handle_command_line(const boost::program_options::variables_map& vm) { - auto data_dir_arg = testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir; + m_testnet = command_line::get_arg(vm, daemon_args::arg_testnet_on); + + auto data_dir_arg = m_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir; m_config_folder = command_line::get_arg(vm, data_dir_arg); + + auto data_dir = boost::filesystem::path(m_config_folder); + + if (!m_testnet) + { + cryptonote::checkpoints checkpoints; + if (!cryptonote::create_checkpoints(checkpoints)) + { + throw std::runtime_error("Failed to initialize checkpoints"); + } + set_checkpoints(std::move(checkpoints)); + + boost::filesystem::path json(JSON_HASH_FILE_NAME); + boost::filesystem::path checkpoint_json_hashfile_fullpath = data_dir / json; + + set_checkpoints_file_path(checkpoint_json_hashfile_fullpath.string()); + } + + + set_enforce_dns_checkpoints(command_line::get_arg(vm, daemon_args::arg_dns_checkpoints)); + test_drop_download_height(command_line::get_arg(vm, command_line::arg_test_drop_download_height)); + + if (command_line::get_arg(vm, command_line::arg_test_drop_download) == true) + test_drop_download(); + return true; } //----------------------------------------------------------------------------------------------- @@ -154,21 +203,64 @@ namespace cryptonote return m_blockchain_storage.get_alternative_blocks_count(); } //----------------------------------------------------------------------------------------------- - bool core::init(const boost::program_options::variables_map& vm, bool testnet) + bool core::init(const boost::program_options::variables_map& vm) { - bool r = handle_command_line(vm, testnet); + bool r = handle_command_line(vm); r = m_mempool.init(m_config_folder); CHECK_AND_ASSERT_MES(r, false, "Failed to initialize memory pool"); - r = m_blockchain_storage.init(m_config_folder, testnet); +#if BLOCKCHAIN_DB == DB_LMDB + std::string db_type = command_line::get_arg(vm, daemon_args::arg_db_type); + + BlockchainDB* db = nullptr; + if (db_type == "lmdb") + { + db = new BlockchainLMDB(); + } + else if (db_type == "berkeley") + { +#ifndef STATICLIB + db = new BlockchainBDB(); +#else + LOG_ERROR("BlockchainBDB not supported on STATIC builds"); + return false; +#endif + } + else + { + LOG_ERROR("Attempted to use non-existant database type"); + return false; + } + + boost::filesystem::path folder(m_config_folder); + + folder /= db->get_db_name(); + + LOG_PRINT_L0("Loading blockchain from folder " << folder.c_str() << " ..."); + + const std::string filename = folder.string(); + try + { + db->open(filename); + } + catch (const DB_ERROR& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + return false; + } + + r = m_blockchain_storage.init(db, m_testnet); +#else + r = m_blockchain_storage.init(m_config_folder, m_testnet); +#endif CHECK_AND_ASSERT_MES(r, false, "Failed to initialize blockchain storage"); // load json & DNS checkpoints, and verify them // with respect to what blocks we already have CHECK_AND_ASSERT_MES(update_checkpoints(), false, "One or more checkpoints loaded from json or dns conflicted with existing checkpoints."); - r = m_miner.init(vm, testnet); + r = m_miner.init(vm, m_testnet); CHECK_AND_ASSERT_MES(r, false, "Failed to initialize blockchain storage"); return load_state_data(); @@ -187,12 +279,51 @@ namespace cryptonote //----------------------------------------------------------------------------------------------- bool core::deinit() { - m_miner.stop(); - m_mempool.deinit(); - m_blockchain_storage.deinit(); + m_miner.stop(); + m_mempool.deinit(); + if (!m_fast_exit) + { + m_blockchain_storage.deinit(); + } return true; } //----------------------------------------------------------------------------------------------- + void core::set_fast_exit() + { + m_fast_exit = true; + } + //----------------------------------------------------------------------------------------------- + bool core::get_fast_exit() + { + return m_fast_exit; + } + //----------------------------------------------------------------------------------------------- + void core::test_drop_download() + { + m_test_drop_download = false; + } + //----------------------------------------------------------------------------------------------- + void core::test_drop_download_height(uint64_t height) + { + m_test_drop_download_height = height; + } + //----------------------------------------------------------------------------------------------- + bool core::get_test_drop_download() + { + return m_test_drop_download; + } + //----------------------------------------------------------------------------------------------- + bool core::get_test_drop_download_height() + { + if (m_test_drop_download_height == 0) + return true; + + if (get_blockchain_storage().get_current_blockchain_height() <= m_test_drop_download_height) + return true; + + return false; + } + //----------------------------------------------------------------------------------------------- bool core::handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool keeped_by_block) { tvc = boost::value_initialized<tx_verification_context>(); @@ -290,9 +421,9 @@ namespace cryptonote return false; } - if(!keeped_by_block && get_object_blobsize(tx) >= m_blockchain_storage.get_current_comulative_blocksize_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE) + if(!keeped_by_block && get_object_blobsize(tx) >= m_blockchain_storage.get_current_cumulative_blocksize_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE) { - LOG_PRINT_RED_L1("tx is too large " << get_object_blobsize(tx) << ", expected not bigger than " << m_blockchain_storage.get_current_comulative_blocksize_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE); + LOG_PRINT_RED_L1("tx is too large " << get_object_blobsize(tx) << ", expected not bigger than " << m_blockchain_storage.get_current_cumulative_blocksize_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE); return false; } @@ -527,6 +658,11 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- + bool core::get_pool_transactions_and_spent_keys_info(std::vector<tx_info>& tx_infos, std::vector<spent_key_image_info>& key_image_infos) const + { + return m_mempool.get_transactions_and_spent_keys_info(tx_infos, key_image_infos); + } + //----------------------------------------------------------------------------------------------- bool core::get_short_chain_history(std::list<crypto::hash>& ids) { return m_blockchain_storage.get_short_chain_history(ids); @@ -577,7 +713,11 @@ namespace cryptonote m_starter_message_showed = true; } +#if BLOCKCHAIN_DB == DB_LMDB + m_store_blockchain_interval.do_call(boost::bind(&Blockchain::store_blockchain, &m_blockchain_storage)); +#else m_store_blockchain_interval.do_call(boost::bind(&blockchain_storage::store_blockchain, &m_blockchain_storage)); +#endif m_miner.on_idle(); m_mempool.on_idle(); return true; @@ -595,4 +735,6 @@ namespace cryptonote { raise(SIGTERM); } + + std::atomic<bool> core::m_fast_exit(false); } diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 10522451c..ff187b4ce 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -39,7 +39,11 @@ #include "cryptonote_protocol/cryptonote_protocol_handler_common.h" #include "storages/portable_storage_template_helper.h" #include "tx_pool.h" +#if BLOCKCHAIN_DB == DB_LMDB +#include "blockchain.h" +#else #include "blockchain_storage.h" +#endif #include "miner.h" #include "connection_context.h" #include "cryptonote_core/cryptonote_stat_info.h" @@ -72,9 +76,15 @@ namespace cryptonote miner& get_miner(){return m_miner;} static void init_options(boost::program_options::options_description& desc); - bool init(const boost::program_options::variables_map& vm, bool testnet); + bool init(const boost::program_options::variables_map& vm); bool set_genesis_block(const block& b); bool deinit(); + static void set_fast_exit(); + static bool get_fast_exit(); + void test_drop_download(); + void test_drop_download_height(uint64_t height); + bool get_test_drop_download(); + bool get_test_drop_download_height(); uint64_t get_current_blockchain_height(); bool get_blockchain_top(uint64_t& heeight, crypto::hash& top_id); bool get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks, std::list<transaction>& txs); @@ -98,6 +108,7 @@ namespace cryptonote void set_enforce_dns_checkpoints(bool enforce_dns); bool get_pool_transactions(std::list<transaction>& txs); + bool get_pool_transactions_and_spent_keys_info(std::vector<tx_info>& tx_infos, std::vector<spent_key_image_info>& key_image_infos) const; size_t get_pool_transactions_count(); size_t get_blockchain_total_transactions(); //bool get_outs(uint64_t amount, std::list<crypto::public_key>& pkeys); @@ -112,7 +123,11 @@ namespace cryptonote bool get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res); void pause_mine(); void resume_mine(); +#if BLOCKCHAIN_DB == DB_LMDB + Blockchain& get_blockchain_storage(){return m_blockchain_storage;} +#else blockchain_storage& get_blockchain_storage(){return m_blockchain_storage;} +#endif //debug functions void print_blockchain(uint64_t start_index, uint64_t end_index); void print_blockchain_index(); @@ -125,6 +140,8 @@ namespace cryptonote bool update_checkpoints(); + void stop(); + private: bool add_new_tx(const transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block); bool add_new_tx(const transaction& tx, tx_verification_context& tvc, bool keeped_by_block); @@ -142,14 +159,20 @@ namespace cryptonote bool check_tx_ring_signature(const txin_to_key& tx, const crypto::hash& tx_prefix_hash, const std::vector<crypto::signature>& sig); bool is_tx_spendtime_unlocked(uint64_t unlock_time); bool update_miner_block_template(); - bool handle_command_line(const boost::program_options::variables_map& vm, bool testnet); + bool handle_command_line(const boost::program_options::variables_map& vm); bool on_update_blocktemplate_interval(); bool check_tx_inputs_keyimages_diff(const transaction& tx); void graceful_exit(); - + static std::atomic<bool> m_fast_exit; + bool m_test_drop_download = true; + uint64_t m_test_drop_download_height = 0; tx_memory_pool m_mempool; +#if BLOCKCHAIN_DB == DB_LMDB + Blockchain m_blockchain_storage; +#else blockchain_storage m_blockchain_storage; +#endif i_cryptonote_protocol* m_pprotocol; epee::critical_section m_incoming_tx_lock; //m_miner and m_miner_addres are probably temporary here @@ -163,9 +186,12 @@ namespace cryptonote uint64_t m_target_blockchain_height; + bool m_testnet; std::string m_checkpoints_path; time_t m_last_dns_checkpoints_update; time_t m_last_json_checkpoints_update; + + std::atomic_flag m_checkpoints_updating; }; } diff --git a/src/cryptonote_core/cryptonote_format_utils.cpp b/src/cryptonote_core/cryptonote_format_utils.cpp index 8c7b2fbaa..1b21894a0 100644 --- a/src/cryptonote_core/cryptonote_format_utils.cpp +++ b/src/cryptonote_core/cryptonote_format_utils.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/cryptonote_format_utils.h b/src/cryptonote_core/cryptonote_format_utils.h index 9fa7f0545..fd785aafd 100644 --- a/src/cryptonote_core/cryptonote_format_utils.h +++ b/src/cryptonote_core/cryptonote_format_utils.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/cryptonote_stat_info.h b/src/cryptonote_core/cryptonote_stat_info.h index 5d0d1769d..20bbe382e 100644 --- a/src/cryptonote_core/cryptonote_stat_info.h +++ b/src/cryptonote_core/cryptonote_stat_info.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/difficulty.cpp b/src/cryptonote_core/difficulty.cpp index d4d733e69..6486d8124 100644 --- a/src/cryptonote_core/difficulty.cpp +++ b/src/cryptonote_core/difficulty.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -56,10 +56,40 @@ namespace cryptonote { #else static inline void mul(uint64_t a, uint64_t b, uint64_t &low, uint64_t &high) { - typedef unsigned __int128 uint128_t; - uint128_t res = (uint128_t) a * (uint128_t) b; - low = (uint64_t) res; - high = (uint64_t) (res >> 64); + // __int128 isn't part of the standard, so the previous function wasn't portable. mul128() in Windows is fine, + // but this portable function should be used elsewhere. Credit for this function goes to latexi95. + + uint64_t aLow = a & 0xFFFFFFFF; + uint64_t aHigh = a >> 32; + uint64_t bLow = b & 0xFFFFFFFF; + uint64_t bHigh = b >> 32; + + uint64_t res = aLow * bLow; + uint64_t lowRes1 = res & 0xFFFFFFFF; + uint64_t carry = res >> 32; + + res = aHigh * bLow + carry; + uint64_t highResHigh1 = res >> 32; + uint64_t highResLow1 = res & 0xFFFFFFFF; + + res = aLow * bHigh; + uint64_t lowRes2 = res & 0xFFFFFFFF; + carry = res >> 32; + + res = aHigh * bHigh + carry; + uint64_t highResHigh2 = res >> 32; + uint64_t highResLow2 = res & 0xFFFFFFFF; + + //Addition + + uint64_t r = highResLow1 + lowRes2; + carry = r >> 32; + low = (r << 32) | lowRes1; + r = highResHigh1 + highResLow2 + carry; + uint64_t d3 = r & 0xFFFFFFFF; + carry = r >> 32; + r = highResHigh2 + carry; + high = d3 | (r << 32); } #endif diff --git a/src/cryptonote_core/difficulty.h b/src/cryptonote_core/difficulty.h index 99c9228f0..42d2df500 100644 --- a/src/cryptonote_core/difficulty.h +++ b/src/cryptonote_core/difficulty.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/miner.cpp b/src/cryptonote_core/miner.cpp index b2adfe39c..b99113d9e 100644 --- a/src/cryptonote_core/miner.cpp +++ b/src/cryptonote_core/miner.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/miner.h b/src/cryptonote_core/miner.h index 44844428c..7ada8171f 100644 --- a/src/cryptonote_core/miner.h +++ b/src/cryptonote_core/miner.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/tx_extra.h b/src/cryptonote_core/tx_extra.h index af91deca5..ccfe4d1d9 100644 --- a/src/cryptonote_core/tx_extra.h +++ b/src/cryptonote_core/tx_extra.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index 96c6ebed9..5543ac64f 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -37,7 +37,11 @@ #include "cryptonote_format_utils.h" #include "cryptonote_boost_serialization.h" #include "cryptonote_config.h" +#if BLOCKCHAIN_DB == DB_LMDB +#include "blockchain.h" +#else #include "blockchain_storage.h" +#endif #include "common/boost_serialization_helper.h" #include "common/int-util.h" #include "misc_language.h" @@ -52,12 +56,19 @@ namespace cryptonote { size_t const TRANSACTION_SIZE_LIMIT = (((CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE * 125) / 100) - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE); } - //--------------------------------------------------------------------------------- +#if BLOCKCHAIN_DB == DB_LMDB + //--------------------------------------------------------------------------------- + tx_memory_pool::tx_memory_pool(Blockchain& bchs): m_blockchain(bchs) + { + + } +#else tx_memory_pool::tx_memory_pool(blockchain_storage& bchs): m_blockchain(bchs) { } +#endif //--------------------------------------------------------------------------------- bool tx_memory_pool::add_tx(const transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, size_t blob_size, tx_verification_context& tvc, bool kept_by_block) { @@ -246,6 +257,7 @@ namespace cryptonote (tx_age > CRYPTONOTE_MEMPOOL_TX_FROM_ALT_BLOCK_LIVETIME && it->second.kept_by_block) ) { LOG_PRINT_L1("Tx " << it->first << " removed from tx pool due to outdated, age: " << tx_age ); + remove_transaction_keyimages(it->second.tx); m_transactions.erase(it++); }else ++it; @@ -265,6 +277,40 @@ namespace cryptonote BOOST_FOREACH(const auto& tx_vt, m_transactions) txs.push_back(tx_vt.second.tx); } + //------------------------------------------------------------------ + bool tx_memory_pool::get_transactions_and_spent_keys_info(std::vector<tx_info>& tx_infos, std::vector<spent_key_image_info>& key_image_infos) const + { + CRITICAL_REGION_LOCAL(m_transactions_lock); + for (const auto& tx_vt : m_transactions) + { + tx_info txi; + const tx_details& txd = tx_vt.second; + txi.id_hash = epee::string_tools::pod_to_hex(tx_vt.first); + txi.tx_json = obj_to_json_str(*const_cast<transaction*>(&txd.tx)); + txi.blob_size = txd.blob_size; + txi.fee = txd.fee; + txi.kept_by_block = txd.kept_by_block; + txi.max_used_block_height = txd.max_used_block_height; + txi.max_used_block_id_hash = epee::string_tools::pod_to_hex(txd.max_used_block_id); + txi.last_failed_height = txd.last_failed_height; + txi.last_failed_id_hash = epee::string_tools::pod_to_hex(txd.last_failed_id); + txi.receive_time = txd.receive_time; + tx_infos.push_back(txi); + } + + for (const key_images_container::value_type& kee : m_spent_key_images) { + const crypto::key_image& k_image = kee.first; + const std::unordered_set<crypto::hash>& kei_image_set = kee.second; + spent_key_image_info ki; + ki.id_hash = epee::string_tools::pod_to_hex(k_image); + for (const crypto::hash& tx_id_hash : kei_image_set) + { + ki.txs_hashes.push_back(epee::string_tools::pod_to_hex(tx_id_hash)); + } + key_image_infos.push_back(ki); + } + return true; + } //--------------------------------------------------------------------------------- bool tx_memory_pool::get_transaction(const crypto::hash& id, transaction& tx) const { diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h index 7617765cc..77f2e3483 100644 --- a/src/cryptonote_core/tx_pool.h +++ b/src/cryptonote_core/tx_pool.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -43,11 +43,15 @@ #include "cryptonote_basic_impl.h" #include "verification_context.h" #include "crypto/hash.h" - +#include "rpc/core_rpc_server_commands_defs.h" namespace cryptonote { +#if BLOCKCHAIN_DB == DB_LMDB + class Blockchain; +#else class blockchain_storage; +#endif /************************************************************************/ /* */ /************************************************************************/ @@ -55,7 +59,11 @@ namespace cryptonote class tx_memory_pool: boost::noncopyable { public: +#if BLOCKCHAIN_DB == DB_LMDB + tx_memory_pool(Blockchain& bchs); +#else tx_memory_pool(blockchain_storage& bchs); +#endif bool add_tx(const transaction &tx, const crypto::hash &id, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block); bool add_tx(const transaction &tx, tx_verification_context& tvc, bool keeped_by_block); //gets tx and remove it from pool @@ -74,6 +82,7 @@ namespace cryptonote bool deinit(); bool fill_block_template(block &bl, size_t median_size, uint64_t already_generated_coins, size_t &total_size, uint64_t &fee); void get_transactions(std::list<transaction>& txs) const; + bool get_transactions_and_spent_keys_info(std::vector<tx_info>& tx_infos, std::vector<spent_key_image_info>& key_image_infos) const; bool get_transaction(const crypto::hash& h, transaction& tx) const; size_t get_transactions_count() const; std::string print_pool(bool short_format) const; @@ -127,7 +136,11 @@ namespace cryptonote //transactions_container m_alternative_transactions; std::string m_config_folder; +#if BLOCKCHAIN_DB == DB_LMDB + Blockchain& m_blockchain; +#else blockchain_storage& m_blockchain; +#endif /************************************************************************/ /* */ /************************************************************************/ @@ -170,9 +183,12 @@ namespace cryptonote uint64_t operator()(const txin_to_scripthash& tx) const {return 0;} }; +#if BLOCKCHAIN_DB == DB_LMDB +#else #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) friend class blockchain_storage; #endif +#endif }; } diff --git a/src/cryptonote_core/verification_context.h b/src/cryptonote_core/verification_context.h index 3be45907b..c467d35f3 100644 --- a/src/cryptonote_core/verification_context.h +++ b/src/cryptonote_core/verification_context.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_protocol/CMakeLists.txt b/src/cryptonote_protocol/CMakeLists.txt new file mode 100644 index 000000000..2ea5662a1 --- /dev/null +++ b/src/cryptonote_protocol/CMakeLists.txt @@ -0,0 +1,46 @@ +# Copyright (c) 2014, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +cmake_minimum_required (VERSION 2.6) +project (bitmonero CXX) + +file(GLOB CRYPTONOTE_PROTOCOL *) +source_group(cryptonote_protocol FILES ${CRYPTONOTE_PROTOCOL}) + +#add_library(p2p ${P2P}) + +#bitmonero_private_headers(p2p ${CRYPTONOTE_PROTOCOL}) +bitmonero_add_library(cryptonote_protocol ${CRYPTONOTE_PROTOCOL}) +#target_link_libraries(p2p) +# LINK_PRIVATE +# ${Boost_CHRONO_LIBRARY} +# ${Boost_REGEX_LIBRARY} +# ${Boost_SYSTEM_LIBRARY} +# ${Boost_THREAD_LIBRARY} +# ${EXTRA_LIBRARIES}) +add_dependencies(cryptonote_protocol + version) diff --git a/src/cryptonote_protocol/blobdatatype.h b/src/cryptonote_protocol/blobdatatype.h index 7cde6bdb3..063f08ef0 100644 --- a/src/cryptonote_protocol/blobdatatype.h +++ b/src/cryptonote_protocol/blobdatatype.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/cryptonote_protocol/cryptonote_protocol_defs.h b/src/cryptonote_protocol/cryptonote_protocol_defs.h index 66fa43dd2..7e019b533 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_defs.h +++ b/src/cryptonote_protocol/cryptonote_protocol_defs.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -46,6 +46,8 @@ namespace cryptonote struct connection_info { bool incoming; + bool localhost; + bool local_ip; std::string ip; std::string port; @@ -62,8 +64,16 @@ namespace cryptonote uint64_t live_time; + uint64_t avg_download; + uint64_t current_download; + + uint64_t avg_upload; + uint64_t current_upload; + BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(incoming) + KV_SERIALIZE(localhost) + KV_SERIALIZE(local_ip) KV_SERIALIZE(ip) KV_SERIALIZE(port) KV_SERIALIZE(peer_id) @@ -73,6 +83,10 @@ namespace cryptonote KV_SERIALIZE(send_idle_time) KV_SERIALIZE(state) KV_SERIALIZE(live_time) + KV_SERIALIZE(avg_download) + KV_SERIALIZE(current_download) + KV_SERIALIZE(avg_upload) + KV_SERIALIZE(current_upload) END_KV_SERIALIZE_MAP() }; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp b/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp new file mode 100644 index 000000000..614ee8fab --- /dev/null +++ b/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp @@ -0,0 +1,176 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief This is the place to implement our handlers for protocol network actions, e.g. for ratelimit for download-requests + +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <boost/asio.hpp> +#include <string> +#include <vector> +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <atomic> + +#include <boost/asio.hpp> +#include <boost/array.hpp> +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <boost/enable_shared_from_this.hpp> +#include <boost/interprocess/detail/atomic.hpp> +#include <boost/thread/thread.hpp> + +#include <memory> + +#include "syncobj.h" + +#include "../../contrib/epee/include/net/net_utils_base.h" +#include "../../contrib/epee/include/misc_log_ex.h" +#include <boost/lambda/bind.hpp> +#include <boost/foreach.hpp> +#include <boost/lambda/lambda.hpp> +#include <boost/uuid/random_generator.hpp> +#include <boost/chrono.hpp> +#include <boost/utility/value_init.hpp> +#include <boost/asio/deadline_timer.hpp> +#include <boost/date_time/posix_time/posix_time.hpp> +#include <boost/thread/thread.hpp> +#include "misc_language.h" +#include "pragma_comp_defs.h" +#include <sstream> +#include <iomanip> +#include <algorithm> + + +#include <boost/asio/basic_socket.hpp> +#include <boost/asio/ip/unicast.hpp> + +#include "../../src/cryptonote_protocol/cryptonote_protocol_handler.h" +#include "../../src/p2p/network_throttle.hpp" + +#include "../../contrib/otshell_utils/utils.hpp" +using namespace nOT::nUtils; + +#include "../../../src/cryptonote_core/cryptonote_core.h" // e.g. for the send_stop_signal() + +// ################################################################################################ +// ################################################################################################ +// the "header part". Not separeted out for .hpp because point of this modification is +// to rebuild just 1 translation unit while working on this code. +// (But maybe common parts will be separated out later though - if needed) +// ################################################################################################ +// ################################################################################################ + +namespace cryptonote { + +class cryptonote_protocol_handler_base_pimpl { // placeholer if needed + public: + +}; + +} // namespace + +// ################################################################################################ +// ################################################################################################ +// ################################################################################################ +// ################################################################################################ + +namespace cryptonote { + +double cryptonote_protocol_handler_base::estimate_one_block_size() noexcept { // for estimating size of blocks to downloa + const double size_min = 500; // XXX 500 + //const int history_len = 20; // how many blocks to average over + + double avg=0; + try { + avg = get_avg_block_size(/*history_len*/); + } catch (...) { } + avg = std::max( size_min , avg); + return avg; +} + +cryptonote_protocol_handler_base::cryptonote_protocol_handler_base() { +} + +cryptonote_protocol_handler_base::~cryptonote_protocol_handler_base() { +} + +void cryptonote_protocol_handler_base::handler_request_blocks_history(std::list<crypto::hash>& ids) { + using namespace epee::net_utils; + LOG_PRINT_L0("### ~~~RRRR~~~~ ### sending request (type 2), limit = " << ids.size()); + LOG_PRINT_RED("RATE LIMIT NOT IMPLEMENTED HERE YET (download at unlimited speed?)" , LOG_LEVEL_0); + _note_c("net/req2", "### ~~~RRRR~~~~ ### sending request (type 2), limit = " << ids.size()); + // TODO +} + +void cryptonote_protocol_handler_base::handler_response_blocks_now(size_t packet_size) { _scope_dbg1(""); + using namespace epee::net_utils; + double delay=0; // will be calculated + _dbg1("Packet size: " << packet_size); + do + { // rate limiting + //XXX + /*if (::cryptonote::core::get_is_stopping()) { + _dbg1("We are stopping - so abort sleep"); + return; + }*/ + /*if (m_was_shutdown) { + _dbg2_c("net/netuse/sleep","m_was_shutdown - so abort sleep"); + return; + }*/ + + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); + delay = network_throttle_manager::get_global_throttle_out().get_sleep_time_after_tick( packet_size ); // decission from global + } + + + delay *= 0.50; + //delay = 0; // XXX + if (delay > 0) { + //delay += rand2*0.1; + long int ms = (long int)(delay * 1000); + _info_c("net/sleep", "Sleeping in " << __FUNCTION__ << " for " << ms << " ms before packet_size="<<packet_size); // XXX debug sleep + _dbg1_c("net/sleep/", "sleep in sleep_before_packet"); + _dbg2("Sleep for " << ms); + boost::this_thread::sleep(boost::posix_time::milliseconds( ms ) ); // TODO randomize sleeps + } + } while(delay > 0); + +// XXX LATER XXX + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); + network_throttle_manager::get_global_throttle_out().handle_trafic_tcp( packet_size ); // increase counter - global + //epee::critical_region_t<decltype(m_throttle_global_lock)> guard(m_throttle_global_lock); // *** critical *** + //m_throttle_global.m_out.handle_trafic_tcp( packet_size ); // increase counter - global + } +} + +} // namespace + + diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.h b/src/cryptonote_protocol/cryptonote_protocol_handler.h index 0e1a6f1b3..571c36dc1 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.h @@ -1,4 +1,8 @@ -// Copyright (c) 2014, The Monero Project +/// @file +/// @author rfree (current maintainer/user in monero.cc project - most of code is from CryptoNote) +/// @brief This is the orginal cryptonote protocol network-events handler, modified by us + +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -41,15 +45,36 @@ #include "cryptonote_core/connection_context.h" #include "cryptonote_core/cryptonote_stat_info.h" #include "cryptonote_core/verification_context.h" +// #include <netinet/in.h> +#include <boost/circular_buffer.hpp> PUSH_WARNINGS DISABLE_VS_WARNINGS(4355) +#define LOCALHOST_INT 2130706433 + namespace cryptonote { + class cryptonote_protocol_handler_base_pimpl; + class cryptonote_protocol_handler_base { + private: + std::unique_ptr<cryptonote_protocol_handler_base_pimpl> mI; + + public: + cryptonote_protocol_handler_base(); + virtual ~cryptonote_protocol_handler_base(); + void handler_request_blocks_history(std::list<crypto::hash>& ids); // before asking for list of objects, we can change the list still + void handler_response_blocks_now(size_t packet_size); + + virtual double get_avg_block_size() = 0; + virtual double estimate_one_block_size() noexcept; // for estimating size of blocks to download + + virtual std::ofstream& get_logreq() const =0; + }; + template<class t_core> - class t_cryptonote_protocol_handler: public i_cryptonote_protocol + class t_cryptonote_protocol_handler: public i_cryptonote_protocol, cryptonote_protocol_handler_base { public: typedef cryptonote_connection_context connection_context; @@ -106,6 +131,12 @@ namespace cryptonote nodetool::i_p2p_endpoint<connection_context>* m_p2p; std::atomic<uint32_t> m_syncronized_connections_count; std::atomic<bool> m_synchronized; + bool m_one_request = true; + + // static std::ofstream m_logreq; + std::mutex m_buffer_mutex; + double get_avg_block_size(); + boost::circular_buffer<size_t> m_avg_buffer = boost::circular_buffer<size_t>(10); template<class t_parametr> bool post_notify(typename t_parametr::request& arg, cryptonote_connection_context& context) @@ -113,6 +144,7 @@ namespace cryptonote LOG_PRINT_L2("[" << epee::net_utils::print_connection_context_short(context) << "] post " << typeid(t_parametr).name() << " -->"); std::string blob; epee::serialization::store_t_to_binary(arg, blob); + //handler_response_blocks_now(blob.size()); // XXX return m_p2p->invoke_notify_to_peer(t_parametr::ID, blob, context); } @@ -124,8 +156,11 @@ namespace cryptonote epee::serialization::store_t_to_binary(arg, arg_buff); return m_p2p->relay_notify_to_all(t_parametr::ID, arg_buff, exlude_context); } + + virtual std::ofstream& get_logreq() const ; }; -} + +} // namespace #include "cryptonote_protocol_handler.inl" diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 19ad74368..1cf66521f 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -1,4 +1,8 @@ -// Copyright (c) 2014, The Monero Project +/// @file +/// @author rfree (current maintainer/user in monero.cc project - most of code is from CryptoNote) +/// @brief This is the orginal cryptonote protocol network-events handler, modified by us + +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -28,14 +32,28 @@ // // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers +// (may contain code and/or modifications by other developers) +// developer rfree: this code is caller of our new network code, and is modded; e.g. for rate limiting + #include <boost/interprocess/detail/atomic.hpp> #include <list> #include "cryptonote_core/cryptonote_format_utils.h" #include "profile_tools.h" +#include "../../contrib/otshell_utils/utils.hpp" +#include "../../src/p2p/network_throttle-detail.hpp" +#include "../../src/p2p/data_logger.hpp" +using namespace nOT::nUtils; + namespace cryptonote { + +// static +// template<class t_core> std::ofstream t_cryptonote_protocol_handler<t_core>::m_logreq("logreq.txt"); // static + + + //----------------------------------------------------------------------------------------------------------------------- template<class t_core> t_cryptonote_protocol_handler<t_core>::t_cryptonote_protocol_handler(t_core& rcore, nodetool::i_p2p_endpoint<connection_context>* p_net_layout):m_core(rcore), @@ -99,24 +117,66 @@ namespace cryptonote void t_cryptonote_protocol_handler<t_core>::log_connections() { std::stringstream ss; + ss.precision(1); + + double down_sum = 0.0; + double down_curr_sum = 0.0; + double up_sum = 0.0; + double up_curr_sum = 0.0; - ss << std::setw(25) << std::left << "Remote Host" + ss << std::setw(30) << std::left << "Remote Host" << std::setw(20) << "Peer id" - << std::setw(25) << "Recv/Sent (inactive,sec)" + << std::setw(30) << "Recv/Sent (inactive,sec)" << std::setw(25) << "State" - << std::setw(20) << "Livetime(seconds)" << ENDL; - + << std::setw(20) << "Livetime(sec)" + << std::setw(12) << "Down (kB/s)" + << std::setw(14) << "Down(now)" + << std::setw(10) << "Up (kB/s)" + << std::setw(13) << "Up(now)" + << ENDL; + + uint32_t ip; m_p2p->for_each_connection([&](const connection_context& cntxt, nodetool::peerid_type peer_id) { - ss << std::setw(25) << std::left << std::string(cntxt.m_is_income ? " [INC]":"[OUT]") + - epee::string_tools::get_ip_string_from_int32(cntxt.m_remote_ip) + ":" + std::to_string(cntxt.m_remote_port) + bool local_ip = false; + ip = ntohl(cntxt.m_remote_ip); + // TODO: local ip in calss A, B + if (ip > 3232235520 && ip < 3232301055) // 192.168.x.x + local_ip = true; + auto connection_time = time(NULL) - cntxt.m_started; + ss << std::setw(30) << std::left << std::string(cntxt.m_is_income ? " [INC]":"[OUT]") + + epee::string_tools::get_ip_string_from_int32(cntxt.m_remote_ip) + ":" + std::to_string(cntxt.m_remote_port) << std::setw(20) << std::hex << peer_id - << std::setw(25) << std::to_string(cntxt.m_recv_cnt)+ "(" + std::to_string(time(NULL) - cntxt.m_last_recv) + ")" + "/" + std::to_string(cntxt.m_send_cnt) + "(" + std::to_string(time(NULL) - cntxt.m_last_send) + ")" + << std::setw(30) << std::to_string(cntxt.m_recv_cnt)+ "(" + std::to_string(time(NULL) - cntxt.m_last_recv) + ")" + "/" + std::to_string(cntxt.m_send_cnt) + "(" + std::to_string(time(NULL) - cntxt.m_last_send) + ")" << std::setw(25) << get_protocol_state_string(cntxt.m_state) - << std::setw(20) << std::to_string(time(NULL) - cntxt.m_started) << ENDL; + << std::setw(20) << std::to_string(time(NULL) - cntxt.m_started) + << std::setw(12) << std::fixed << (connection_time == 0 ? 0.0 : cntxt.m_recv_cnt / connection_time / 1024) + << std::setw(14) << std::fixed << cntxt.m_current_speed_down / 1024 + << std::setw(10) << std::fixed << (connection_time == 0 ? 0.0 : cntxt.m_send_cnt / connection_time / 1024) + << std::setw(13) << std::fixed << cntxt.m_current_speed_up / 1024 + << (local_ip ? "[LAN]" : "") + << std::left << (ip == LOCALHOST_INT ? "[LOCALHOST]" : "") // 127.0.0.1 + << ENDL; + + if (connection_time > 1) + { + down_sum += (cntxt.m_recv_cnt / connection_time / 1024); + up_sum += (cntxt.m_send_cnt / connection_time / 1024); + } + + down_curr_sum += (cntxt.m_current_speed_down / 1024); + up_curr_sum += (cntxt.m_current_speed_up / 1024); + return true; }); - LOG_PRINT_L0("Connections: " << ENDL << ss.str()); + ss << ENDL + << std::setw(125) << " " + << std::setw(12) << down_sum + << std::setw(14) << down_curr_sum + << std::setw(10) << up_sum + << std::setw(13) << up_curr_sum + << ENDL; + LOG_PRINT_L0("Connections: " << ENDL << ss.str()); } //------------------------------------------------------------------------------------------------------------------------ // Returns a list of connection_info objects describing each open p2p connection @@ -150,6 +210,42 @@ namespace cryptonote cnx.live_time = timestamp - cntxt.m_started; + uint32_t ip; + ip = ntohl(cntxt.m_remote_ip); + if (ip == LOCALHOST_INT) + { + cnx.localhost = true; + } + else + { + cnx.localhost = false; + } + + if (ip > 3232235520 && ip < 3232301055) // 192.168.x.x + { + cnx.local_ip = true; + } + else + { + cnx.local_ip = false; + } + + auto connection_time = time(NULL) - cntxt.m_started; + if (connection_time == 0) + { + cnx.avg_download = 0; + cnx.avg_upload = 0; + } + + else + { + cnx.avg_download = cntxt.m_recv_cnt / connection_time / 1024; + cnx.avg_upload = cntxt.m_send_cnt / connection_time / 1024; + } + + cnx.current_download = cntxt.m_current_speed_down / 1024; + cnx.current_upload = cntxt.m_current_speed_up / 1024; + connections.push_back(cnx); return true; @@ -234,11 +330,11 @@ namespace cryptonote block_verification_context bvc = boost::value_initialized<block_verification_context>(); m_core.pause_mine(); - m_core.handle_incoming_block(arg.b.block, bvc); + m_core.handle_incoming_block(arg.b.block, bvc); // got block from handle_notify_new_block m_core.resume_mine(); if(bvc.m_verifivation_failed) { - LOG_PRINT_CCONTEXT_L1("Block verification failed, dropping connection"); + LOG_PRINT_CCONTEXT_L0("Block verification failed, dropping connection"); m_p2p->drop_connection(context); return 1; } @@ -304,13 +400,70 @@ namespace cryptonote LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_RESPONSE_GET_OBJECTS: blocks.size()=" << rsp.blocks.size() << ", txs.size()=" << rsp.txs.size() << ", rsp.m_current_blockchain_height=" << rsp.current_blockchain_height << ", missed_ids.size()=" << rsp.missed_ids.size()); post_notify<NOTIFY_RESPONSE_GET_OBJECTS>(rsp, context); + //handler_response_blocks_now(sizeof(rsp)); // XXX + //handler_response_blocks_now(200); return 1; } //------------------------------------------------------------------------------------------------------------------------ + + + template<class t_core> + double t_cryptonote_protocol_handler<t_core>::get_avg_block_size() { + // return m_core.get_blockchain_storage().get_avg_block_size(count); // this does not count too well the actuall network-size of data we need to download + + CRITICAL_REGION_LOCAL(m_buffer_mutex); + double avg = 0; + if (m_avg_buffer.size() == 0) { + _warn("m_avg_buffer.size() == 0"); + return 500; + } + + const bool dbg_poke_lock = 0; // debug: try to trigger an error by poking around with locks. TODO: configure option + long int dbg_repeat=0; + do { + for (auto element : m_avg_buffer) avg += element; + } while(dbg_poke_lock && (dbg_repeat++)<100000); // in debug/poke mode, repeat this calculation to trigger hidden locking error if there is one + return avg / m_avg_buffer.size(); + } + + template<class t_core> int t_cryptonote_protocol_handler<t_core>::handle_response_get_objects(int command, NOTIFY_RESPONSE_GET_OBJECTS::request& arg, cryptonote_connection_context& context) { LOG_PRINT_CCONTEXT_L2("NOTIFY_RESPONSE_GET_OBJECTS"); + + // calculate size of request - mainly for logging/debug + size_t size = 0; + for (auto element : arg.txs) size += element.size(); + + for (auto element : arg.blocks) { + size += element.block.size(); + for (auto tx : element.txs) + size += tx.size(); + } + + for (auto element : arg.missed_ids) + size += sizeof(element.data); + + size += sizeof(arg.current_blockchain_height); + { + CRITICAL_REGION_LOCAL(m_buffer_mutex); + m_avg_buffer.push_back(size); + + const bool dbg_poke_lock = 0; // debug: try to trigger an error by poking around with locks. TODO: configure option + long int dbg_repeat=0; + do { + m_avg_buffer.push_back(666); // a test value + m_avg_buffer.erase_end(1); + } while(dbg_poke_lock && (dbg_repeat++)<100000); // in debug/poke mode, repeat this calculation to trigger hidden locking error if there is one + } + /*using namespace boost::chrono; + auto point = steady_clock::now(); + auto time_from_epoh = point.time_since_epoch(); + auto sec = duration_cast< seconds >( time_from_epoh ).count();*/ + + //epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size()); + if(context.m_last_response_height > arg.current_blockchain_height) { LOG_ERROR_CCONTEXT("sent wrong NOTIFY_HAVE_OBJECTS: arg.m_current_blockchain_height=" << arg.current_blockchain_height @@ -373,53 +526,67 @@ namespace cryptonote return 1; } + { m_core.pause_mine(); epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler( boost::bind(&t_core::resume_mine, &m_core)); - BOOST_FOREACH(const block_complete_entry& block_entry, arg.blocks) - { - //process transactions - TIME_MEASURE_START(transactions_process_time); - BOOST_FOREACH(auto& tx_blob, block_entry.txs) - { - tx_verification_context tvc = AUTO_VAL_INIT(tvc); - m_core.handle_incoming_tx(tx_blob, tvc, true); - if(tvc.m_verifivation_failed) - { - LOG_ERROR_CCONTEXT("transaction verification failed on NOTIFY_RESPONSE_GET_OBJECTS, \r\ntx_id = " - << epee::string_tools::pod_to_hex(get_blob_hash(tx_blob)) << ", dropping connection"); - m_p2p->drop_connection(context); - return 1; - } - } - TIME_MEASURE_FINISH(transactions_process_time); - - //process block - TIME_MEASURE_START(block_process_time); - block_verification_context bvc = boost::value_initialized<block_verification_context>(); - - m_core.handle_incoming_block(block_entry.block, bvc, false); - - if(bvc.m_verifivation_failed) - { - LOG_PRINT_CCONTEXT_L1("Block verification failed, dropping connection"); - m_p2p->drop_connection(context); - return 1; - } - if(bvc.m_marked_as_orphaned) - { - LOG_PRINT_CCONTEXT_L1("Block received at sync phase was marked as orphaned, dropping connection"); - m_p2p->drop_connection(context); - return 1; - } - - TIME_MEASURE_FINISH(block_process_time); - LOG_PRINT_CCONTEXT_L2("Block process time: " << block_process_time + transactions_process_time << "(" << transactions_process_time << "/" << block_process_time << ")ms"); - } + LOG_PRINT_CCONTEXT_YELLOW( "Got NEW BLOCKS inside of " << __FUNCTION__ << ": size: " << arg.blocks.size() , LOG_LEVEL_0); + + if (m_core.get_test_drop_download() && m_core.get_test_drop_download_height()) { // DISCARD BLOCKS for testing + + + BOOST_FOREACH(const block_complete_entry& block_entry, arg.blocks) + { + // process transactions + TIME_MEASURE_START(transactions_process_time); + BOOST_FOREACH(auto& tx_blob, block_entry.txs) + { + tx_verification_context tvc = AUTO_VAL_INIT(tvc); + m_core.handle_incoming_tx(tx_blob, tvc, true); + if(tvc.m_verifivation_failed) + { + LOG_ERROR_CCONTEXT("transaction verification failed on NOTIFY_RESPONSE_GET_OBJECTS, \r\ntx_id = " + << epee::string_tools::pod_to_hex(get_blob_hash(tx_blob)) << ", dropping connection"); + m_p2p->drop_connection(context); + return 1; + } + } + TIME_MEASURE_FINISH(transactions_process_time); + + // process block + + TIME_MEASURE_START(block_process_time); + block_verification_context bvc = boost::value_initialized<block_verification_context>(); + + m_core.handle_incoming_block(block_entry.block, bvc, false); // <--- process block + + if(bvc.m_verifivation_failed) + { + LOG_PRINT_CCONTEXT_L1("Block verification failed, dropping connection"); + m_p2p->drop_connection(context); + return 1; + } + if(bvc.m_marked_as_orphaned) + { + LOG_PRINT_CCONTEXT_L1("Block received at sync phase was marked as orphaned, dropping connection"); + m_p2p->drop_connection(context); + return 1; + } + + TIME_MEASURE_FINISH(block_process_time); + LOG_PRINT_CCONTEXT_L2("Block process time: " << block_process_time + transactions_process_time << "(" << transactions_process_time << "/" << block_process_time << ")ms"); + + epee::net_utils::data_logger::get_instance().add_data("calc_time", block_process_time + transactions_process_time); + epee::net_utils::data_logger::get_instance().add_data("block_processing", 1); + + } // each download block + + } // if not DISCARD BLOCK + + } - request_missing_objects(context, true); return 1; } @@ -448,6 +615,15 @@ namespace cryptonote template<class t_core> bool t_cryptonote_protocol_handler<t_core>::request_missing_objects(cryptonote_connection_context& context, bool check_having_blocks) { + //if (!m_one_request == false) + //return true; + m_one_request = false; + // save request size to log (dr monero) + /*using namespace boost::chrono; + auto point = steady_clock::now(); + auto time_from_epoh = point.time_since_epoch(); + auto sec = duration_cast< seconds >( time_from_epoh ).count();*/ + if(context.m_needed_objects.size()) { //we know objects that we need, request this objects @@ -455,6 +631,8 @@ namespace cryptonote size_t count = 0; auto it = context.m_needed_objects.begin(); + size_t count_limit = BLOCKS_SYNCHRONIZING_DEFAULT_COUNT; + _note_c("net/req-calc" , "Setting count_limit: " << count_limit); while(it != context.m_needed_objects.end() && count < BLOCKS_SYNCHRONIZING_DEFAULT_COUNT) { if( !(check_having_blocks && m_core.have_block(*it))) @@ -465,14 +643,24 @@ namespace cryptonote } context.m_needed_objects.erase(it++); } - LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_REQUEST_GET_OBJECTS: blocks.size()=" << req.blocks.size() << ", txs.size()=" << req.txs.size()); + LOG_PRINT_CCONTEXT_L0("-->>NOTIFY_REQUEST_GET_OBJECTS: blocks.size()=" << req.blocks.size() << ", txs.size()=" << req.txs.size() + << "requested blocks count=" << count << " / " << count_limit); + //epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size()); + post_notify<NOTIFY_REQUEST_GET_OBJECTS>(req, context); }else if(context.m_last_response_height < context.m_remote_blockchain_height-1) {//we have to fetch more objects ids, request blockchain entry NOTIFY_REQUEST_CHAIN::request r = boost::value_initialized<NOTIFY_REQUEST_CHAIN::request>(); m_core.get_short_chain_history(r.block_ids); - LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); + handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?) + + //std::string blob; // for calculate size of request + //epee::serialization::store_t_to_binary(r, blob); + //epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size()); + LOG_PRINT_CCONTEXT_L0("r = " << 200); + + LOG_PRINT_CCONTEXT_L0("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); post_notify<NOTIFY_REQUEST_CHAIN>(r, context); }else { @@ -575,4 +763,18 @@ namespace cryptonote { return relay_post_notify<NOTIFY_NEW_TRANSACTIONS>(arg, exclude_context); } -} + + /// @deprecated + template<class t_core> std::ofstream& t_cryptonote_protocol_handler<t_core>::get_logreq() const { + static std::ofstream * logreq=NULL; + if (!logreq) { + LOG_PRINT_RED("LOG OPENED",LOG_LEVEL_0); + logreq = new std::ofstream("logreq.txt"); // leak mem (singleton) + *logreq << "Opened log" << std::endl; + } + LOG_PRINT_YELLOW("LOG USED",LOG_LEVEL_0); + (*logreq) << "log used" << std::endl; + return *logreq; + } + +} // namespace diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler_common.h b/src/cryptonote_protocol/cryptonote_protocol_handler_common.h index 4532645ce..9ccf6aa56 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler_common.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler_common.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt index adcc61c43..bf25bbca9 100644 --- a/src/daemon/CMakeLists.txt +++ b/src/daemon/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # @@ -27,12 +27,27 @@ # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. set(daemon_sources - daemon.cpp) + command_parser_executor.cpp + command_server.cpp + daemon.cpp + executor.cpp + main.cpp + rpc_command_executor.cpp +) set(daemon_headers) set(daemon_private_headers + command_parser_executor.h + command_server.h + core.h + daemon.h daemon_commands_handler.h + executor.h + p2p.h + protocol.h + rpc.h + rpc_command_executor.h # cryptonote_protocol ../cryptonote_protocol/blobdatatype.h @@ -59,9 +74,14 @@ bitmonero_add_executable(daemon target_link_libraries(daemon LINK_PRIVATE rpc + blockchain_db cryptonote_core crypto common + otshell_utils + p2p + cryptonote_protocol + daemonizer ${Boost_CHRONO_LIBRARY} ${Boost_FILESYSTEM_LIBRARY} ${Boost_PROGRAM_OPTIONS_LIBRARY} diff --git a/src/daemon/command_line_args.h b/src/daemon/command_line_args.h new file mode 100644 index 000000000..2bd918478 --- /dev/null +++ b/src/daemon/command_line_args.h @@ -0,0 +1,81 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef DAEMON_COMMAND_LINE_ARGS_H +#define DAEMON_COMMAND_LINE_ARGS_H + +#include "common/command_line.h" +#include "cryptonote_config.h" +#include <boost/program_options.hpp> + +namespace daemon_args +{ + std::string const WINDOWS_SERVICE_NAME = "Monero Daemon"; + + const command_line::arg_descriptor<std::string> arg_config_file = { + "config-file" + , "Specify configuration file" + , std::string(CRYPTONOTE_NAME ".conf") + }; + const command_line::arg_descriptor<std::string> arg_log_file = { + "log-file" + , "Specify log file" + , "" + }; + const command_line::arg_descriptor<int> arg_log_level = { + "log-level" + , "" + , LOG_LEVEL_0 + }; + const command_line::arg_descriptor<std::vector<std::string>> arg_command = { + "daemon_command" + , "Hidden" + }; + const command_line::arg_descriptor<bool> arg_os_version = { + "os-version" + , "OS for which this executable was compiled" + }; + const command_line::arg_descriptor<bool> arg_testnet_on = { + "testnet" + , "Run on testnet. The wallet must be launched with --testnet flag." + , false + }; + const command_line::arg_descriptor<bool> arg_dns_checkpoints = { + "enforce-dns-checkpointing" + , "checkpoints from DNS server will be enforced" + , false + }; + const command_line::arg_descriptor<std::string> arg_db_type = { + "db-type" + , "Specify database type" + , "lmdb" + }; + +} // namespace daemon_args + +#endif // DAEMON_COMMAND_LINE_ARGS_H diff --git a/src/daemon/command_parser_executor.cpp b/src/daemon/command_parser_executor.cpp new file mode 100644 index 000000000..fd7b40be2 --- /dev/null +++ b/src/daemon/command_parser_executor.cpp @@ -0,0 +1,336 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "cryptonote_core/cryptonote_basic_impl.h" +#include "daemon/command_parser_executor.h" + +namespace daemonize { + +t_command_parser_executor::t_command_parser_executor( + uint32_t ip + , uint16_t port + , bool is_rpc + , cryptonote::core_rpc_server* rpc_server + ) + : m_executor(ip, port, is_rpc, rpc_server) +{} + +bool t_command_parser_executor::print_peer_list(const std::vector<std::string>& args) +{ + if (!args.empty()) return false; + + return m_executor.print_peer_list(); +} + +bool t_command_parser_executor::save_blockchain(const std::vector<std::string>& args) +{ + if (!args.empty()) return false; + + return m_executor.save_blockchain(); +} + +bool t_command_parser_executor::show_hash_rate(const std::vector<std::string>& args) +{ + if (!args.empty()) return false; + + return m_executor.show_hash_rate(); +} + +bool t_command_parser_executor::hide_hash_rate(const std::vector<std::string>& args) +{ + if (!args.empty()) return false; + + return m_executor.hide_hash_rate(); +} + +bool t_command_parser_executor::show_difficulty(const std::vector<std::string>& args) +{ + if (!args.empty()) return false; + + return m_executor.show_difficulty(); +} + +bool t_command_parser_executor::print_connections(const std::vector<std::string>& args) +{ + if (!args.empty()) return false; + + return m_executor.print_connections(); +} + +bool t_command_parser_executor::print_blockchain_info(const std::vector<std::string>& args) +{ + if(!args.size()) + { + std::cout << "need block index parameter" << std::endl; + return false; + } + uint64_t start_index = 0; + uint64_t end_index = 0; + if(!epee::string_tools::get_xtype_from_string(start_index, args[0])) + { + std::cout << "wrong starter block index parameter" << std::endl; + return false; + } + if(args.size() >1 && !epee::string_tools::get_xtype_from_string(end_index, args[1])) + { + std::cout << "wrong end block index parameter" << std::endl; + return false; + } + + return m_executor.print_blockchain_info(start_index, end_index); +} + +bool t_command_parser_executor::set_log_level(const std::vector<std::string>& args) +{ + if(args.size() != 1) + { + std::cout << "use: set_log <log_level_number_0-4>" << std::endl; + return true; + } + + uint16_t l = 0; + if(!epee::string_tools::get_xtype_from_string(l, args[0])) + { + std::cout << "wrong number format, use: set_log <log_level_number_0-4>" << std::endl; + return true; + } + + if(LOG_LEVEL_4 < l) + { + std::cout << "wrong number range, use: set_log <log_level_number_0-4>" << std::endl; + return true; + } + + return m_executor.set_log_level(l); +} + +bool t_command_parser_executor::print_height(const std::vector<std::string>& args) +{ + if (!args.empty()) return false; + + return m_executor.print_height(); +} + +bool t_command_parser_executor::print_block(const std::vector<std::string>& args) +{ + if (args.empty()) + { + std::cout << "expected: print_block (<block_hash> | <block_height>)" << std::endl; + return false; + } + + const std::string& arg = args.front(); + try + { + uint64_t height = boost::lexical_cast<uint64_t>(arg); + return m_executor.print_block_by_height(height); + } + catch (boost::bad_lexical_cast&) + { + crypto::hash block_hash; + if (parse_hash256(arg, block_hash)) + { + return m_executor.print_block_by_hash(block_hash); + } + } + + return false; +} + +bool t_command_parser_executor::print_transaction(const std::vector<std::string>& args) +{ + if (args.empty()) + { + std::cout << "expected: print_tx <transaction hash>" << std::endl; + return true; + } + + const std::string& str_hash = args.front(); + crypto::hash tx_hash; + if (parse_hash256(str_hash, tx_hash)) + { + m_executor.print_transaction(tx_hash); + } + + return true; +} + +bool t_command_parser_executor::print_transaction_pool_long(const std::vector<std::string>& args) +{ + if (!args.empty()) return false; + + return m_executor.print_transaction_pool_long(); +} + +bool t_command_parser_executor::print_transaction_pool_short(const std::vector<std::string>& args) +{ + if (!args.empty()) return false; + + return m_executor.print_transaction_pool_short(); +} + +bool t_command_parser_executor::start_mining(const std::vector<std::string>& args) +{ + if(!args.size()) + { + std::cout << "Please specify a wallet address to mine for: start_mining <addr> [threads=1]" << std::endl; + return true; + } + + cryptonote::account_public_address adr; + if(!cryptonote::get_account_address_from_str(adr, false, args.front())) + { + if(!cryptonote::get_account_address_from_str(adr, true, args.front())) + { + std::cout << "target account address has wrong format" << std::endl; + return true; + } + std::cout << "Mining to a testnet address, make sure this is intentional!" << std::endl; + } + uint64_t threads_count = 1; + if(args.size() > 2) + { + return false; + } + else if(args.size() == 2) + { + bool ok = epee::string_tools::get_xtype_from_string(threads_count, args[1]); + threads_count = (ok && 0 < threads_count) ? threads_count : 1; + } + + m_executor.start_mining(adr, threads_count); + + return true; +} + +bool t_command_parser_executor::stop_mining(const std::vector<std::string>& args) +{ + if (!args.empty()) return false; + + return m_executor.stop_mining(); +} + +bool t_command_parser_executor::stop_daemon(const std::vector<std::string>& args) +{ + if (!args.empty()) return false; + + return m_executor.stop_daemon(); +} + +bool t_command_parser_executor::print_status(const std::vector<std::string>& args) +{ + if (!args.empty()) return false; + + return m_executor.print_status(); +} + +bool t_command_parser_executor::set_limit(const std::vector<std::string>& args) +{ + if(args.size()!=1) return false; + int limit; + try { + limit = std::stoi(args[0]); + } + catch(std::invalid_argument& ex) { + return false; + } + if (limit==-1) limit=128; + limit *= 1024; + + return m_executor.set_limit(limit); +} + +bool t_command_parser_executor::set_limit_up(const std::vector<std::string>& args) +{ + if(args.size()!=1) return false; + int limit; + try { + limit = std::stoi(args[0]); + } + catch(std::invalid_argument& ex) { + return false; + } + if (limit==-1) limit=128; + limit *= 1024; + + return m_executor.set_limit_up(limit); +} + +bool t_command_parser_executor::set_limit_down(const std::vector<std::string>& args) +{ + if(args.size()!=1) return false; + int limit; + try { + limit = std::stoi(args[0]); + } + catch(std::invalid_argument& ex) { + return false; + } + if (limit==-1) limit=128; + limit *= 1024; + + return m_executor.set_limit_down(limit); +} + +bool t_command_parser_executor::fast_exit(const std::vector<std::string>& args) +{ + if (!args.empty()) return false; + return m_executor.fast_exit(); +} + +bool t_command_parser_executor::out_peers(const std::vector<std::string>& args) +{ + if (args.empty()) return false; + + unsigned int limit; + try { + limit = std::stoi(args[0]); + } + + catch(std::invalid_argument& ex) { + _erro("stoi exception"); + return false; + } + + return m_executor.out_peers(limit); +} + +bool t_command_parser_executor::start_save_graph(const std::vector<std::string>& args) +{ + if (!args.empty()) return false; + return m_executor.start_save_graph(); +} + +bool t_command_parser_executor::stop_save_graph(const std::vector<std::string>& args) +{ + if (!args.empty()) return false; + return m_executor.stop_save_graph(); +} + + +} // namespace daemonize diff --git a/src/daemon/command_parser_executor.h b/src/daemon/command_parser_executor.h new file mode 100644 index 000000000..27ffabc1c --- /dev/null +++ b/src/daemon/command_parser_executor.h @@ -0,0 +1,105 @@ +/** +@file +@details + +@image html images/other/runtime-commands.png + +*/ + +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include "daemon/rpc_command_executor.h" +#include "rpc/core_rpc_server.h" + +namespace daemonize { + +class t_command_parser_executor final +{ +private: + t_rpc_command_executor m_executor; +public: + t_command_parser_executor( + uint32_t ip + , uint16_t port + , bool is_rpc + , cryptonote::core_rpc_server* rpc_server = NULL + ); + + bool print_peer_list(const std::vector<std::string>& args); + + bool save_blockchain(const std::vector<std::string>& args); + + bool show_hash_rate(const std::vector<std::string>& args); + + bool hide_hash_rate(const std::vector<std::string>& args); + + bool show_difficulty(const std::vector<std::string>& args); + + bool print_connections(const std::vector<std::string>& args); + + bool print_blockchain_info(const std::vector<std::string>& args); + + bool set_log_level(const std::vector<std::string>& args); + + bool print_height(const std::vector<std::string>& args); + + bool print_block(const std::vector<std::string>& args); + + bool print_transaction(const std::vector<std::string>& args); + + bool print_transaction_pool_long(const std::vector<std::string>& args); + + bool print_transaction_pool_short(const std::vector<std::string>& args); + + bool start_mining(const std::vector<std::string>& args); + + bool stop_mining(const std::vector<std::string>& args); + + bool stop_daemon(const std::vector<std::string>& args); + + bool print_status(const std::vector<std::string>& args); + + bool set_limit(const std::vector<std::string>& args); + + bool set_limit_up(const std::vector<std::string>& args); + + bool set_limit_down(const std::vector<std::string>& args); + + bool fast_exit(const std::vector<std::string>& args); + + bool out_peers(const std::vector<std::string>& args); + + bool start_save_graph(const std::vector<std::string>& args); + + bool stop_save_graph(const std::vector<std::string>& args); +}; + +} // namespace daemonize diff --git a/src/daemon/command_server.cpp b/src/daemon/command_server.cpp new file mode 100644 index 000000000..f0f4cd676 --- /dev/null +++ b/src/daemon/command_server.cpp @@ -0,0 +1,233 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "cryptonote_config.h" +#include "version.h" +#include "daemon/command_server.h" + +namespace daemonize { + +namespace p = std::placeholders; + +t_command_server::t_command_server( + uint32_t ip + , uint16_t port + , bool is_rpc + , cryptonote::core_rpc_server* rpc_server + ) + : m_parser(ip, port, is_rpc, rpc_server) + , m_command_lookup() + , m_is_rpc(is_rpc) +{ + m_command_lookup.set_handler( + "q" + , [] (const std::vector<std::string>& args) {return true;} + , "ignored" + ); + m_command_lookup.set_handler( + "help" + , std::bind(&t_command_server::help, this, p::_1) + , "Show this help" + ); + m_command_lookup.set_handler( + "print_height" + , std::bind(&t_command_parser_executor::print_height, &m_parser, p::_1) + , "Print local blockchain height" + ); + m_command_lookup.set_handler( + "print_pl" + , std::bind(&t_command_parser_executor::print_peer_list, &m_parser, p::_1) + , "Print peer list" + ); + m_command_lookup.set_handler( + "print_cn" + , std::bind(&t_command_parser_executor::print_connections, &m_parser, p::_1) + , "Print connections" + ); + m_command_lookup.set_handler( + "print_bc" + , std::bind(&t_command_parser_executor::print_blockchain_info, &m_parser, p::_1) + , "Print blockchain info in a given blocks range, print_bc <begin_height> [<end_height>]" + ); + m_command_lookup.set_handler( + "print_block" + , std::bind(&t_command_parser_executor::print_block, &m_parser, p::_1) + , "Print block, print_block <block_hash> | <block_height>" + ); + m_command_lookup.set_handler( + "print_tx" + , std::bind(&t_command_parser_executor::print_transaction, &m_parser, p::_1) + , "Print transaction, print_tx <transaction_hash>" + ); + m_command_lookup.set_handler( + "start_mining" + , std::bind(&t_command_parser_executor::start_mining, &m_parser, p::_1) + , "Start mining for specified address, start_mining <addr> [threads=1]" + ); + m_command_lookup.set_handler( + "stop_mining" + , std::bind(&t_command_parser_executor::stop_mining, &m_parser, p::_1) + , "Stop mining" + ); + m_command_lookup.set_handler( + "print_pool" + , std::bind(&t_command_parser_executor::print_transaction_pool_long, &m_parser, p::_1) + , "Print transaction pool (long format)" + ); + m_command_lookup.set_handler( + "print_pool_sh" + , std::bind(&t_command_parser_executor::print_transaction_pool_short, &m_parser, p::_1) + , "Print transaction pool (short format)" + ); + m_command_lookup.set_handler( + "show_hr" + , std::bind(&t_command_parser_executor::show_hash_rate, &m_parser, p::_1) + , "Start showing hash rate" + ); + m_command_lookup.set_handler( + "hide_hr" + , std::bind(&t_command_parser_executor::hide_hash_rate, &m_parser, p::_1) + , "Stop showing hash rate" + ); + m_command_lookup.set_handler( + "save" + , std::bind(&t_command_parser_executor::save_blockchain, &m_parser, p::_1) + , "Save blockchain" + ); + m_command_lookup.set_handler( + "set_log" + , std::bind(&t_command_parser_executor::set_log_level, &m_parser, p::_1) + , "set_log <level> - Change current log detalization level, <level> is a number 0-4" + ); + m_command_lookup.set_handler( + "diff" + , std::bind(&t_command_parser_executor::show_difficulty, &m_parser, p::_1) + , "Show difficulty" + ); + m_command_lookup.set_handler( + "stop_daemon" + , std::bind(&t_command_parser_executor::stop_daemon, &m_parser, p::_1) + , "Stop the daemon" + ); + m_command_lookup.set_handler( + "exit" + , std::bind(&t_command_parser_executor::stop_daemon, &m_parser, p::_1) + , "Stop the daemon" + ); + m_command_lookup.set_handler( + "print_status" + , std::bind(&t_command_parser_executor::print_status, &m_parser, p::_1) + , "Prints daemon status" + ); + m_command_lookup.set_handler( + "limit" + , std::bind(&t_command_parser_executor::set_limit, &m_parser, p::_1) + , "limit <kB/s> - Set download and upload limit" + ); + m_command_lookup.set_handler( + "limit_up" + , std::bind(&t_command_parser_executor::set_limit_up, &m_parser, p::_1) + , "limit <kB/s> - Set upload limit" + ); + m_command_lookup.set_handler( + "limit_down" + , std::bind(&t_command_parser_executor::set_limit_down, &m_parser, p::_1) + , "limit <kB/s> - Set download limit" + ); + m_command_lookup.set_handler( + "fast_exit" + , std::bind(&t_command_parser_executor::fast_exit, &m_parser, p::_1) + , "Exit" + ); + m_command_lookup.set_handler( + "out_peers" + , std::bind(&t_command_parser_executor::out_peers, &m_parser, p::_1) + , "Set max limit of out peers" + ); + m_command_lookup.set_handler( + "start_save_graph" + , std::bind(&t_command_parser_executor::start_save_graph, &m_parser, p::_1) + , "Start save data for dr monero" + ); + m_command_lookup.set_handler( + "stop_save_graph" + , std::bind(&t_command_parser_executor::stop_save_graph, &m_parser, p::_1) + , "Stop save data for dr monero" + ); +} + +bool t_command_server::process_command_str(const std::string& cmd) +{ + return m_command_lookup.process_command_str(cmd); +} + +bool t_command_server::process_command_vec(const std::vector<std::string>& cmd) +{ + bool result = m_command_lookup.process_command_vec(cmd); + if (!result) + { + help(std::vector<std::string>()); + } + return result; +} + +bool t_command_server::start_handling() +{ + if (m_is_rpc) return false; + + m_command_lookup.start_handling("", get_commands_str()); + + return true; +} + +void t_command_server::stop_handling() +{ + if (m_is_rpc) return; + + m_command_lookup.stop_handling(); +} + +bool t_command_server::help(const std::vector<std::string>& args) +{ + std::cout << get_commands_str() << std::endl; + return true; +} + +std::string t_command_server::get_commands_str() +{ + std::stringstream ss; + ss << CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL << std::endl; + ss << "Commands: " << std::endl; + std::string usage = m_command_lookup.get_usage(); + boost::replace_all(usage, "\n", "\n "); + usage.insert(0, " "); + ss << usage << std::endl; + return ss.str(); +} + +} // namespace daemonize diff --git a/src/daemon/command_server.h b/src/daemon/command_server.h new file mode 100644 index 000000000..494f44d8b --- /dev/null +++ b/src/daemon/command_server.h @@ -0,0 +1,75 @@ +/** +@file +@details + + +Passing RPC commands: + +@image html images/other/runtime-commands.png + +*/ + +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include "console_handler.h" +#include "daemon/command_parser_executor.h" + +namespace daemonize { + +class t_command_server { +private: + t_command_parser_executor m_parser; + epee::console_handlers_binder m_command_lookup; + bool m_is_rpc; + +public: + t_command_server( + uint32_t ip + , uint16_t port + , bool is_rpc = true + , cryptonote::core_rpc_server* rpc_server = NULL + ); + + bool process_command_str(const std::string& cmd); + + bool process_command_vec(const std::vector<std::string>& cmd); + + bool start_handling(); + + void stop_handling(); + +private: + bool help(const std::vector<std::string>& args); + + std::string get_commands_str(); +}; + +} // namespace daemonize diff --git a/src/daemon/core.h b/src/daemon/core.h new file mode 100644 index 000000000..6564e5314 --- /dev/null +++ b/src/daemon/core.h @@ -0,0 +1,99 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include "cryptonote_core/checkpoints_create.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_protocol/cryptonote_protocol_handler.h" +#include "misc_log_ex.h" +#include <stdexcept> +#include <boost/program_options.hpp> +#include "daemon/command_line_args.h" + +namespace daemonize +{ + +class t_core final +{ +public: + static void init_options(boost::program_options::options_description & option_spec) + { + cryptonote::core::init_options(option_spec); + cryptonote::miner::init_options(option_spec); + } +private: + typedef cryptonote::t_cryptonote_protocol_handler<cryptonote::core> t_protocol_raw; + cryptonote::core m_core; + // TEMPORARY HACK - Yes, this creates a copy, but otherwise the original + // variable map could go out of scope before the run method is called + boost::program_options::variables_map const m_vm_HACK; +public: + t_core( + boost::program_options::variables_map const & vm + ) + : m_core{nullptr} + , m_vm_HACK{vm} + { + } + + // TODO - get rid of circular dependencies in internals + void set_protocol(t_protocol_raw & protocol) + { + m_core.set_cryptonote_protocol(&protocol); + } + + void run() + { + //initialize core here + LOG_PRINT_L0("Initializing core..."); + if (!m_core.init(m_vm_HACK)) + { + throw std::runtime_error("Failed to initialize core"); + } + LOG_PRINT_L0("Core initialized OK"); + } + + cryptonote::core & get() + { + return m_core; + } + + ~t_core() + { + LOG_PRINT_L0("Deinitializing core..."); + try { + m_core.deinit(); + m_core.set_cryptonote_protocol(nullptr); + } catch (...) { + LOG_PRINT_L0("Failed to deinitialize core..."); + } + } +}; + +} diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index da09acfd4..7931ba03f 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -1,21 +1,21 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. -// +// // Redistribution and use in source and binary forms, with or without modification, are // permitted provided that the following conditions are met: -// +// // 1. Redistributions of source code must retain the above copyright notice, this list of // conditions and the following disclaimer. -// +// // 2. Redistributions in binary form must reproduce the above copyright notice, this list // of conditions and the following disclaimer in the documentation and/or other // materials provided with the distribution. -// +// // 3. Neither the name of the copyright holder nor the names of its contributors may be // used to endorse or promote products derived from this software without specific // prior written permission. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL @@ -25,278 +25,142 @@ // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// +// // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers -// node.cpp : Defines the entry point for the console application. -// Does this file exist? - +#include "daemon/daemon.h" -#include "include_base_utils.h" +#include "common/util.h" +#include "daemon/core.h" +#include "daemon/p2p.h" +#include "daemon/protocol.h" +#include "daemon/rpc.h" +#include "daemon/command_server.h" +#include "misc_log_ex.h" #include "version.h" +#include "../../contrib/epee/include/syncobj.h" using namespace epee; #include <boost/program_options.hpp> +#include <functional> +#include <memory> + +unsigned int epee::g_test_dbg_lock_sleep = 0; + +namespace daemonize { + +struct t_internals { +private: + t_protocol protocol; +public: + t_core core; + t_p2p p2p; + t_rpc rpc; + + t_internals( + boost::program_options::variables_map const & vm + ) + : core{vm} + , protocol{vm, core} + , p2p{vm, protocol} + , rpc{vm, core, p2p} + { + // Handle circular dependencies + protocol.set_p2p_endpoint(p2p.get()); + core.set_protocol(protocol.get()); + } +}; -#include "crypto/hash.h" -#include "console_handler.h" -#include "p2p/net_node.h" -#include "cryptonote_config.h" -#include "cryptonote_core/checkpoints_create.h" -#include "cryptonote_core/checkpoints.h" -#include "cryptonote_core/cryptonote_core.h" -#include "rpc/core_rpc_server.h" -#include "cryptonote_protocol/cryptonote_protocol_handler.h" -#include "daemon_commands_handler.h" -#include "version.h" +void t_daemon::init_options(boost::program_options::options_description & option_spec) +{ + t_core::init_options(option_spec); + t_p2p::init_options(option_spec); + t_rpc::init_options(option_spec); +} -#if defined(WIN32) -#include <crtdbg.h> -#endif +t_daemon::t_daemon( + boost::program_options::variables_map const & vm + ) + : mp_internals{new t_internals{vm}} +{} -namespace po = boost::program_options; +t_daemon::~t_daemon() = default; -namespace +// MSVC is brain-dead and can't default this... +t_daemon::t_daemon(t_daemon && other) { - const command_line::arg_descriptor<std::string> arg_config_file = {"config-file", "Specify configuration file", std::string(CRYPTONOTE_NAME ".conf")}; - const command_line::arg_descriptor<bool> arg_os_version = {"os-version", ""}; - const command_line::arg_descriptor<std::string> arg_log_file = {"log-file", "", ""}; - const command_line::arg_descriptor<int> arg_log_level = {"log-level", "", LOG_LEVEL_0}; - const command_line::arg_descriptor<bool> arg_console = {"no-console", "Disable daemon console commands"}; - const command_line::arg_descriptor<bool> arg_testnet_on = { - "testnet" - , "Run on testnet. The wallet must be launched with --testnet flag." - , false - }; - const command_line::arg_descriptor<bool> arg_dns_checkpoints = {"enforce-dns-checkpointing", "checkpoints from DNS server will be enforced", false}; + if (this != &other) + { + mp_internals = std::move(other.mp_internals); + other.mp_internals.reset(nullptr); + } } -bool command_line_preprocessor(const boost::program_options::variables_map& vm) +// or this +t_daemon & t_daemon::operator=(t_daemon && other) { - bool exit = false; - if (command_line::get_arg(vm, command_line::arg_version)) - { - std::cout << CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL << ENDL; - exit = true; - } - if (command_line::get_arg(vm, arg_os_version)) + if (this != &other) { - std::cout << "OS: " << tools::get_os_version_string() << ENDL; - exit = true; + mp_internals = std::move(other.mp_internals); + other.mp_internals.reset(nullptr); } + return *this; +} - if (exit) +bool t_daemon::run(bool interactive) +{ + if (nullptr == mp_internals) { - return true; + throw std::runtime_error{"Can't run stopped daemon"}; } + tools::signal_handler::install(std::bind(&daemonize::t_daemon::stop, this)); - int new_log_level = command_line::get_arg(vm, arg_log_level); - if(new_log_level < LOG_LEVEL_MIN || new_log_level > LOG_LEVEL_MAX) + try { - LOG_PRINT_L0("Wrong log level value: "); - } - else if (log_space::get_set_log_detalisation_level(false) != new_log_level) - { - log_space::get_set_log_detalisation_level(true, new_log_level); - LOG_PRINT_L0("LOG_LEVEL set to " << new_log_level); - } - - return false; -} - -int main(int argc, char* argv[]) -{ - - string_tools::set_module_name_and_folder(argv[0]); -#ifdef WIN32 - _CrtSetDbgFlag ( _CRTDBG_ALLOC_MEM_DF | _CRTDBG_LEAK_CHECK_DF ); -#endif - log_space::get_set_log_detalisation_level(true, LOG_LEVEL_0); - log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL); - LOG_PRINT_L0("Starting..."); + mp_internals->core.run(); + mp_internals->rpc.run(); - TRY_ENTRY(); + daemonize::t_command_server* rpc_commands; - boost::filesystem::path default_data_path {tools::get_default_data_dir()}; - boost::filesystem::path default_testnet_data_path {default_data_path / "testnet"}; + if (interactive) + { + rpc_commands = new daemonize::t_command_server(0, 0, false, mp_internals->rpc.get_server()); + rpc_commands->start_handling(); + } - po::options_description desc_cmd_only("Command line options"); - po::options_description desc_cmd_sett("Command line options and settings options"); + mp_internals->p2p.run(); // blocks until p2p goes down - command_line::add_arg(desc_cmd_only, command_line::arg_help); - command_line::add_arg(desc_cmd_only, command_line::arg_version); - command_line::add_arg(desc_cmd_only, arg_os_version); - // tools::get_default_data_dir() can't be called during static initialization - command_line::add_arg(desc_cmd_only, command_line::arg_data_dir, default_data_path.string()); - command_line::add_arg(desc_cmd_only, command_line::arg_testnet_data_dir, default_testnet_data_path.string()); - command_line::add_arg(desc_cmd_only, arg_config_file); - - command_line::add_arg(desc_cmd_sett, arg_log_file); - command_line::add_arg(desc_cmd_sett, arg_log_level); - command_line::add_arg(desc_cmd_sett, arg_console); - command_line::add_arg(desc_cmd_sett, arg_testnet_on); - command_line::add_arg(desc_cmd_sett, arg_dns_checkpoints); - - cryptonote::core::init_options(desc_cmd_sett); - cryptonote::core_rpc_server::init_options(desc_cmd_sett); - nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<cryptonote::core> >::init_options(desc_cmd_sett); - cryptonote::miner::init_options(desc_cmd_sett); - - po::options_description desc_options("Allowed options"); - desc_options.add(desc_cmd_only).add(desc_cmd_sett); - - po::variables_map vm; - bool r = command_line::handle_error_helper(desc_options, [&]() - { - po::store(po::parse_command_line(argc, argv, desc_options), vm); - po::notify(vm); + if (interactive) + { + rpc_commands->stop_handling(); + } + mp_internals->rpc.stop(); + LOG_PRINT("Node stopped.", LOG_LEVEL_0); return true; - }); - if (!r) - return 1; - - if (command_line::get_arg(vm, command_line::arg_help)) - { - std::cout << CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL << ENDL << ENDL; - std::cout << desc_options << std::endl; - return false; } - - bool testnet_mode = command_line::get_arg(vm, arg_testnet_on); - - auto data_dir_arg = testnet_mode ? command_line::arg_testnet_data_dir : command_line::arg_data_dir; - - std::string data_dir = command_line::get_arg(vm, data_dir_arg); - tools::create_directories_if_necessary(data_dir); - std::string config = command_line::get_arg(vm, arg_config_file); - - boost::filesystem::path data_dir_path(data_dir); - boost::filesystem::path config_path(config); - if (!config_path.has_parent_path()) + catch (std::exception const & ex) { - config_path = data_dir_path / config_path; + LOG_ERROR("Uncaught exception! " << ex.what()); + return false; } - - boost::system::error_code ec; - if (boost::filesystem::exists(config_path, ec)) + catch (...) { - po::store(po::parse_config_file<char>(config_path.string<std::string>().c_str(), desc_cmd_sett), vm); - } - - //set up logging options - boost::filesystem::path log_file_path(command_line::get_arg(vm, arg_log_file)); - if (log_file_path.empty()) - log_file_path = log_space::log_singletone::get_default_log_file(); - std::string log_dir; - log_dir = log_file_path.has_parent_path() ? log_file_path.parent_path().string() : log_space::log_singletone::get_default_log_folder(); - - log_space::log_singletone::add_logger(LOGGER_FILE, log_file_path.filename().string().c_str(), log_dir.c_str()); - LOG_PRINT_L0(CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL); - - if (command_line_preprocessor(vm)) - { - return 0; - } - - LOG_PRINT("Module folder: " << argv[0], LOG_LEVEL_0); - - bool res = true; - cryptonote::checkpoints checkpoints; - res = cryptonote::create_checkpoints(checkpoints); - CHECK_AND_ASSERT_MES(res, 1, "Failed to initialize checkpoints"); - boost::filesystem::path json(JSON_HASH_FILE_NAME); - boost::filesystem::path checkpoint_json_hashfile_fullpath = data_dir / json; - - //create objects and link them - cryptonote::core ccore(NULL); - - // tell core if we're enforcing dns checkpoints - bool enforce_dns = command_line::get_arg(vm, arg_dns_checkpoints); - ccore.set_enforce_dns_checkpoints(enforce_dns); - - if (testnet_mode) { - LOG_PRINT_L0("Starting in testnet mode!"); - } else { - ccore.set_checkpoints(std::move(checkpoints)); - ccore.set_checkpoints_file_path(checkpoint_json_hashfile_fullpath.string()); + LOG_ERROR("Uncaught exception!"); + return false; } +} - cryptonote::t_cryptonote_protocol_handler<cryptonote::core> cprotocol(ccore, NULL); - nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<cryptonote::core> > p2psrv { - cprotocol - , testnet_mode ? std::move(config::testnet::NETWORK_ID) : std::move(config::NETWORK_ID) - }; - cryptonote::core_rpc_server rpc_server {ccore, p2psrv, testnet_mode}; - cprotocol.set_p2p_endpoint(&p2psrv); - ccore.set_cryptonote_protocol(&cprotocol); - daemon_cmmands_handler dch(p2psrv, testnet_mode); - - //initialize objects - LOG_PRINT_L0("Initializing P2P server..."); - res = p2psrv.init(vm, testnet_mode); - CHECK_AND_ASSERT_MES(res, 1, "Failed to initialize P2P server."); - LOG_PRINT_L0("P2P server initialized OK"); - - LOG_PRINT_L0("Initializing protocol..."); - res = cprotocol.init(vm); - CHECK_AND_ASSERT_MES(res, 1, "Failed to initialize protocol."); - LOG_PRINT_L0("Protocol initialized OK"); - - LOG_PRINT_L0("Initializing core RPC server..."); - res = rpc_server.init(vm); - CHECK_AND_ASSERT_MES(res, 1, "Failed to initialize core RPC server."); - LOG_PRINT_GREEN("Core RPC server initialized OK on port: " << rpc_server.get_binded_port(), LOG_LEVEL_0); - - //initialize core here - LOG_PRINT_L0("Initializing core..."); - res = ccore.init(vm, testnet_mode); - CHECK_AND_ASSERT_MES(res, 1, "Failed to initialize core"); - LOG_PRINT_L0("Core initialized OK"); - - // start components - if(!command_line::has_arg(vm, arg_console)) +void t_daemon::stop() +{ + if (nullptr == mp_internals) { - dch.start_handling(); + throw std::runtime_error{"Can't stop stopped daemon"}; } - - LOG_PRINT_L0("Starting core RPC server..."); - res = rpc_server.run(2, false); - CHECK_AND_ASSERT_MES(res, 1, "Failed to initialize core RPC server."); - LOG_PRINT_L0("Core RPC server started ok"); - - tools::signal_handler::install([&dch, &p2psrv] { - dch.stop_handling(); - p2psrv.send_stop_signal(); - }); - - LOG_PRINT_L0("Starting P2P net loop..."); - p2psrv.run(); - LOG_PRINT_L0("P2P net loop stopped"); - - //stop components - LOG_PRINT_L0("Stopping core rpc server..."); - rpc_server.send_stop_signal(); - rpc_server.timed_wait_server_stop(5000); - - //deinitialize components - LOG_PRINT_L0("Deinitializing core..."); - ccore.deinit(); - LOG_PRINT_L0("Deinitializing RPC server ..."); - rpc_server.deinit(); - LOG_PRINT_L0("Deinitializing protocol..."); - cprotocol.deinit(); - LOG_PRINT_L0("Deinitializing P2P..."); - p2psrv.deinit(); - - - ccore.set_cryptonote_protocol(NULL); - cprotocol.set_p2p_endpoint(NULL); - - LOG_PRINT("Node stopped.", LOG_LEVEL_0); - return 0; - - CATCH_ENTRY_L0("main", 1); + mp_internals->p2p.stop(); + mp_internals->rpc.stop(); + mp_internals.reset(nullptr); // Ensure resources are cleaned up before we return } +} // namespace daemonize diff --git a/src/daemon/daemon.h b/src/daemon/daemon.h new file mode 100644 index 000000000..b0869b25b --- /dev/null +++ b/src/daemon/daemon.h @@ -0,0 +1,54 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <memory> +#include <boost/program_options.hpp> + +namespace daemonize { + +class t_internals; + +class t_daemon final { +public: + static void init_options(boost::program_options::options_description & option_spec); +private: + std::unique_ptr<t_internals> mp_internals; +public: + t_daemon( + boost::program_options::variables_map const & vm + ); + t_daemon(t_daemon && other); + t_daemon & operator=(t_daemon && other); + ~t_daemon(); + + bool run(bool interactive = false); + void stop(); +}; +} diff --git a/src/daemon/daemon_commands_handler.h b/src/daemon/daemon_commands_handler.h index 869c2adf7..215cf26de 100644 --- a/src/daemon/daemon_commands_handler.h +++ b/src/daemon/daemon_commands_handler.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -40,22 +40,20 @@ #include "common/util.h" #include "crypto/hash.h" #include "version.h" +#include "../../contrib/otshell_utils/utils.hpp" + +//#include "net/net_helper.h" +//#include "../p2p/p2p_protocol_defs.h" +//#include "../p2p/net_peerlist_boost_serialization.h" +//#include "net/local_ip.h" +//#include "crypto/crypto.h" +//#include "storages/levin_abstract_invoke2.h" -/*! - * \brief I don't really know right now - * - * - */ class daemon_cmmands_handler { nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<cryptonote::core> >& m_srv; public: - daemon_cmmands_handler( - nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<cryptonote::core> >& srv - , bool testnet - ) - : m_srv(srv) - , m_testnet(testnet) + daemon_cmmands_handler(nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<cryptonote::core> >& srv):m_srv(srv) { m_cmd_binder.set_handler("help", boost::bind(&daemon_cmmands_handler::help, this, _1), "Show this help"); m_cmd_binder.set_handler("print_pl", boost::bind(&daemon_cmmands_handler::print_pl, this, _1), "Print peer list"); @@ -74,6 +72,14 @@ public: m_cmd_binder.set_handler("save", boost::bind(&daemon_cmmands_handler::save, this, _1), "Save blockchain"); m_cmd_binder.set_handler("set_log", boost::bind(&daemon_cmmands_handler::set_log, this, _1), "set_log <level> - Change current log detalization level, <level> is a number 0-4"); m_cmd_binder.set_handler("diff", boost::bind(&daemon_cmmands_handler::diff, this, _1), "Show difficulty"); + m_cmd_binder.set_handler("out_peers", boost::bind(&daemon_cmmands_handler::out_peers_limit, this, _1), "Set max limit of out peers"); + m_cmd_binder.set_handler("limit_up", boost::bind(&daemon_cmmands_handler::limit_up, this, _1), "Set upload limit [kB/s]"); + m_cmd_binder.set_handler("limit_down", boost::bind(&daemon_cmmands_handler::limit_down, this, _1), "Set download limit [kB/s]"); + m_cmd_binder.set_handler("limit", boost::bind(&daemon_cmmands_handler::limit, this, _1), "Set download and upload limit [kB/s]"); + m_cmd_binder.set_handler("fast_exit", boost::bind(&daemon_cmmands_handler::fast_exit, this, _1), "Exit"); + m_cmd_binder.set_handler("test_drop_download", boost::bind(&daemon_cmmands_handler::test_drop_download, this, _1), "For network testing, drop downloaded blocks instead checking/adding them to blockchain. Can fake-download blocks very fast."); + m_cmd_binder.set_handler("start_save_graph", boost::bind(&daemon_cmmands_handler::start_save_graph, this, _1), ""); + m_cmd_binder.set_handler("stop_save_graph", boost::bind(&daemon_cmmands_handler::stop_save_graph, this, _1), ""); } bool start_handling() @@ -89,7 +95,7 @@ public: private: epee::srv_console_handlers_binder<nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<cryptonote::core> > > m_cmd_binder; - bool m_testnet; + //-------------------------------------------------------------------------------- std::string get_commands_str() @@ -122,6 +128,122 @@ private: return true; } //-------------------------------------------------------------------------------- + bool limit_up(const std::vector<std::string>& args) + { + if(args.size()!=1) { + std::cout << "Usage: limit_up <speed>" << ENDL; + return false; + } + + int limit; + try { + limit = std::stoi(args[0]); + } + catch(std::invalid_argument& ex) { + return false; + } + + if (limit==-1) { + limit=128; + //this->islimitup=false; + } + + limit *= 1024; + + + //nodetool::epee::net_utils::connection<epee::levin::async_protocol_handler<nodetool::p2p_connection_context> >::set_rate_up_limit( limit ); + epee::net_utils::connection_basic::set_rate_up_limit( limit ); + std::cout << "Set limit-up to " << limit/1024 << " kB/s" << std::endl; + + return true; + } + + //-------------------------------------------------------------------------------- + bool limit_down(const std::vector<std::string>& args) + { + + if(args.size()!=1) { + std::cout << "Usage: limit_down <speed>" << ENDL; + return true; + } + + int limit; + try { + limit = std::stoi(args[0]); + } + + catch(std::invalid_argument& ex) { + return false; + } + + if (limit==-1) { + limit=128; + //this->islimitup=false; + } + + limit *= 1024; + + + //nodetool::epee::net_utils::connection<epee::levin::async_protocol_handler<nodetool::p2p_connection_context> >::set_rate_up_limit( limit ); + epee::net_utils::connection_basic::set_rate_down_limit( limit ); + std::cout << "Set limit-down to " << limit/1024 << " kB/s" << std::endl; + + return true; + } + +//-------------------------------------------------------------------------------- + bool limit(const std::vector<std::string>& args) + { + if(args.size()!=1) { + std::cout << "Usage: limit_down <speed>" << ENDL; + return true; + } + + int limit; + try { + limit = std::stoi(args[0]); + } + catch(std::invalid_argument& ex) { + return false; + } + + if (limit==-1) { + limit=128; + //this->islimitup=false; + } + + limit *= 1024; + + + //nodetool::epee::net_utils::connection<epee::levin::async_protocol_handler<nodetool::p2p_connection_context> >::set_rate_up_limit( limit ); + epee::net_utils::connection_basic::set_rate_down_limit( limit ); + epee::net_utils::connection_basic::set_rate_up_limit( limit ); + std::cout << "Set limit-down to " << limit/1024 << " kB/s" << std::endl; + std::cout << "Set limit-up to " << limit/1024 << " kB/s" << std::endl; + + return true; + } + //-------------------------------------------------------------------------------- + bool out_peers_limit(const std::vector<std::string>& args) { + if(args.size()!=1) { + std::cout << "Usage: limit_down <speed>" << ENDL; + return true; + } + + int limit; + try { + limit = std::stoi(args[0]); + } + + catch(std::invalid_argument& ex) { + return false; + } + + LOG_PRINT_RED_L0("connections_count: " << limit); + m_srv.m_config.m_net_config.connections_count = limit; + return true; + } + //-------------------------------------------------------------------------------- bool show_hr(const std::vector<std::string>& args) { if(!m_srv.get_payload_object().get_core().get_miner().is_mining()) @@ -234,7 +356,13 @@ private: return true; } + // TODO what the hell causes compilation warning in following code line +PUSH_WARNINGS +DISABLE_GCC_WARNING(maybe-uninitialized) log_space::log_singletone::get_set_log_detalisation_level(true, l); + int otshell_utils_log_level = 100 - (l * 25); + gCurrentLogger.setDebugLevel(otshell_utils_log_level); +POP_WARNINGS return true; } @@ -374,7 +502,7 @@ private: } cryptonote::account_public_address adr; - if(!cryptonote::get_account_address_from_str(adr, m_testnet, args.front())) + if(!cryptonote::get_account_address_from_str(adr, args.front())) { std::cout << "target account address has wrong format" << std::endl; return true; diff --git a/src/daemon/executor.cpp b/src/daemon/executor.cpp new file mode 100644 index 000000000..bb188c174 --- /dev/null +++ b/src/daemon/executor.cpp @@ -0,0 +1,71 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "daemon/executor.h" + +#include "misc_log_ex.h" + +#include "common/command_line.h" +#include "cryptonote_config.h" +#include "version.h" + +#include <string> + +namespace daemonize +{ + std::string const t_executor::NAME = "Monero Daemon"; + + void t_executor::init_options( + boost::program_options::options_description & configurable_options + ) + { + t_daemon::init_options(configurable_options); + } + + std::string const & t_executor::name() + { + return NAME; + } + + t_daemon t_executor::create_daemon( + boost::program_options::variables_map const & vm + ) + { + LOG_PRINT_L0(CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL); + return t_daemon{vm}; + } + + bool t_executor::run_interactive( + boost::program_options::variables_map const & vm + ) + { + epee::log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL); + return t_daemon{vm}.run(true); + } +} + diff --git a/src/daemon/executor.h b/src/daemon/executor.h new file mode 100644 index 000000000..553896875 --- /dev/null +++ b/src/daemon/executor.h @@ -0,0 +1,60 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include "daemon/daemon.h" +#include <boost/program_options/options_description.hpp> +#include <boost/program_options/variables_map.hpp> +#include <string> +#include <vector> + +namespace daemonize +{ + class t_executor final + { + public: + typedef ::daemonize::t_daemon t_daemon; + + static std::string const NAME; + + static void init_options( + boost::program_options::options_description & configurable_options + ); + + std::string const & name(); + + t_daemon create_daemon( + boost::program_options::variables_map const & vm + ); + + bool run_interactive( + boost::program_options::variables_map const & vm + ); + }; +} diff --git a/src/daemon/main.cpp b/src/daemon/main.cpp new file mode 100644 index 000000000..62b655ccd --- /dev/null +++ b/src/daemon/main.cpp @@ -0,0 +1,265 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#include "common/command_line.h" +#include "common/scoped_message_writer.h" +#include "common/util.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_core/miner.h" +#include "daemon/command_server.h" +#include "daemon/daemon.h" +#include "daemon/executor.h" +#include "daemonizer/daemonizer.h" +#include "misc_log_ex.h" +#include "p2p/net_node.h" +#include "rpc/core_rpc_server.h" +#include <boost/program_options.hpp> +#include "daemon/command_line_args.h" +#include "blockchain_db/db_types.h" + +namespace po = boost::program_options; +namespace bf = boost::filesystem; + +int main(int argc, char const * argv[]) +{ + try { + + _note_c("dbg/main", "Begin of main()"); + // TODO parse the debug options like set log level right here at start + + epee::string_tools::set_module_name_and_folder(argv[0]); + + // Build argument description + po::options_description all_options("All"); + po::options_description hidden_options("Hidden"); + po::options_description visible_options("Options"); + po::options_description core_settings("Settings"); + po::positional_options_description positional_options; + { + bf::path default_data_dir = daemonizer::get_default_data_dir(); + bf::path default_testnet_data_dir = {default_data_dir / "testnet"}; + + // Misc Options + + command_line::add_arg(visible_options, command_line::arg_help); + command_line::add_arg(visible_options, command_line::arg_version); + command_line::add_arg(visible_options, daemon_args::arg_os_version); + command_line::add_arg(visible_options, command_line::arg_data_dir, default_data_dir.string()); + command_line::add_arg(visible_options, command_line::arg_testnet_data_dir, default_testnet_data_dir.string()); + bf::path default_conf = default_data_dir / std::string(CRYPTONOTE_NAME ".conf"); + command_line::add_arg(visible_options, daemon_args::arg_config_file, default_conf.string()); + command_line::add_arg(visible_options, command_line::arg_test_drop_download); + command_line::add_arg(visible_options, command_line::arg_test_dbg_lock_sleep); + command_line::add_arg(visible_options, command_line::arg_test_drop_download_height); + + // Settings + bf::path default_log = default_data_dir / std::string(CRYPTONOTE_NAME ".log"); + command_line::add_arg(core_settings, daemon_args::arg_log_file, default_log.string()); + command_line::add_arg(core_settings, daemon_args::arg_log_level); + command_line::add_arg(core_settings, daemon_args::arg_testnet_on); + command_line::add_arg(core_settings, daemon_args::arg_dns_checkpoints); + command_line::add_arg(core_settings, daemon_args::arg_db_type); + daemonizer::init_options(hidden_options, visible_options); + daemonize::t_executor::init_options(core_settings); + + // Hidden options + command_line::add_arg(hidden_options, daemon_args::arg_command); + + visible_options.add(core_settings); + all_options.add(visible_options); + all_options.add(hidden_options); + + // Positional + positional_options.add(daemon_args::arg_command.name, -1); // -1 for unlimited arguments + } + + // Do command line parsing + po::variables_map vm; + bool ok = command_line::handle_error_helper(visible_options, [&]() + { + boost::program_options::store( + boost::program_options::command_line_parser(argc, argv) + .options(all_options).positional(positional_options).run() + , vm + ); + + return true; + }); + if (!ok) return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL << ENDL << ENDL; + std::cout << "Usage: " + std::string{argv[0]} + " [options|settings] [daemon_command...]" << std::endl << std::endl; + std::cout << visible_options << std::endl; + return 0; + } + + // Monero Version + if (command_line::get_arg(vm, command_line::arg_version)) + { + std::cout << CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL << ENDL; + return 0; + } + + // OS + if (command_line::get_arg(vm, daemon_args::arg_os_version)) + { + std::cout << "OS: " << tools::get_os_version_string() << ENDL; + return 0; + } + + epee::g_test_dbg_lock_sleep = command_line::get_arg(vm, command_line::arg_test_dbg_lock_sleep); + + std::string db_type = command_line::get_arg(vm, daemon_args::arg_db_type); + + // verify that blockchaindb type is valid + if(cryptonote::blockchain_db_types.count(db_type) == 0) + { + std::cout << "Invalid database type (" << db_type << "), available types are:" << std::endl; + for (const auto& type : cryptonote::blockchain_db_types) + { + std::cout << "\t" << type << std::endl; + } + return 0; + } + + bool testnet_mode = command_line::get_arg(vm, daemon_args::arg_testnet_on); + + auto data_dir_arg = testnet_mode ? command_line::arg_testnet_data_dir : command_line::arg_data_dir; + + // Create data dir if it doesn't exist + boost::filesystem::path data_dir = boost::filesystem::absolute( + command_line::get_arg(vm, data_dir_arg)); + tools::create_directories_if_necessary(data_dir.string()); + + // FIXME: not sure on windows implementation default, needs further review + //bf::path relative_path_base = daemonizer::get_relative_path_base(vm); + bf::path relative_path_base = data_dir; + + std::string config = command_line::get_arg(vm, daemon_args::arg_config_file); + + boost::filesystem::path data_dir_path(data_dir); + boost::filesystem::path config_path(config); + if (!config_path.has_parent_path()) + { + config_path = data_dir / config_path; + } + + boost::system::error_code ec; + if (bf::exists(config_path, ec)) + { + po::store(po::parse_config_file<char>(config_path.string<std::string>().c_str(), core_settings), vm); + } + po::notify(vm); + + // If there are positional options, we're running a daemon command + { + auto command = command_line::get_arg(vm, daemon_args::arg_command); + + if (command.size()) + { + auto rpc_ip_str = command_line::get_arg(vm, cryptonote::core_rpc_server::arg_rpc_bind_ip); + auto rpc_port_str = command_line::get_arg(vm, cryptonote::core_rpc_server::arg_rpc_bind_port); + if (testnet_mode) + { + rpc_port_str = command_line::get_arg(vm, cryptonote::core_rpc_server::arg_testnet_rpc_bind_port); + } + + uint32_t rpc_ip; + uint16_t rpc_port; + if (!epee::string_tools::get_ip_int32_from_string(rpc_ip, rpc_ip_str)) + { + std::cerr << "Invalid IP: " << rpc_ip_str << std::endl; + return 1; + } + if (!epee::string_tools::get_xtype_from_string(rpc_port, rpc_port_str)) + { + std::cerr << "Invalid port: " << rpc_port_str << std::endl; + return 1; + } + + daemonize::t_command_server rpc_commands{rpc_ip, rpc_port}; + if (rpc_commands.process_command_vec(command)) + { + return 0; + } + else + { + std::cerr << "Unknown command" << std::endl; + return 1; + } + } + } + + // Start with log level 0 + epee::log_space::get_set_log_detalisation_level(true, LOG_LEVEL_0); + + // Set log level + { + int new_log_level = command_line::get_arg(vm, daemon_args::arg_log_level); + if(new_log_level < LOG_LEVEL_MIN || new_log_level > LOG_LEVEL_MAX) + { + LOG_PRINT_L0("Wrong log level value: " << new_log_level); + } + else if (epee::log_space::get_set_log_detalisation_level(false) != new_log_level) + { + epee::log_space::get_set_log_detalisation_level(true, new_log_level); + int otshell_utils_log_level = 100 - (new_log_level * 20); + gCurrentLogger.setDebugLevel(otshell_utils_log_level); + LOG_PRINT_L0("LOG_LEVEL set to " << new_log_level); + } + } + + // Set log file + { + bf::path log_file_path{bf::absolute(command_line::get_arg(vm, daemon_args::arg_log_file), relative_path_base)}; + + epee::log_space::log_singletone::add_logger( + LOGGER_FILE + , log_file_path.filename().string().c_str() + , log_file_path.parent_path().string().c_str() + ); + } + + _note_c("dbg/main", "Moving from main() into the daemonize now."); + + return daemonizer::daemonize(argc, argv, daemonize::t_executor{}, vm); + } + catch (std::exception const & ex) + { + LOG_ERROR("Exception in main! " << ex.what()); + } + catch (...) + { + LOG_ERROR("Exception in main!"); + } + return 1; +} diff --git a/src/daemon/p2p.h b/src/daemon/p2p.h new file mode 100644 index 000000000..355a784b1 --- /dev/null +++ b/src/daemon/p2p.h @@ -0,0 +1,99 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#pragma once + +#include "cryptonote_protocol/cryptonote_protocol_handler.h" +#include "daemon/protocol.h" +#include "misc_log_ex.h" +#include "p2p/net_node.h" +#include <stdexcept> +#include <boost/program_options.hpp> + +namespace daemonize +{ + +class t_p2p final +{ +private: + typedef cryptonote::t_cryptonote_protocol_handler<cryptonote::core> t_protocol_raw; + typedef nodetool::node_server<t_protocol_raw> t_node_server; +public: + static void init_options(boost::program_options::options_description & option_spec) + { + t_node_server::init_options(option_spec); + } +private: + t_node_server m_server; +public: + t_p2p( + boost::program_options::variables_map const & vm + , t_protocol & protocol + ) + : m_server{protocol.get()} + { + //initialize objects + LOG_PRINT_L0("Initializing p2p server..."); + if (!m_server.init(vm)) + { + throw std::runtime_error("Failed to initialize p2p server."); + } + LOG_PRINT_L0("P2p server initialized OK"); + } + + t_node_server & get() + { + return m_server; + } + + void run() + { + LOG_PRINT_L0("Starting p2p net loop..."); + m_server.run(); + LOG_PRINT_L0("p2p net loop stopped"); + } + + void stop() + { + m_server.send_stop_signal(); + } + + ~t_p2p() + { + LOG_PRINT_L0("Deinitializing p2p..."); + try { + m_server.deinit(); + } catch (...) { + LOG_PRINT_L0("Failed to deinitialize p2p..."); + } + } +}; + +} diff --git a/src/daemon/protocol.h b/src/daemon/protocol.h new file mode 100644 index 000000000..2f6a66f49 --- /dev/null +++ b/src/daemon/protocol.h @@ -0,0 +1,88 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#pragma once + +#include "cryptonote_protocol/cryptonote_protocol_handler.h" +#include "misc_log_ex.h" +#include "p2p/net_node.h" +#include <stdexcept> +#include <boost/program_options.hpp> + +namespace daemonize +{ + +class t_protocol final +{ +private: + typedef cryptonote::t_cryptonote_protocol_handler<cryptonote::core> t_protocol_raw; + typedef nodetool::node_server<t_protocol_raw> t_node_server; + + t_protocol_raw m_protocol; +public: + t_protocol( + boost::program_options::variables_map const & vm + , t_core & core + ) + : m_protocol{core.get(), nullptr} + { + LOG_PRINT_L0("Initializing cryptonote protocol..."); + if (!m_protocol.init(vm)) + { + throw std::runtime_error("Failed to initialize cryptonote protocol."); + } + LOG_PRINT_L0("Cryptonote protocol initialized OK"); + } + + t_protocol_raw & get() + { + return m_protocol; + } + + void set_p2p_endpoint( + t_node_server & server + ) + { + m_protocol.set_p2p_endpoint(&server); + } + + ~t_protocol() + { + LOG_PRINT_L0("Deinitializing cryptonote_protocol..."); + try { + m_protocol.deinit(); + m_protocol.set_p2p_endpoint(nullptr); + } catch (...) { + LOG_PRINT_L0("Failed to deinitialize protocol..."); + } + } +}; + +} diff --git a/src/daemon/rpc.h b/src/daemon/rpc.h new file mode 100644 index 000000000..6477f440e --- /dev/null +++ b/src/daemon/rpc.h @@ -0,0 +1,101 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#pragma once + +#include "daemon/core.h" +#include "daemon/p2p.h" +#include "misc_log_ex.h" +#include "rpc/core_rpc_server.h" +#include <boost/program_options.hpp> +#include <stdexcept> + +namespace daemonize +{ + +class t_rpc final +{ +public: + static void init_options(boost::program_options::options_description & option_spec) + { + cryptonote::core_rpc_server::init_options(option_spec); + } +private: + cryptonote::core_rpc_server m_server; +public: + t_rpc( + boost::program_options::variables_map const & vm + , t_core & core + , t_p2p & p2p + ) + : m_server{core.get(), p2p.get()} + { + LOG_PRINT_L0("Initializing core rpc server..."); + if (!m_server.init(vm)) + { + throw std::runtime_error("Failed to initialize core rpc server."); + } + LOG_PRINT_GREEN("Core rpc server initialized OK on port: " << m_server.get_binded_port(), LOG_LEVEL_0); + } + + void run() + { + LOG_PRINT_L0("Starting core rpc server..."); + if (!m_server.run(2, false)) + { + throw std::runtime_error("Failed to start core rpc server."); + } + LOG_PRINT_L0("Core rpc server started ok"); + } + + void stop() + { + LOG_PRINT_L0("Stopping core rpc server..."); + m_server.send_stop_signal(); + m_server.timed_wait_server_stop(5000); + } + + cryptonote::core_rpc_server* get_server() + { + return &m_server; + } + + ~t_rpc() + { + LOG_PRINT_L0("Deinitializing rpc server..."); + try { + m_server.deinit(); + } catch (...) { + LOG_PRINT_L0("Failed to deinitialize rpc server..."); + } + } +}; + +} diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp new file mode 100644 index 000000000..8ef91600c --- /dev/null +++ b/src/daemon/rpc_command_executor.cpp @@ -0,0 +1,872 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#include "string_tools.h" +#include "common/scoped_message_writer.h" +#include "daemon/rpc_command_executor.h" +#include "rpc/core_rpc_server_commands_defs.h" +#include "cryptonote_core/cryptonote_core.h" +#include <boost/format.hpp> +#include <ctime> +#include <string> + +namespace daemonize { + +namespace { + void print_peer(std::string const & prefix, cryptonote::peer const & peer) + { + time_t now; + time(&now); + time_t last_seen = static_cast<time_t>(peer.last_seen); + + std::string id_str; + std::string port_str; + std::string elapsed = epee::misc_utils::get_time_interval_string(now - last_seen); + std::string ip_str = epee::string_tools::get_ip_string_from_int32(peer.ip); + epee::string_tools::xtype_to_string(peer.id, id_str); + epee::string_tools::xtype_to_string(peer.port, port_str); + std::string addr_str = ip_str + ":" + port_str; + tools::msg_writer() << boost::format("%-10s %-25s %-25s %s") % prefix % id_str % addr_str % elapsed; + } + + void print_block_header(cryptonote::block_header_responce const & header) + { + tools::success_msg_writer() + << "timestamp: " << boost::lexical_cast<std::string>(header.timestamp) << std::endl + << "previous hash: " << header.prev_hash << std::endl + << "nonce: " << boost::lexical_cast<std::string>(header.nonce) << std::endl + << "is orphan: " << header.orphan_status << std::endl + << "height: " << boost::lexical_cast<std::string>(header.height) << std::endl + << "depth: " << boost::lexical_cast<std::string>(header.depth) << std::endl + << "hash: " << header.hash + << "difficulty: " << boost::lexical_cast<std::string>(header.difficulty) << std::endl + << "reward: " << boost::lexical_cast<std::string>(header.reward); + } +} + +t_rpc_command_executor::t_rpc_command_executor( + uint32_t ip + , uint16_t port + , bool is_rpc + , cryptonote::core_rpc_server* rpc_server + ) + : m_rpc_client(NULL), m_rpc_server(rpc_server) +{ + if (is_rpc) + { + m_rpc_client = new tools::t_rpc_client(ip, port); + } + else + { + if (rpc_server == NULL) + { + throw std::runtime_error("If not calling commands via RPC, rpc_server pointer must be non-null"); + } + } + + m_is_rpc = is_rpc; +} + +t_rpc_command_executor::~t_rpc_command_executor() +{ + if (m_rpc_client != NULL) + { + delete m_rpc_client; + } +} + +bool t_rpc_command_executor::print_peer_list() { + cryptonote::COMMAND_RPC_GET_PEER_LIST::request req; + cryptonote::COMMAND_RPC_GET_PEER_LIST::response res; + + std::string failure_message = "Couldn't retrieve peer list"; + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/get_peer_list", failure_message.c_str())) + { + return false; + } + } + else + { + if (!m_rpc_server->on_get_peer_list(req, res)) + { + tools::fail_msg_writer() << failure_message; + return false; + } + } + + for (auto & peer : res.white_list) + { + print_peer("white", peer); + } + + for (auto & peer : res.gray_list) + { + print_peer("gray", peer); + } + + return true; +} + +bool t_rpc_command_executor::save_blockchain() { + cryptonote::COMMAND_RPC_SAVE_BC::request req; + cryptonote::COMMAND_RPC_SAVE_BC::response res; + + std::string fail_message = "Couldn't save blockchain"; + + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/save_bc", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_save_bc(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + } + } + + tools::success_msg_writer() << "Blockchain saved"; + + return true; +} + +bool t_rpc_command_executor::show_hash_rate() { + cryptonote::COMMAND_RPC_SET_LOG_HASH_RATE::request req; + cryptonote::COMMAND_RPC_SET_LOG_HASH_RATE::response res; + req.visible = true; + + std::string fail_message = "Unsuccessful"; + + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/set_log_hash_rate", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_set_log_hash_rate(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + } + } + + tools::success_msg_writer() << "Hash rate logging is on"; + + return true; +} + +bool t_rpc_command_executor::hide_hash_rate() { + cryptonote::COMMAND_RPC_SET_LOG_HASH_RATE::request req; + cryptonote::COMMAND_RPC_SET_LOG_HASH_RATE::response res; + req.visible = false; + + std::string fail_message = "Unsuccessful"; + + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/set_log_hash_rate", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_set_log_hash_rate(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + tools::success_msg_writer() << "Hash rate logging is off"; + + return true; +} + +bool t_rpc_command_executor::show_difficulty() { + cryptonote::COMMAND_RPC_GET_INFO::request req; + cryptonote::COMMAND_RPC_GET_INFO::response res; + + std::string fail_message = "Problem fetching info"; + + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/getinfo", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_get_info(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + tools::success_msg_writer() << "BH: " << res.height + << ", DIFF: " << res.difficulty + << ", HR: " << (int) res.difficulty / 60L << " H/s"; + + return true; +} + +bool t_rpc_command_executor::print_connections() { + cryptonote::COMMAND_RPC_GET_CONNECTIONS::request req; + cryptonote::COMMAND_RPC_GET_CONNECTIONS::response res; + epee::json_rpc::error error_resp; + + std::string fail_message = "Unsuccessful"; + + if (m_is_rpc) + { + if (!m_rpc_client->json_rpc_request(req, res, "/get_connections", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_get_connections(req, res, error_resp)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + tools::msg_writer() << std::setw(30) << std::left << "Remote Host" + << std::setw(20) << "Peer id" + << std::setw(30) << "Recv/Sent (inactive,sec)" + << std::setw(25) << "State" + << std::setw(20) << "Livetime(sec)" + << std::setw(12) << "Down (kB/s)" + << std::setw(14) << "Down(now)" + << std::setw(10) << "Up (kB/s)" + << std::setw(13) << "Up(now)" + << std::endl; + + for (auto & info : res.connections) + { + std::string address = info.incoming ? "INC " : "OUT "; + address += info.ip + ":" + info.port; + //std::string in_out = info.incoming ? "INC " : "OUT "; + tools::msg_writer() + //<< std::setw(30) << std::left << in_out + << std::setw(30) << std::left << address + << std::setw(20) << info.peer_id + << std::setw(30) << std::to_string(info.recv_count) + "(" + std::to_string(info.recv_idle_time) + ")/" + std::to_string(info.send_count) + "(" + std::to_string(info.send_idle_time) + ")" + << std::setw(25) << info.state + << std::setw(20) << info.live_time + << std::setw(12) << info.avg_download + << std::setw(14) << info.current_download + << std::setw(10) << info.avg_upload + << std::setw(13) << info.current_upload + + << std::left << (info.localhost ? "[LOCALHOST]" : "") + << std::left << (info.local_ip ? "[LAN]" : ""); + //tools::msg_writer() << boost::format("%-25s peer_id: %-25s %s") % address % info.peer_id % in_out; + + } + + return true; +} + +bool t_rpc_command_executor::print_blockchain_info(uint64_t start_block_index, uint64_t end_block_index) { + +// this function appears to not exist in the json rpc api, and so is commented +// until such a time as it does. + +/* + cryptonote::COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::request req; + cryptonote::COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::response res; + epee::json_rpc::error error_resp; + + req.start_height = start_block_index; + req.end_height = end_block_index; + + std::string fail_message = "Unsuccessful"; + + if (m_is_rpc) + { + if (!m_rpc_client->json_rpc_request(req, res, "getblockheadersrange", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_getblockheadersrange(req, res, error_resp)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + for (auto & header : res.headers) + { + std::cout + << "major version: " << header.major_version << std::endl + << "minor version: " << header.minor_version << std::endl + << "height: " << header.height << ", timestamp: " << header.timestamp << ", difficulty: " << header.difficulty << std::endl + << "block id: " << header.hash << std::endl + << "previous block id: " << header.prev_hash << std::endl + << "difficulty: " << header.difficulty << ", nonce " << header.nonce << std::endl; + } + +*/ + return true; +} + +bool t_rpc_command_executor::set_log_level(int8_t level) { + cryptonote::COMMAND_RPC_SET_LOG_LEVEL::request req; + cryptonote::COMMAND_RPC_SET_LOG_LEVEL::response res; + req.level = level; + + std::string fail_message = "Unsuccessful"; + + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/set_log_level", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_set_log_level(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + tools::success_msg_writer() << "Log level is now " << std::to_string(level); + + return true; +} + +bool t_rpc_command_executor::print_height() { + cryptonote::COMMAND_RPC_GET_HEIGHT::request req; + cryptonote::COMMAND_RPC_GET_HEIGHT::response res; + + std::string fail_message = "Unsuccessful"; + + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/getheight", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_get_height(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + tools::success_msg_writer() << boost::lexical_cast<std::string>(res.height); + + return true; +} + +bool t_rpc_command_executor::print_block_by_hash(crypto::hash block_hash) { + cryptonote::COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::request req; + cryptonote::COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::response res; + epee::json_rpc::error error_resp; + + req.hash = epee::string_tools::pod_to_hex(block_hash); + + std::string fail_message = "Unsuccessful"; + + if (m_is_rpc) + { + if (!m_rpc_client->json_rpc_request(req, res, "getblockheaderbyhash", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_get_block_header_by_hash(req, res, error_resp)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + print_block_header(res.block_header); + + return true; +} + +bool t_rpc_command_executor::print_block_by_height(uint64_t height) { + cryptonote::COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::request req; + cryptonote::COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::response res; + epee::json_rpc::error error_resp; + + req.height = height; + + std::string fail_message = "Unsuccessful"; + + if (m_is_rpc) + { + if (!m_rpc_client->json_rpc_request(req, res, "getblockheaderbyheight", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_get_block_header_by_height(req, res, error_resp)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + print_block_header(res.block_header); + + return true; +} + +bool t_rpc_command_executor::print_transaction(crypto::hash transaction_hash) { + cryptonote::COMMAND_RPC_GET_TRANSACTIONS::request req; + cryptonote::COMMAND_RPC_GET_TRANSACTIONS::response res; + + std::string fail_message = "Problem fetching transaction"; + + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/gettransactions", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_get_transactions(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + if (1 == res.txs_as_hex.size()) + { + tools::success_msg_writer() << res.txs_as_hex.front(); + } + else + { + tools::fail_msg_writer() << "transaction wasn't found: <" << transaction_hash << '>' << std::endl; + } + + return true; +} + +bool t_rpc_command_executor::print_transaction_pool_long() { + cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL::request req; + cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL::response res; + + std::string fail_message = "Problem fetching transaction pool"; + + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/get_transaction_pool", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_get_transaction_pool(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + if (res.transactions.empty() && res.spent_key_images.empty()) + { + tools::msg_writer() << "Pool is empty" << std::endl; + } + if (! res.transactions.empty()) + { + tools::msg_writer() << "Transactions: "; + for (auto & tx_info : res.transactions) + { + tools::msg_writer() << "id: " << tx_info.id_hash << std::endl + << tx_info.tx_json << std::endl + << "blob_size: " << tx_info.blob_size << std::endl + << "fee: " << cryptonote::print_money(tx_info.fee) << std::endl + << "kept_by_block: " << (tx_info.kept_by_block ? 'T' : 'F') << std::endl + << "max_used_block_height: " << tx_info.max_used_block_height << std::endl + << "max_used_block_id: " << tx_info.max_used_block_id_hash << std::endl + << "last_failed_height: " << tx_info.last_failed_height << std::endl + << "last_failed_id: " << tx_info.last_failed_id_hash << std::endl; + } + if (res.spent_key_images.empty()) + { + tools::msg_writer() << "WARNING: Inconsistent pool state - no spent key images"; + } + } + if (! res.spent_key_images.empty()) + { + tools::msg_writer() << ""; // one newline + tools::msg_writer() << "Spent key images: "; + for (const cryptonote::spent_key_image_info& kinfo : res.spent_key_images) + { + tools::msg_writer() << "key image: " << kinfo.id_hash; + if (kinfo.txs_hashes.size() == 1) + { + tools::msg_writer() << " tx: " << kinfo.txs_hashes[0]; + } + else if (kinfo.txs_hashes.size() == 0) + { + tools::msg_writer() << " WARNING: spent key image has no txs associated"; + } + else + { + tools::msg_writer() << " NOTE: key image for multiple txs: " << kinfo.txs_hashes.size(); + for (const std::string& tx_id : kinfo.txs_hashes) + { + tools::msg_writer() << " tx: " << tx_id; + } + } + } + if (res.transactions.empty()) + { + tools::msg_writer() << "WARNING: Inconsistent pool state - no transactions"; + } + } + + return true; +} + +bool t_rpc_command_executor::print_transaction_pool_short() { + cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL::request req; + cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL::response res; + + std::string fail_message = "Problem fetching transaction pool"; + + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/get_transaction_pool", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_get_transaction_pool(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + if (res.transactions.empty()) + { + tools::msg_writer() << "Pool is empty" << std::endl; + } + for (auto & tx_info : res.transactions) + { + tools::msg_writer() << "id: " << tx_info.id_hash << std::endl + << "blob_size: " << tx_info.blob_size << std::endl + << "fee: " << cryptonote::print_money(tx_info.fee) << std::endl + << "kept_by_block: " << (tx_info.kept_by_block ? 'T' : 'F') << std::endl + << "max_used_block_height: " << tx_info.max_used_block_height << std::endl + << "max_used_block_id: " << tx_info.max_used_block_id_hash << std::endl + << "last_failed_height: " << tx_info.last_failed_height << std::endl + << "last_failed_id: " << tx_info.last_failed_id_hash << std::endl; + } + + return true; +} + +// TODO: update this for testnet +bool t_rpc_command_executor::start_mining(cryptonote::account_public_address address, uint64_t num_threads) { + cryptonote::COMMAND_RPC_START_MINING::request req; + cryptonote::COMMAND_RPC_START_MINING::response res; + req.miner_address = cryptonote::get_account_address_as_str(false, address); + req.threads_count = num_threads; + + std::string fail_message = "Mining did not start"; + + if (m_is_rpc) + { + if (m_rpc_client->rpc_request(req, res, "/start_mining", fail_message.c_str())) + { + tools::success_msg_writer() << "Mining started"; + } + } + else + { + if (!m_rpc_server->on_start_mining(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + return true; +} + +bool t_rpc_command_executor::stop_mining() { + cryptonote::COMMAND_RPC_STOP_MINING::request req; + cryptonote::COMMAND_RPC_STOP_MINING::response res; + + std::string fail_message = "Mining did not stop"; + + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/stop_mining", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_stop_mining(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + tools::success_msg_writer() << "Mining stopped"; + return true; +} + +bool t_rpc_command_executor::stop_daemon() +{ + cryptonote::COMMAND_RPC_STOP_DAEMON::request req; + cryptonote::COMMAND_RPC_STOP_DAEMON::response res; + +//# ifdef WIN32 +// // Stop via service API +// // TODO - this is only temporary! Get rid of hard-coded constants! +// bool ok = windows::stop_service("BitMonero Daemon"); +// ok = windows::uninstall_service("BitMonero Daemon"); +// //bool ok = windows::stop_service(SERVICE_NAME); +// //ok = windows::uninstall_service(SERVICE_NAME); +// if (ok) +// { +// return true; +// } +//# endif + + // Stop via RPC + std::string fail_message = "Daemon did not stop"; + + if (m_is_rpc) + { + if(!m_rpc_client->rpc_request(req, res, "/stop_daemon", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_stop_daemon(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + tools::success_msg_writer() << "Stop signal sent"; + + return true; +} + +bool t_rpc_command_executor::print_status() +{ + if (!m_is_rpc) + { + tools::success_msg_writer() << "print_status makes no sense in interactive mode"; + return true; + } + + bool daemon_is_alive = m_rpc_client->check_connection(); + + if(daemon_is_alive) { + tools::success_msg_writer() << "bitmonerod is running"; + } + else { + tools::fail_msg_writer() << "bitmonerod is NOT running"; + } + + return true; +} + +bool t_rpc_command_executor::set_limit(int limit) +{ + epee::net_utils::connection_basic::set_rate_down_limit( limit ); + epee::net_utils::connection_basic::set_rate_up_limit( limit ); + std::cout << "Set limit-down to " << limit/1024 << " kB/s" << std::endl; + std::cout << "Set limit-up to " << limit/1024 << " kB/s" << std::endl; + return true; +} + +bool t_rpc_command_executor::set_limit_up(int limit) +{ + epee::net_utils::connection_basic::set_rate_up_limit( limit ); + std::cout << "Set limit-up to " << limit/1024 << " kB/s" << std::endl; + return true; +} + +bool t_rpc_command_executor::set_limit_down(int limit) +{ + epee::net_utils::connection_basic::set_rate_down_limit( limit ); + std::cout << "Set limit-down to " << limit/1024 << " kB/s" << std::endl; + return true; +} + +bool t_rpc_command_executor::fast_exit() +{ + cryptonote::COMMAND_RPC_FAST_EXIT::request req; + cryptonote::COMMAND_RPC_FAST_EXIT::response res; + + std::string fail_message = "Daemon did not stop"; + + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/fast_exit", fail_message.c_str())) + { + return true; + } + } + + else + { + if (!m_rpc_server->on_fast_exit(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + tools::success_msg_writer() << "Daemon stopped"; + return true; +} + +bool t_rpc_command_executor::out_peers(uint64_t limit) +{ + cryptonote::COMMAND_RPC_OUT_PEERS::request req; + cryptonote::COMMAND_RPC_OUT_PEERS::response res; + + epee::json_rpc::error error_resp; + + req.out_peers = limit; + + std::string fail_message = "Unsuccessful"; + + if (m_is_rpc) + { + if (!m_rpc_client->json_rpc_request(req, res, "/out_peers", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_out_peers(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + return true; +} + +bool t_rpc_command_executor::start_save_graph() +{ + cryptonote::COMMAND_RPC_START_SAVE_GRAPH::request req; + cryptonote::COMMAND_RPC_START_SAVE_GRAPH::response res; + std::string fail_message = "Unsuccessful"; + + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/start_save_graph", fail_message.c_str())) + { + return true; + } + } + + else + { + if (!m_rpc_server->on_start_save_graph(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + + return true; +} + +bool t_rpc_command_executor::stop_save_graph() +{ + cryptonote::COMMAND_RPC_STOP_SAVE_GRAPH::request req; + cryptonote::COMMAND_RPC_STOP_SAVE_GRAPH::response res; + std::string fail_message = "Unsuccessful"; + + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/stop_save_graph", fail_message.c_str())) + { + return true; + } + } + + else + { + if (!m_rpc_server->on_stop_save_graph(req, res)) + { + tools::fail_msg_writer() << fail_message.c_str(); + return true; + } + } + return true; +} + +}// namespace daemonize diff --git a/src/daemon/rpc_command_executor.h b/src/daemon/rpc_command_executor.h new file mode 100644 index 000000000..43b8a9fe0 --- /dev/null +++ b/src/daemon/rpc_command_executor.h @@ -0,0 +1,117 @@ +/** +@file +@details + +@image html images/other/runtime-commands.png + +*/ + +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#pragma once + +#include "common/rpc_client.h" +#include "misc_log_ex.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_protocol/cryptonote_protocol_handler.h" +#include "p2p/net_node.h" +#include "rpc/core_rpc_server.h" + +namespace daemonize { + +class t_rpc_command_executor final { +private: + tools::t_rpc_client* m_rpc_client; + cryptonote::core_rpc_server* m_rpc_server; + bool m_is_rpc; + +public: + t_rpc_command_executor( + uint32_t ip + , uint16_t port + , bool is_rpc = true + , cryptonote::core_rpc_server* rpc_server = NULL + ); + + ~t_rpc_command_executor(); + + bool print_peer_list(); + + bool save_blockchain(); + + bool show_hash_rate(); + + bool hide_hash_rate(); + + bool show_difficulty(); + + bool print_connections(); + + bool print_blockchain_info(uint64_t start_block_index, uint64_t end_block_index); + + bool set_log_level(int8_t level); + + bool print_height(); + + bool print_block_by_hash(crypto::hash block_hash); + + bool print_block_by_height(uint64_t height); + + bool print_transaction(crypto::hash transaction_hash); + + bool print_transaction_pool_long(); + + bool print_transaction_pool_short(); + + bool start_mining(cryptonote::account_public_address address, uint64_t num_threads); + + bool stop_mining(); + + bool stop_daemon(); + + bool print_status(); + + bool set_limit(int limit); + + bool set_limit_up(int limit); + + bool set_limit_down(int limit); + + bool fast_exit(); + + bool out_peers(uint64_t limit); + + bool start_save_graph(); + + bool stop_save_graph(); +}; + +} // namespace daemonize diff --git a/src/daemonizer/CMakeLists.txt b/src/daemonizer/CMakeLists.txt new file mode 100644 index 000000000..d5830111c --- /dev/null +++ b/src/daemonizer/CMakeLists.txt @@ -0,0 +1,74 @@ +# Copyright (c) 2014-2015, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +if(MSVC OR MINGW) + set(daemonizer_sources + windows_service.cpp + windows_daemonizer.inl + ) +else() + set(daemonizer_sources + posix_fork.cpp + posix_daemonizer.inl + ) +endif() + +set(daemonizer_headers +) + +if(MSVC OR MINGW) + set(daemonizer_private_headers + daemonizer.h + windows_service.h + windows_service_runner.h + ) +else() + set(daemonizer_private_headers + daemonizer.h + posix_fork.h + ) +endif() + +bitmonero_private_headers(daemonizer + ${daemonizer_private_headers}) +bitmonero_add_library(daemonizer + ${daemonizer_sources} + ${daemonizer_headers} + ${daemonizer_private_headers}) +target_link_libraries(daemonizer + LINK_PRIVATE + common + ${Boost_CHRONO_LIBRARY} + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_PROGRAM_OPTIONS_LIBRARY} + ${Boost_REGEX_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${UPNP_LIBRARIES} + ${EXTRA_LIBRARIES}) diff --git a/src/daemonizer/daemonizer.h b/src/daemonizer/daemonizer.h new file mode 100644 index 000000000..6097a58f6 --- /dev/null +++ b/src/daemonizer/daemonizer.h @@ -0,0 +1,38 @@ +#pragma once + +#include <boost/filesystem/path.hpp> +#include <boost/program_options/options_description.hpp> +#include <boost/program_options/variables_map.hpp> + +namespace daemonizer +{ + void init_options( + boost::program_options::options_description & hidden_options + , boost::program_options::options_description & normal_options + ); + + boost::filesystem::path get_default_data_dir(); + + boost::filesystem::path get_relative_path_base( + boost::program_options::variables_map const & vm + ); + + /** + * @arg create_before_detach - this indicates that the daemon should be + * created before the fork, giving it a chance to report initialization + * errors. At the time of this writing, this is not possible in the primary + * daemon (likely due to the size of the blockchain in memory). + */ + template <typename T_executor> + bool daemonize( + int argc, char const * argv[] + , T_executor && executor // universal ref + , boost::program_options::variables_map const & vm + ); +} + +#ifdef WIN32 +# include "daemonizer/windows_daemonizer.inl" +#else +# include "daemonizer/posix_daemonizer.inl" +#endif diff --git a/src/daemonizer/posix_daemonizer.inl b/src/daemonizer/posix_daemonizer.inl new file mode 100644 index 000000000..e06d43d61 --- /dev/null +++ b/src/daemonizer/posix_daemonizer.inl @@ -0,0 +1,60 @@ +#pragma once + +#include "common/scoped_message_writer.h" +#include "common/util.h" +#include "daemonizer/posix_fork.h" + +#include <boost/filesystem/operations.hpp> +#include <boost/filesystem/path.hpp> + +namespace daemonizer +{ + namespace + { + const command_line::arg_descriptor<bool> arg_detach = { + "detach" + , "Run as daemon" + }; + } + + inline void init_options( + boost::program_options::options_description & hidden_options + , boost::program_options::options_description & normal_options + ) + { + command_line::add_arg(normal_options, arg_detach); + } + + inline boost::filesystem::path get_default_data_dir() + { + return boost::filesystem::absolute(tools::get_default_data_dir()); + } + + inline boost::filesystem::path get_relative_path_base( + boost::program_options::variables_map const & vm + ) + { + return boost::filesystem::current_path(); + } + + template <typename T_executor> + inline bool daemonize( + int argc, char const * argv[] + , T_executor && executor // universal ref + , boost::program_options::variables_map const & vm + ) + { + if (command_line::has_arg(vm, arg_detach)) + { + auto daemon = executor.create_daemon(vm); + tools::success_msg_writer() << "Forking to background..."; + posix::fork(); + return daemon.run(); + } + else + { + //LOG_PRINT_L0(CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL); + return executor.run_interactive(vm); + } + } +} diff --git a/src/daemonizer/posix_fork.cpp b/src/daemonizer/posix_fork.cpp new file mode 100644 index 000000000..c068912ec --- /dev/null +++ b/src/daemonizer/posix_fork.cpp @@ -0,0 +1,108 @@ +// Copyright (c) 2003-2011 Christopher M. Kohlhoff (chris at kohlhoff dot com) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// + +#include "daemonizer/posix_fork.h" +#include "misc_log_ex.h" + +#include <cstdlib> +#include <fcntl.h> +#include <unistd.h> +#include <stdexcept> +#include <string> +#include <sys/stat.h> + +namespace posix { + +namespace { + void quit(std::string const & message) + { + LOG_ERROR(message); + throw std::runtime_error(message); + } +} + +void fork() +{ + // Fork the process and have the parent exit. If the process was started + // from a shell, this returns control to the user. Forking a new process is + // also a prerequisite for the subsequent call to setsid(). + if (pid_t pid = ::fork()) + { + if (pid > 0) + { + // We're in the parent process and need to exit. + // + // When the exit() function is used, the program terminates without + // invoking local variables' destructors. Only global variables are + // destroyed. + exit(0); + } + else + { + quit("First fork failed"); + } + } + + // Make the process a new session leader. This detaches it from the + // terminal. + setsid(); + + // A process inherits its working directory from its parent. This could be + // on a mounted filesystem, which means that the running daemon would + // prevent this filesystem from being unmounted. Changing to the root + // directory avoids this problem. + if (chdir("/") < 0) + { + quit("Unable to change working directory to root"); + } + + // The file mode creation mask is also inherited from the parent process. + // We don't want to restrict the permissions on files created by the + // daemon, so the mask is cleared. + umask(0); + + // A second fork ensures the process cannot acquire a controlling terminal. + if (pid_t pid = ::fork()) + { + if (pid > 0) + { + exit(0); + } + else + { + quit("Second fork failed"); + } + } + + // Close the standard streams. This decouples the daemon from the terminal + // that started it. + close(0); + close(1); + close(2); + + // We don't want the daemon to have any standard input. + if (open("/dev/null", O_RDONLY) < 0) + { + quit("Unable to open /dev/null"); + } + + // Send standard output to a log file. + const char* output = "/tmp/bitmonero.daemon.stdout.stderr"; + const int flags = O_WRONLY | O_CREAT | O_APPEND; + const mode_t mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH; + if (open(output, flags, mode) < 0) + { + quit("Unable to open output file: " + std::string(output)); + } + + // Also send standard error to the same log file. + if (dup(1) < 0) + { + quit("Unable to dup output descriptor"); + } +} + +} // namespace posix diff --git a/src/daemonizer/posix_fork.h b/src/daemonizer/posix_fork.h new file mode 100644 index 000000000..f099685e9 --- /dev/null +++ b/src/daemonizer/posix_fork.h @@ -0,0 +1,11 @@ +#pragma once + +#ifndef WIN32 + +namespace posix { + +void fork(); + +} + +#endif diff --git a/src/daemonizer/windows_daemonizer.inl b/src/daemonizer/windows_daemonizer.inl new file mode 100644 index 000000000..c099f4097 --- /dev/null +++ b/src/daemonizer/windows_daemonizer.inl @@ -0,0 +1,156 @@ +#pragma once + +#include "common/util.h" +#include "daemonizer/windows_service.h" +#include "daemonizer/windows_service_runner.h" + +#include <shlobj.h> +#include <boost/filesystem/operations.hpp> +#include <boost/filesystem/path.hpp> + +namespace daemonizer +{ + namespace + { + const command_line::arg_descriptor<bool> arg_install_service = { + "install-service" + , "Install Windows service" + }; + const command_line::arg_descriptor<bool> arg_uninstall_service = { + "uninstall-service" + , "Uninstall Windows service" + }; + const command_line::arg_descriptor<bool> arg_start_service = { + "start-service" + , "Start Windows service" + }; + const command_line::arg_descriptor<bool> arg_stop_service = { + "stop-service" + , "Stop Windows service" + }; + const command_line::arg_descriptor<bool> arg_is_service = { + "run-as-service" + , "Hidden -- true if running as windows service" + }; + + std::string get_argument_string(int argc, char const * argv[]) + { + std::string result = ""; + for (int i = 1; i < argc; ++i) + { + result += " " + std::string{argv[i]}; + } + return result; + } + } + + inline void init_options( + boost::program_options::options_description & hidden_options + , boost::program_options::options_description & normal_options + ) + { + command_line::add_arg(normal_options, arg_install_service); + command_line::add_arg(normal_options, arg_uninstall_service); + command_line::add_arg(normal_options, arg_start_service); + command_line::add_arg(normal_options, arg_stop_service); + command_line::add_arg(hidden_options, arg_is_service); + } + + inline boost::filesystem::path get_default_data_dir() + { + bool admin; + if (!windows::check_admin(admin)) + { + admin = false; + } + if (admin) + { + return boost::filesystem::absolute( + tools::get_special_folder_path(CSIDL_COMMON_APPDATA, true) + "\\" + CRYPTONOTE_NAME + ); + } + else + { + return boost::filesystem::absolute( + tools::get_special_folder_path(CSIDL_APPDATA, true) + "\\" + CRYPTONOTE_NAME + ); + } + } + + inline boost::filesystem::path get_relative_path_base( + boost::program_options::variables_map const & vm + ) + { + if (command_line::has_arg(vm, arg_is_service)) + { + if (command_line::has_arg(vm, command_line::arg_data_dir)) + { + return command_line::get_arg(vm, command_line::arg_data_dir); + } + else + { + return tools::get_default_data_dir(); + } + } + else + { + return boost::filesystem::current_path(); + } + } + + template <typename T_executor> + inline bool daemonize( + int argc, char const * argv[] + , T_executor && executor // universal ref + , boost::program_options::variables_map const & vm + ) + { + std::string arguments = get_argument_string(argc, argv); + + if (command_line::has_arg(vm, arg_is_service)) + { + // TODO - Set the service status here for return codes + windows::t_service_runner<typename T_executor::t_daemon>::run( + executor.name() + , executor.create_daemon(vm) + ); + return true; + } + else if (command_line::has_arg(vm, arg_install_service)) + { + if (windows::ensure_admin(arguments)) + { + arguments += " --run-as-service"; + return windows::install_service(executor.name(), arguments); + } + } + else if (command_line::has_arg(vm, arg_uninstall_service)) + { + if (windows::ensure_admin(arguments)) + { + return windows::uninstall_service(executor.name()); + } + } + else if (command_line::has_arg(vm, arg_start_service)) + { + if (windows::ensure_admin(arguments)) + { + return windows::start_service(executor.name()); + } + } + else if (command_line::has_arg(vm, arg_stop_service)) + { + if (windows::ensure_admin(arguments)) + { + return windows::stop_service(executor.name()); + } + } + else // interactive + { + //LOG_PRINT_L0(CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL); + return executor.run_interactive(vm); + } + + return false; + } +} diff --git a/src/daemonizer/windows_service.cpp b/src/daemonizer/windows_service.cpp new file mode 100644 index 000000000..1b0ee2ef4 --- /dev/null +++ b/src/daemonizer/windows_service.cpp @@ -0,0 +1,341 @@ +#undef UNICODE +#undef _UNICODE + +#include "common/scoped_message_writer.h" +#include "daemonizer/windows_service.h" +#include "string_tools.h" +#include <chrono> +#include <iostream> +#include <utility> +#include <memory> +#include <shellapi.h> +#include <thread> +#include <windows.h> + +namespace windows { + +namespace { + typedef std::unique_ptr<std::remove_pointer<SC_HANDLE>::type, decltype(&::CloseServiceHandle)> service_handle; + + std::string get_last_error() + { + LPSTR p_error_text = nullptr; + + FormatMessage( + FORMAT_MESSAGE_FROM_SYSTEM + | FORMAT_MESSAGE_ALLOCATE_BUFFER + | FORMAT_MESSAGE_IGNORE_INSERTS + , nullptr + , GetLastError() + , MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT) + , reinterpret_cast<LPSTR>(&p_error_text) + , 0 + , nullptr + ); + + if (nullptr == p_error_text) + { + return ""; + } + else + { + return std::string{p_error_text}; + LocalFree(p_error_text); + } + } + + bool relaunch_as_admin( + std::string const & command + , std::string const & arguments + ) + { + SHELLEXECUTEINFO info{}; + info.cbSize = sizeof(info); + info.lpVerb = "runas"; + info.lpFile = command.c_str(); + info.lpParameters = arguments.c_str(); + info.hwnd = nullptr; + info.nShow = SW_SHOWNORMAL; + if (!ShellExecuteEx(&info)) + { + tools::fail_msg_writer() << "Admin relaunch failed: " << get_last_error(); + return false; + } + else + { + return true; + } + } + + // When we relaunch as admin, Windows opens a new window. This just pauses + // to allow the user to read any output. + void pause_to_display_admin_window_messages() + { + std::chrono::milliseconds how_long{1500}; + std::this_thread::sleep_for(how_long); + } +} + +bool check_admin(bool & result) +{ + BOOL is_admin = FALSE; + PSID p_administrators_group = nullptr; + + SID_IDENTIFIER_AUTHORITY nt_authority = SECURITY_NT_AUTHORITY; + + if (!AllocateAndInitializeSid( + &nt_authority + , 2 + , SECURITY_BUILTIN_DOMAIN_RID + , DOMAIN_ALIAS_RID_ADMINS + , 0, 0, 0, 0, 0, 0 + , &p_administrators_group + )) + { + tools::fail_msg_writer() << "Security Identifier creation failed: " << get_last_error(); + return false; + } + + if (!CheckTokenMembership( + nullptr + , p_administrators_group + , &is_admin + )) + { + tools::fail_msg_writer() << "Permissions check failed: " << get_last_error(); + return false; + } + + result = is_admin ? true : false; + + return true; +} + +bool ensure_admin( + std::string const & arguments + ) +{ + bool admin; + + if (!check_admin(admin)) + { + return false; + } + + if (admin) + { + return true; + } + else + { + std::string command = epee::string_tools::get_current_module_path(); + relaunch_as_admin(command, arguments); + return false; + } +} + +bool install_service( + std::string const & service_name + , std::string const & arguments + ) +{ + std::string command = epee::string_tools::get_current_module_path(); + std::string full_command = command + arguments; + + service_handle p_manager{ + OpenSCManager( + nullptr + , nullptr + , SC_MANAGER_CONNECT | SC_MANAGER_CREATE_SERVICE + ) + , &::CloseServiceHandle + }; + if (p_manager == nullptr) + { + tools::fail_msg_writer() << "Couldn't connect to service manager: " << get_last_error(); + return false; + } + + service_handle p_service{ + CreateService( + p_manager.get() + , service_name.c_str() + , service_name.c_str() + , 0 + //, GENERIC_EXECUTE | GENERIC_READ + , SERVICE_WIN32_OWN_PROCESS + , SERVICE_DEMAND_START + , SERVICE_ERROR_NORMAL + , full_command.c_str() + , nullptr + , nullptr + , "" + //, "NT AUTHORITY\\LocalService" + , nullptr // Implies LocalSystem account + , nullptr + ) + , &::CloseServiceHandle + }; + if (p_service == nullptr) + { + tools::fail_msg_writer() << "Couldn't create service: " << get_last_error(); + return false; + } + + tools::success_msg_writer() << "Service installed"; + + pause_to_display_admin_window_messages(); + + return true; +} + +bool start_service( + std::string const & service_name + ) +{ + tools::msg_writer() << "Starting service"; + + SERVICE_STATUS_PROCESS service_status = {}; + DWORD unused = 0; + + service_handle p_manager{ + OpenSCManager( + nullptr + , nullptr + , SC_MANAGER_CONNECT + ) + , &::CloseServiceHandle + }; + if (p_manager == nullptr) + { + tools::fail_msg_writer() << "Couldn't connect to service manager: " << get_last_error(); + return false; + } + + service_handle p_service{ + OpenService( + p_manager.get() + , service_name.c_str() + //, SERVICE_START | SERVICE_QUERY_STATUS + , SERVICE_START + ) + , &::CloseServiceHandle + }; + if (p_service == nullptr) + { + tools::fail_msg_writer() << "Couldn't find service: " << get_last_error(); + return false; + } + + if (!StartService( + p_service.get() + , 0 + , nullptr + )) + { + tools::fail_msg_writer() << "Service start request failed: " << get_last_error(); + return false; + } + + tools::success_msg_writer() << "Service started"; + + pause_to_display_admin_window_messages(); + + return true; +} + +bool stop_service( + std::string const & service_name + ) +{ + tools::msg_writer() << "Stopping service"; + + service_handle p_manager{ + OpenSCManager( + nullptr + , nullptr + , SC_MANAGER_CONNECT + ) + , &::CloseServiceHandle + }; + if (p_manager == nullptr) + { + tools::fail_msg_writer() << "Couldn't connect to service manager: " << get_last_error(); + return false; + } + + service_handle p_service{ + OpenService( + p_manager.get() + , service_name.c_str() + , SERVICE_STOP | SERVICE_QUERY_STATUS + ) + , &::CloseServiceHandle + }; + if (p_service == nullptr) + { + tools::fail_msg_writer() << "Couldn't find service: " << get_last_error(); + return false; + } + + SERVICE_STATUS status = {}; + if (!ControlService(p_service.get(), SERVICE_CONTROL_STOP, &status)) + { + tools::fail_msg_writer() << "Couldn't request service stop: " << get_last_error(); + return false; + } + + tools::success_msg_writer() << "Service stopped"; + + pause_to_display_admin_window_messages(); + + return true; +} + +bool uninstall_service( + std::string const & service_name + ) +{ + service_handle p_manager{ + OpenSCManager( + nullptr + , nullptr + , SC_MANAGER_CONNECT + ) + , &::CloseServiceHandle + }; + if (p_manager == nullptr) + { + tools::fail_msg_writer() << "Couldn't connect to service manager: " << get_last_error(); + return false; + } + + service_handle p_service{ + OpenService( + p_manager.get() + , service_name.c_str() + , SERVICE_QUERY_STATUS | DELETE + ) + , &::CloseServiceHandle + }; + if (p_service == nullptr) + { + tools::fail_msg_writer() << "Couldn't find service: " << get_last_error(); + return false; + } + + SERVICE_STATUS status = {}; + if (!DeleteService(p_service.get())) + { + tools::fail_msg_writer() << "Couldn't uninstall service: " << get_last_error(); + return false; + } + + tools::success_msg_writer() << "Service uninstalled"; + + pause_to_display_admin_window_messages(); + + return true; +} + +} // namespace windows diff --git a/src/daemonizer/windows_service.h b/src/daemonizer/windows_service.h new file mode 100644 index 000000000..11f5fdcdc --- /dev/null +++ b/src/daemonizer/windows_service.h @@ -0,0 +1,36 @@ +#pragma once + +#ifdef WIN32 + +#undef UNICODE +#undef _UNICODE + +#include <string> +#include <windows.h> + +namespace windows +{ + bool check_admin(bool & result); + + bool ensure_admin( + std::string const & arguments + ); + + bool install_service( + std::string const & service_name + , std::string const & arguments + ); + + bool uninstall_service( + std::string const & service_name + ); + + bool start_service( + std::string const & service_name + ); + + bool stop_service( + std::string const & service_name + ); +} +#endif diff --git a/src/daemonizer/windows_service_runner.h b/src/daemonizer/windows_service_runner.h new file mode 100644 index 000000000..79b10b136 --- /dev/null +++ b/src/daemonizer/windows_service_runner.h @@ -0,0 +1,157 @@ +#pragma once + +#ifdef WIN32 + +#undef UNICODE +#undef _UNICODE + +#include "daemonizer/windows_service.h" +#include <memory> +#include <string> +#include <vector> +#include <windows.h> + +namespace windows { + namespace + { + std::vector<char> vecstring(std::string const & str) + { + std::vector<char> result{str.begin(), str.end()}; + result.push_back('\0'); + return result; + } + } + + template <typename T_handler> + class t_service_runner final + { + private: + SERVICE_STATUS_HANDLE m_status_handle{nullptr}; + SERVICE_STATUS m_status{}; + std::mutex m_lock{}; + std::string m_name; + T_handler m_handler; + + static std::unique_ptr<t_service_runner<T_handler>> sp_instance; + public: + t_service_runner( + std::string name + , T_handler handler + ) + : m_name{std::move(name)} + , m_handler{std::move(handler)} + { + m_status.dwServiceType = SERVICE_WIN32; + m_status.dwCurrentState = SERVICE_STOPPED; + m_status.dwControlsAccepted = 0; + m_status.dwWin32ExitCode = NO_ERROR; + m_status.dwServiceSpecificExitCode = NO_ERROR; + m_status.dwCheckPoint = 0; + m_status.dwWaitHint = 0; + } + + t_service_runner & operator=(t_service_runner && other) + { + if (this != &other) + { + m_status_handle = std::move(other.m_status_handle); + m_status = std::move(other.m_status); + m_name = std::move(other.m_name); + m_handler = std::move(other.m_handler); + } + return *this; + } + + static void run( + std::string name + , T_handler handler + ) + { + sp_instance.reset(new t_service_runner<T_handler>{ + std::move(name) + , std::move(handler) + }); + + sp_instance->run_(); + } + + private: + void run_() + { + SERVICE_TABLE_ENTRY table[] = + { + { vecstring(m_name).data(), &service_main } + , { 0, 0 } + }; + + StartServiceCtrlDispatcher(table); + } + + void report_status(DWORD status) + { + m_status.dwCurrentState = status; + if (status == SERVICE_RUNNING) + { + m_status.dwControlsAccepted = SERVICE_ACCEPT_STOP | SERVICE_ACCEPT_SHUTDOWN; + } + else if(status == SERVICE_STOP_PENDING) + { + m_status.dwControlsAccepted = 0; + } + SetServiceStatus(m_status_handle, &m_status); + } + + static void WINAPI service_main(DWORD argc, LPSTR * argv) + { + sp_instance->service_main_(argc, argv); + } + + void service_main_(DWORD argc, LPSTR * argv) + { + m_status_handle = RegisterServiceCtrlHandler(m_name.c_str(), &on_state_change_request); + if (m_status_handle == nullptr) return; + + report_status(SERVICE_START_PENDING); + + report_status(SERVICE_RUNNING); + + m_handler.run(); + + on_state_change_request_(SERVICE_CONTROL_STOP); + + // Ensure that the service is uninstalled + uninstall_service(m_name); + } + + static void WINAPI on_state_change_request(DWORD control_code) + { + sp_instance->on_state_change_request_(control_code); + } + + void on_state_change_request_(DWORD control_code) + { + switch (control_code) + { + case SERVICE_CONTROL_INTERROGATE: + break; + case SERVICE_CONTROL_SHUTDOWN: + case SERVICE_CONTROL_STOP: + report_status(SERVICE_STOP_PENDING); + m_handler.stop(); + report_status(SERVICE_STOPPED); + break; + case SERVICE_CONTROL_PAUSE: + break; + case SERVICE_CONTROL_CONTINUE: + break; + default: + break; + } + } + }; + + template <typename T_handler> + std::unique_ptr<t_service_runner<T_handler>> t_service_runner<T_handler>::sp_instance; +} + +#endif diff --git a/src/miner/CMakeLists.txt b/src/miner/CMakeLists.txt index 83bda57cc..2a55c88f7 100644 --- a/src/miner/CMakeLists.txt +++ b/src/miner/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # diff --git a/src/miner/simpleminer.cpp b/src/miner/simpleminer.cpp index 3223bdd50..6212f88f5 100644 --- a/src/miner/simpleminer.cpp +++ b/src/miner/simpleminer.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -41,6 +41,7 @@ using namespace epee; namespace po = boost::program_options; +unsigned int epee::g_test_dbg_lock_sleep = 0; int main(int argc, char** argv) { diff --git a/src/miner/simpleminer.h b/src/miner/simpleminer.h index f6897e20e..9b8e84b3e 100644 --- a/src/miner/simpleminer.h +++ b/src/miner/simpleminer.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/miner/simpleminer_protocol_defs.h b/src/miner/simpleminer_protocol_defs.h index 22ea958e7..c63246578 100644 --- a/src/miner/simpleminer_protocol_defs.h +++ b/src/miner/simpleminer_protocol_defs.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/miner/target_helper.h b/src/miner/target_helper.h index 723d61a3c..0214b8038 100644 --- a/src/miner/target_helper.h +++ b/src/miner/target_helper.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/mnemonics/CMakeLists.txt b/src/mnemonics/CMakeLists.txt index 66ef4f3f1..ce9520956 100644 --- a/src/mnemonics/CMakeLists.txt +++ b/src/mnemonics/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # diff --git a/src/mnemonics/electrum-words.cpp b/src/mnemonics/electrum-words.cpp index 7258b8a15..590320550 100644 --- a/src/mnemonics/electrum-words.cpp +++ b/src/mnemonics/electrum-words.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/mnemonics/electrum-words.h b/src/mnemonics/electrum-words.h index b75de30e9..5cde7fb02 100644 --- a/src/mnemonics/electrum-words.h +++ b/src/mnemonics/electrum-words.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/mnemonics/english.h b/src/mnemonics/english.h index 8ca0a3fe3..fee428817 100644 --- a/src/mnemonics/english.h +++ b/src/mnemonics/english.h @@ -1,6 +1,6 @@ // Word list originally created as part of the Electrum project, Copyright (C) 2014 Thomas Voegtlin
//
-// Copyright (c) 2014, The Monero Project
+// Copyright (c) 2014-2015, The Monero Project
//
// All rights reserved.
//
@@ -728,7 +728,7 @@ namespace Language "initiate",
"injury",
"inkling",
- "incline",
+ "inline",
"inmate",
"innocent",
"inorganic",
@@ -1073,7 +1073,7 @@ namespace Language "ouch",
"ought",
"ounce",
- "launchpad",
+ "ourselves",
"oust",
"outbreak",
"oval",
diff --git a/src/mnemonics/japanese.h b/src/mnemonics/japanese.h index fc3d1ee74..5f7adce18 100644 --- a/src/mnemonics/japanese.h +++ b/src/mnemonics/japanese.h @@ -1,6 +1,6 @@ // Word list originally created by dabura667
//
-// Copyright (c) 2014, The Monero Project
+// Copyright (c) 2014-2015, The Monero Project
//
// All rights reserved.
//
diff --git a/src/mnemonics/language_base.h b/src/mnemonics/language_base.h index 06815e39d..0e7d60a02 100644 --- a/src/mnemonics/language_base.h +++ b/src/mnemonics/language_base.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project
+// Copyright (c) 2014-2015, The Monero Project
//
// All rights reserved.
//
diff --git a/src/mnemonics/old_english.h b/src/mnemonics/old_english.h index b91a593b6..f7974f4d5 100644 --- a/src/mnemonics/old_english.h +++ b/src/mnemonics/old_english.h @@ -1,6 +1,6 @@ // Word list originally created as part of the Electrum project, Copyright (C) 2014 Thomas Voegtlin
//
-// Copyright (c) 2014, The Monero Project
+// Copyright (c) 2014-2015, The Monero Project
//
// All rights reserved.
//
diff --git a/src/mnemonics/portuguese.h b/src/mnemonics/portuguese.h index 6a90659e0..f60991a4d 100644 --- a/src/mnemonics/portuguese.h +++ b/src/mnemonics/portuguese.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project
+// Copyright (c) 2014-2015, The Monero Project
//
// All rights reserved.
//
diff --git a/src/mnemonics/singleton.h b/src/mnemonics/singleton.h index 74e121e4e..ab4e95cf2 100644 --- a/src/mnemonics/singleton.h +++ b/src/mnemonics/singleton.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project
+// Copyright (c) 2014-2015, The Monero Project
//
// All rights reserved.
//
diff --git a/src/mnemonics/spanish.h b/src/mnemonics/spanish.h index f71627086..b638ec1b0 100644 --- a/src/mnemonics/spanish.h +++ b/src/mnemonics/spanish.h @@ -1,6 +1,6 @@ // Word list originally created as part of the Electrum project, Copyright (C) 2014 Thomas Voegtlin
//
-// Copyright (c) 2014, The Monero Project
+// Copyright (c) 2014-2015, The Monero Project
//
// All rights reserved.
//
@@ -1690,4 +1690,4 @@ namespace Language };
}
-#endif
\ No newline at end of file +#endif diff --git a/src/p2p/CMakeLists.txt b/src/p2p/CMakeLists.txt new file mode 100644 index 000000000..541b90fa9 --- /dev/null +++ b/src/p2p/CMakeLists.txt @@ -0,0 +1,46 @@ +# Copyright (c) 2014, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +cmake_minimum_required (VERSION 2.6) +project (bitmonero CXX) + +file(GLOB P2P *) +source_group(p2p FILES ${P2P}) + +#add_library(p2p ${P2P}) + +#bitmonero_private_headers(p2p ${P2P}) +bitmonero_add_library(p2p ${P2P}) +#target_link_libraries(p2p) +# LINK_PRIVATE +# ${Boost_CHRONO_LIBRARY} +# ${Boost_REGEX_LIBRARY} +# ${Boost_SYSTEM_LIBRARY} +# ${Boost_THREAD_LIBRARY} +# ${EXTRA_LIBRARIES}) +add_dependencies(p2p + version) diff --git a/src/p2p/connection_basic.cpp b/src/p2p/connection_basic.cpp new file mode 100644 index 000000000..0454f30e3 --- /dev/null +++ b/src/p2p/connection_basic.cpp @@ -0,0 +1,299 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief base for connection, contains e.g. the ratelimit hooks + +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/* rfree: implementation for the non-template base, can be used by connection<> template class in abstract_tcp_server2 file */ + +#include "connection_basic.hpp" + +#include <boost/asio.hpp> +#include <string> +#include <vector> +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <atomic> + +#include <boost/asio.hpp> +#include <boost/array.hpp> +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <boost/enable_shared_from_this.hpp> +#include <boost/interprocess/detail/atomic.hpp> +#include <boost/thread/thread.hpp> + +#include <memory> + +#include "syncobj.h" + +#include "../../contrib/epee/include/net/net_utils_base.h" +#include "../../contrib/epee/include/misc_log_ex.h" +#include <boost/lambda/bind.hpp> +#include <boost/foreach.hpp> +#include <boost/lambda/lambda.hpp> +#include <boost/uuid/random_generator.hpp> +#include <boost/chrono.hpp> +#include <boost/utility/value_init.hpp> +#include <boost/asio/deadline_timer.hpp> +#include <boost/date_time/posix_time/posix_time.hpp> +#include <boost/thread/thread.hpp> +#include <boost/filesystem.hpp> +#include "misc_language.h" +#include "pragma_comp_defs.h" +#include <fstream> +#include <sstream> +#include <iomanip> +#include <algorithm> +#include <mutex> + +#include <boost/asio/basic_socket.hpp> +#include <boost/asio/ip/unicast.hpp> +#include "../../contrib/epee/include/net/abstract_tcp_server2.h" + +#include "../../contrib/otshell_utils/utils.hpp" +#include "data_logger.hpp" +using namespace nOT::nUtils; + +// TODO: +#include "../../src/p2p/network_throttle-detail.hpp" +#include "../../src/cryptonote_core/cryptonote_core.h" + +// ################################################################################################ +// local (TU local) headers +// ################################################################################################ + +namespace epee +{ +namespace net_utils +{ + + std::string to_string(t_connection_type type) + { + if (type == e_connection_type_NET) + return std::string("NET"); + else if (type == e_connection_type_RPC) + return std::string("RPC"); + else if (type == e_connection_type_P2P) + return std::string("P2P"); + + return std::string("UNKNOWN"); + } + + +/* ============================================================================ */ + +class connection_basic_pimpl { + public: + connection_basic_pimpl(const std::string &name); + + static int m_default_tos; + + network_throttle_bw m_throttle; // per-perr + critical_section m_throttle_lock; + + int m_peer_number; // e.g. for debug/stats +}; + + +} // namespace +} // namespace + +// ################################################################################################ +// The implementation part +// ################################################################################################ + +namespace epee +{ +namespace net_utils +{ + +// ================================================================================================ +// connection_basic_pimpl +// ================================================================================================ + +connection_basic_pimpl::connection_basic_pimpl(const std::string &name) : m_throttle(name) { } + +// ================================================================================================ +// connection_basic +// ================================================================================================ + +// static variables: +int connection_basic_pimpl::m_default_tos; + +// methods: +connection_basic::connection_basic(boost::asio::io_service& io_service, std::atomic<long> &ref_sock_count, std::atomic<long> &sock_number) + : + mI( new connection_basic_pimpl("peer") ), + strand_(io_service), + socket_(io_service), + m_want_close_connection(false), + m_was_shutdown(false), + m_ref_sock_count(ref_sock_count) +{ + ++ref_sock_count; // increase the global counter + mI->m_peer_number = sock_number.fetch_add(1); // use, and increase the generated number + + string remote_addr_str = "?"; + try { remote_addr_str = socket_.remote_endpoint().address().to_string(); } catch(...){} ; + + _note("Spawned connection p2p#"<<mI->m_peer_number<<" to " << remote_addr_str << " currently we have sockets count:" << m_ref_sock_count); + //boost::filesystem::create_directories("log/dr-monero/net/"); +} + +connection_basic::~connection_basic() { + string remote_addr_str = "?"; + try { remote_addr_str = socket_.remote_endpoint().address().to_string(); } catch(...){} ; + _note("Destructing connection p2p#"<<mI->m_peer_number << " to " << remote_addr_str); +} + +void connection_basic::set_rate_up_limit(uint64_t limit) { + + // TODO remove __SCALING_FACTOR... + const double SCALING_FACTOR = 2.1; // to acheve the best performance + limit *= SCALING_FACTOR; + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); + network_throttle_manager::get_global_throttle_out().set_target_speed(limit); + network_throttle_manager::get_global_throttle_out().set_real_target_speed(limit / SCALING_FACTOR); + } + save_limit_to_file(limit); +} + +void connection_basic::set_rate_down_limit(uint64_t limit) { + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_in ); + network_throttle_manager::get_global_throttle_in().set_target_speed(limit); + } + + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_inreq ); + network_throttle_manager::get_global_throttle_inreq().set_target_speed(limit); + } + save_limit_to_file(limit); +} + + +void connection_basic::save_limit_to_file(int limit) { + // saving limit to file + if (!epee::net_utils::data_logger::m_save_graph) + return; + + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); + epee::net_utils::data_logger::get_instance().add_data("upload_limit", network_throttle_manager::get_global_throttle_out().get_terget_speed() / 1024); + } + + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_in ); + epee::net_utils::data_logger::get_instance().add_data("download_limit", network_throttle_manager::get_global_throttle_in().get_terget_speed() / 1024); + } +} + +void connection_basic::set_tos_flag(int tos) { + connection_basic_pimpl::m_default_tos = tos; +} + +int connection_basic::get_tos_flag() { + return connection_basic_pimpl::m_default_tos; +} + +void connection_basic::sleep_before_packet(size_t packet_size, int phase, int q_len) { + double delay=0; // will be calculated + do + { // rate limiting + if (m_was_shutdown) { + _dbg2("m_was_shutdown - so abort sleep"); + return; + } + + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); + delay = network_throttle_manager::get_global_throttle_out().get_sleep_time_after_tick( packet_size ); // decission from global + } + + delay *= 0.50; + if (delay > 0) { + long int ms = (long int)(delay * 1000); + _info_c("net/sleep", "Sleeping in " << __FUNCTION__ << " for " << ms << " ms before packet_size="<<packet_size); // debug sleep + _dbg1("sleep in sleep_before_packet"); + epee::net_utils::data_logger::get_instance().add_data("sleep_up", ms); + boost::this_thread::sleep(boost::posix_time::milliseconds( ms ) ); + } + } while(delay > 0); + +// XXX LATER XXX + { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); + network_throttle_manager::get_global_throttle_out().handle_trafic_exact( packet_size * 700); // increase counter - global + } + +} +void connection_basic::set_start_time() { + CRITICAL_REGION_LOCAL( network_throttle_manager::m_lock_get_global_throttle_out ); + m_start_time = network_throttle_manager::get_global_throttle_out().get_time_seconds(); +} + +void connection_basic::do_send_handler_write(const void* ptr , size_t cb ) { + sleep_before_packet(cb,1,-1); + _info_c("net/out/size", "handler_write (direct) - before ASIO write, for packet="<<cb<<" B (after sleep)"); + set_start_time(); +} + +void connection_basic::do_send_handler_write_from_queue( const boost::system::error_code& e, size_t cb, int q_len ) { + sleep_before_packet(cb,2,q_len); + _info_c("net/out/size", "handler_write (after write, from queue="<<q_len<<") - before ASIO write, for packet="<<cb<<" B (after sleep)"); + + set_start_time(); +} + +void connection_basic::logger_handle_net_read(size_t size) { // network data read + size /= 1024; + epee::net_utils::data_logger::get_instance().add_data("download", size); +} + +void connection_basic::logger_handle_net_write(size_t size) { + size /= 1024; + epee::net_utils::data_logger::get_instance().add_data("upload", size); +} + +double connection_basic::get_sleep_time(size_t cb) { + CRITICAL_REGION_LOCAL(epee::net_utils::network_throttle_manager::network_throttle_manager::m_lock_get_global_throttle_out); + auto t = network_throttle_manager::get_global_throttle_out().get_sleep_time(cb); + return t; +} + +void connection_basic::set_save_graph(bool save_graph) { + epee::net_utils::data_logger::m_save_graph = save_graph; +} + + +} // namespace +} // namespace + diff --git a/src/p2p/connection_basic.hpp b/src/p2p/connection_basic.hpp new file mode 100644 index 000000000..d8101afe4 --- /dev/null +++ b/src/p2p/connection_basic.hpp @@ -0,0 +1,140 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief base for connection, contains e.g. the ratelimit hooks + +// ! This file might contain variable names same as in template class connection<> +// ! from files contrib/epee/include/net/abstract_tcp_server2.* +// ! I am not a lawyer; afaik APIs, var names etc are not copyrightable ;) +// ! (how ever if in some wonderful juristdictions that is not the case, then why not make another sub-class withat that members and licence it as epee part) +// ! Working on above premise, IF this is valid in your juristdictions, then consider this code as released as: + +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +/* rfree: place for hanlers for the non-template base, can be used by connection<> template class in abstract_tcp_server2 file */ + +#ifndef INCLUDED_p2p_connection_basic_hpp +#define INCLUDED_p2p_connection_basic_hpp + + +#include <boost/asio.hpp> +#include <string> +#include <vector> +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <atomic> + +#include <boost/asio.hpp> +#include <boost/array.hpp> +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <boost/enable_shared_from_this.hpp> +#include <boost/interprocess/detail/atomic.hpp> +#include <boost/thread/thread.hpp> + +#include <memory> + +#include "../../contrib/epee/include/net/net_utils_base.h" +#include "../../contrib/epee/include/syncobj.h" + +namespace epee +{ +namespace net_utils +{ + + /************************************************************************/ + /* */ + /************************************************************************/ + /// Represents a single connection from a client. + +class connection_basic_pimpl; // PIMPL for this class + + enum t_connection_type { // type of the connection (of this server), e.g. so that we will know how to limit it + e_connection_type_NET = 0, // default (not used?) + e_connection_type_RPC = 1, // the rpc commands (probably not rate limited, not chunked, etc) + e_connection_type_P2P = 2 // to other p2p node (probably limited) + }; + + std::string to_string(t_connection_type type); + +class connection_basic { // not-templated base class for rapid developmet of some code parts + public: + std::unique_ptr< connection_basic_pimpl > mI; // my Implementation + + // moved here from orginal connecton<> - common member variables that do not depend on template in connection<> + volatile uint32_t m_want_close_connection; + std::atomic<bool> m_was_shutdown; + critical_section m_send_que_lock; + std::list<std::string> m_send_que; + volatile bool m_is_multithreaded; + double m_start_time; + /// Strand to ensure the connection's handlers are not called concurrently. + boost::asio::io_service::strand strand_; + /// Socket for the connection. + boost::asio::ip::tcp::socket socket_; + + std::atomic<long> &m_ref_sock_count; // reference to external counter of existing sockets that we will ++/-- + public: + // first counter is the ++/-- count of current sockets, the other socket_number is only-increasing ++ number generator + connection_basic(boost::asio::io_service& io_service, std::atomic<long> &ref_sock_count, std::atomic<long> &sock_number); + + virtual ~connection_basic(); + + // various handlers to be called from connection class: + void do_send_handler_write(const void * ptr , size_t cb); + void do_send_handler_write_from_queue(const boost::system::error_code& e, size_t cb , int q_len); // from handle_write, sending next part + + void logger_handle_net_write(size_t size); // network data written + void logger_handle_net_read(size_t size); // network data read + + void set_start_time(); + + // config for rate limit + + static void set_rate_up_limit(uint64_t limit); + static void set_rate_down_limit(uint64_t limit); + + // config misc + static void set_tos_flag(int tos); // ToS / QoS flag + static int get_tos_flag(); + + // handlers and sleep + void sleep_before_packet(size_t packet_size, int phase, int q_len); // execute a sleep ; phase is not really used now(?) + static void save_limit_to_file(int limit); ///< for dr-monero + static double get_sleep_time(size_t cb); + + static void set_save_graph(bool save_graph); +}; + +} // nameserver +} // nameserver + +#endif + + diff --git a/src/p2p/data_logger.cpp b/src/p2p/data_logger.cpp new file mode 100644 index 000000000..54fd33e82 --- /dev/null +++ b/src/p2p/data_logger.cpp @@ -0,0 +1,173 @@ +#include "data_logger.hpp" +#include <stdexcept> + +#include <boost/chrono.hpp> +#include <boost/filesystem.hpp> +#include <chrono> +#include "../../contrib/otshell_utils/utils.hpp" + +namespace epee +{ +namespace net_utils +{ + data_logger &data_logger::get_instance() { + std::call_once(m_singleton, + [] { + _info_c("dbg/data","Creating singleton of data_logger"); + if (m_state != data_logger_state::state_before_init) { _erro_c("dbg/data","Internal error in singleton"); throw std::runtime_error("data_logger singleton"); } + m_state = data_logger_state::state_during_init; + m_obj.reset(new data_logger()); + m_state = data_logger_state::state_ready_to_use; + } + ); + + if (m_state != data_logger_state::state_ready_to_use) { + _erro ("trying to use not working data_logger"); + throw std::runtime_error("data_logger ctor state"); + } + + return * m_obj; + } + + data_logger::data_logger() { + _note_c("dbg/data","Starting data logger (for graphs data)"); + if (m_state != data_logger_state::state_during_init) { _erro_c("dbg/data","Singleton ctor state"); throw std::runtime_error("data_logger ctor state"); } + std::lock_guard<std::mutex> lock(mMutex); // lock + + // prepare all the files for given data channels: + mFilesMap["peers"] = data_logger::fileData("log/dr-monero/peers.data"); + mFilesMap["download"] = data_logger::fileData("log/dr-monero/net/in-all.data"); + mFilesMap["upload"] = data_logger::fileData("log/dr-monero/net/out-all.data"); + mFilesMap["request"] = data_logger::fileData("log/dr-monero/net/req-all.data"); + mFilesMap["sleep_down"] = data_logger::fileData("log/dr-monero/down_sleep_log.data"); + mFilesMap["sleep_up"] = data_logger::fileData("log/dr-monero/up_sleep_log.data"); + mFilesMap["calc_time"] = data_logger::fileData("log/dr-monero/get_objects_calc_time.data"); + mFilesMap["blockchain_processing_time"] = data_logger::fileData("log/dr-monero/blockchain_log.data"); + mFilesMap["block_processing"] = data_logger::fileData("log/dr-monero/block_proc.data"); + + mFilesMap["peers_limit"] = data_logger::fileData("log/dr-monero/peers_limit.info"); + mFilesMap["download_limit"] = data_logger::fileData("log/dr-monero/limit_down.info"); + mFilesMap["upload_limit"] = data_logger::fileData("log/dr-monero/limit_up.info"); + + mFilesMap["peers_limit"].mLimitFile = true; + mFilesMap["download_limit"].mLimitFile = true; + mFilesMap["upload_limit"].mLimitFile = true; + + // do NOT modify mFilesMap below this point, since there is no locking for this used (yet) + + _info_c("dbg/data","Creating thread for data logger"); // create timer thread + m_thread_maybe_running=true; + std::shared_ptr<std::thread> logger_thread(new std::thread([&]() { + _info_c("dbg/data","Inside thread for data logger"); + while (m_state == data_logger_state::state_during_init) { // wait for creation to be done (in other thread, in singleton) before actually running + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + _info_c("dbg/data","Inside thread for data logger - going into main loop"); + while (m_state == data_logger_state::state_ready_to_use) { // run as long as we are not closing the single object + std::this_thread::sleep_for(std::chrono::seconds(1)); + saveToFile(); // save all the pending data + } + _info_c("dbg/data","Inside thread for data logger - done the main loop"); + m_thread_maybe_running=false; + })); + logger_thread->detach(); + _info_c("dbg/data","Data logger constructed"); + } + + data_logger::~data_logger() { + _note_c("dbg/data","Destructor of the data logger"); + { + std::lock_guard<std::mutex> lock(mMutex); + m_state = data_logger_state::state_dying; + } + _info_c("dbg/data","State was set to dying"); + while(m_thread_maybe_running) { // wait for the thread to exit + std::this_thread::sleep_for(std::chrono::seconds(1)); + _info_c("dbg/data","Waiting for background thread to exit"); + } + _info_c("dbg/data","Thread exited"); + } + + void data_logger::kill_instance() { + m_state = data_logger_state::state_dying; + m_obj.reset(); + } + + void data_logger::add_data(std::string filename, unsigned int data) { + std::lock_guard<std::mutex> lock(mMutex); + if (m_state != data_logger_state::state_ready_to_use) { _info_c("dbg/data","Data logger is not ready, returning."); return; } + + if (mFilesMap.find(filename) == mFilesMap.end()) { // no such file/counter + _erro_c("dbg/data","Trying to use not opened data file filename="<<filename); + _erro_c("dbg/data","Disabling saving of graphs due to error"); + m_save_graph=false; // <--- disabling saving graphs + return; + } + + if (mFilesMap[filename].mLimitFile) { // this holds a number (that is not additive) - e.g. the limit setting + mFilesMap[filename].mDataToSave = data; + } else { + mFilesMap[filename].mDataToSave += data; // this holds a number that should be sum of all accumulated samples + } + } + + bool data_logger::is_dying() { + if (m_state == data_logger_state::state_dying) { + return true; + } + else { + return false; + } + } + + void data_logger::saveToFile() { + _dbg2_c("dbg/data","saving to files"); + std::lock_guard<std::mutex> lock(mMutex); + if (m_state != data_logger_state::state_ready_to_use) { _info_c("dbg/data","Data logger is not ready, returning."); return; } + nOT::nUtils::cFilesystemUtils::CreateDirTree("log/dr-monero/net/"); + for (auto &element : mFilesMap) + { + element.second.save(); + if (!element.second.mLimitFile) element.second.mDataToSave = 0; + } + } + + // the inner class: + + double data_logger::fileData::get_current_time() { + #if defined(__APPLE__) + auto point = std::chrono::system_clock::now(); + #else + auto point = std::chrono::steady_clock::now(); + #endif + auto time_from_epoh = point.time_since_epoch(); + auto ms = std::chrono::duration_cast< std::chrono::milliseconds >( time_from_epoh ).count(); + double ms_f = ms; + return ms_f / 1000.; + } + + data_logger::fileData::fileData(std::string pFile) { + _dbg3_c("dbg/data","opening data file named pFile="<<pFile<<" for this="<<this); + mFile = std::make_shared<std::ofstream> (pFile); + _dbg1_c("dbg/data","opened data file named pFile="<<pFile<<" in mFile="<<mFile<<" for this="<<this); + mPath = pFile; + } + + void data_logger::fileData::save() { + if (!data_logger::m_save_graph) return; // <--- disabled + _dbg2_c("dbg/data","saving to the file now, mFile="<<mFile); + mFile->open(mPath, std::ios::app); + *mFile << static_cast<int>(get_current_time()) << " " << mDataToSave << std::endl; + mFile->close(); + } + + +data_logger_state data_logger::m_state(data_logger_state::state_before_init); ///< (static) state of the singleton object +std::atomic<bool> data_logger::m_save_graph(false); // (static) +std::atomic<bool> data_logger::m_thread_maybe_running(false); // (static) +std::once_flag data_logger::m_singleton; // (static) +std::unique_ptr<data_logger> data_logger::m_obj; // (static) + +} // namespace +} // namespace + diff --git a/src/p2p/data_logger.hpp b/src/p2p/data_logger.hpp new file mode 100644 index 000000000..f38dacdcb --- /dev/null +++ b/src/p2p/data_logger.hpp @@ -0,0 +1,76 @@ +#ifndef INCLUDED_p2p_data_logger_hpp +#define INCLUDED_p2p_data_logger_hpp + +#include <string> +#include <map> +#include <fstream> +#include <memory> +#include <thread> +#include <mutex> +#include <atomic> + +namespace epee +{ +namespace net_utils +{ + +enum class data_logger_state { state_before_init, state_during_init, state_ready_to_use, state_dying }; + +/*** +@note: use it ONLY via singleton! It will be spawned then, and will auto destruct on program exit. +@note: do call ::kill_instance() before exiting main, at end of main. But before make sure no one else (e.g. no other threads) will try to use this/singleton +@note: it is not allowed to use this class from code "runnig before or after main", e.g. from ctors of static objects, because of static-creation-order races +@note: on creation (e.g. from singleton), it spawns a thread that saves all data in background +*/ + class data_logger { + public: + static data_logger &get_instance(); ///< singleton + static void kill_instance(); ///< call this before ending main to allow more gracefull shutdown of the main singleton and it's background thread + ~data_logger(); ///< destr, will be called when singleton is killed when global m_obj dies. will kill theads etc + + private: + data_logger(); ///< constructor is private, use only via singleton get_instance + + public: + data_logger(const data_logger &ob) = delete; // use only one per program + data_logger(data_logger &&ob) = delete; + data_logger & operator=(const data_logger&) = delete; + data_logger & operator=(data_logger&&) = delete; + + void add_data(std::string filename, unsigned int data); ///< use this to append data here. Use it only the singleton. It locks itself. + + static std::atomic<bool> m_save_graph; ///< global setting flag, should we save all the data or not (can disable logging graphs data) + static bool is_dying(); + + private: + static std::once_flag m_singleton; ///< to guarantee singleton creates the object exactly once + static data_logger_state m_state; ///< state of the singleton object + static std::atomic<bool> m_thread_maybe_running; ///< is the background thread (more or less) running, or is it fully finished + static std::unique_ptr<data_logger> m_obj; ///< the singleton object. Only use it via get_instance(). Can be killed by kill_instance() + + /*** + * one graph/file with data + */ + class fileData { + public: + fileData() = default; + fileData(const fileData &ob) = delete; + fileData(std::string pFile); + + std::shared_ptr<std::ofstream> mFile; + long int mDataToSave = 0; ///< sum of the data (in current interval, will be counted from 0 on next interval) + static double get_current_time(); + void save(); + std::string mPath; + bool mLimitFile = false; ///< this holds a number (that is not additive) - e.g. the limit setting + }; + + std::map<std::string, fileData> mFilesMap; + std::mutex mMutex; + void saveToFile(); ///< write data to the target files. do not use this directly + }; + +} // namespace +} // namespace + +#endif diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 220bda0c2..df726703f 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -80,14 +80,15 @@ namespace nodetool public: typedef t_payload_net_handler payload_net_handler; - node_server( - t_payload_net_handler& payload_handler - , boost::uuids::uuid network_id - ) - : m_payload_handler(payload_handler) - , m_allow_local_ip(false) - , m_hide_my_port(false) - , m_network_id(std::move(network_id)) + node_server(t_payload_net_handler& payload_handler) + :m_payload_handler(payload_handler), + m_current_number_of_out_peers(0), + m_allow_local_ip(false), + m_hide_my_port(false), + m_no_igd(false), + m_save_graph(false), + is_closing(false), + m_net_server( epee::net_utils::e_connection_type_P2P ) // this is a P2P connection of the main p2p node server, because this is class node_server<> {} virtual ~node_server() {} @@ -95,7 +96,7 @@ namespace nodetool static void init_options(boost::program_options::options_description& desc); bool run(); - bool init(const boost::program_options::variables_map& vm, bool testnet); + bool init(const boost::program_options::variables_map& vm); bool deinit(); bool send_stop_signal(); uint32_t get_this_peer_port(){return m_listenning_port;} @@ -113,6 +114,7 @@ namespace nodetool virtual uint64_t get_connections_count(); size_t get_outgoing_connections_count(); peerlist_manager& get_peerlist_manager(){return m_peerlist;} + void delete_connections(size_t count); private: const std::vector<std::string> m_seed_nodes_list = { "seeds.moneroseeds.se" @@ -120,6 +122,9 @@ namespace nodetool , "seeds.moneroseeds.ch" , "seeds.moneroseeds.li" }; + + bool islimitup=false; + bool islimitdown=false; typedef COMMAND_REQUEST_STAT_INFO_T<typename t_payload_net_handler::stat_info> COMMAND_REQUEST_STAT_INFO; @@ -199,6 +204,20 @@ namespace nodetool template <class Container> bool parse_peers_and_add_to_container(const boost::program_options::variables_map& vm, const command_line::arg_descriptor<std::vector<std::string> > & arg, Container& container); + bool set_max_out_peers(const boost::program_options::variables_map& vm, int64_t max); + bool set_tos_flag(const boost::program_options::variables_map& vm, int limit); + + bool set_rate_up_limit(const boost::program_options::variables_map& vm, int64_t limit); + bool set_rate_down_limit(const boost::program_options::variables_map& vm, int64_t limit); + bool set_rate_limit(const boost::program_options::variables_map& vm, uint64_t limit); + + void kill() { ///< will be called e.g. from deinit() + _info("Killing the net_node"); + is_closing = true; + mPeersLoggerThread->join(); // make sure the thread finishes + _info("Joined extra background net_node threads"); + } + //debug functions std::string print_connections_container(); @@ -216,7 +235,16 @@ namespace nodetool END_KV_SERIALIZE_MAP() }; - config m_config; + public: + config m_config; // TODO was private, add getters? + std::atomic<unsigned int> m_current_number_of_out_peers; + + void set_save_graph(bool save_graph) + { + m_save_graph = save_graph; + epee::net_utils::connection_basic::set_save_graph(save_graph); + } + private: std::string m_config_folder; bool m_have_address; @@ -226,7 +254,10 @@ namespace nodetool uint32_t m_ip_address; bool m_allow_local_ip; bool m_hide_my_port; - + bool m_no_igd; + std::atomic<bool> m_save_graph; + std::atomic<bool> is_closing; + std::unique_ptr<std::thread> mPeersLoggerThread; //critical_section m_connections_lock; //connections_indexed_container m_connections; diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index b6337e6b5..cede6d0a0 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -31,6 +31,9 @@ #pragma once #include <algorithm> +#include <boost/date_time/posix_time/posix_time.hpp> +#include <boost/thread/thread.hpp> +#include <atomic> #include "version.h" #include "string_tools.h" @@ -43,16 +46,18 @@ #include "net/local_ip.h" #include "crypto/crypto.h" #include "storages/levin_abstract_invoke2.h" +#include "data_logger.hpp" +#include "daemon/command_line_args.h" // We have to look for miniupnpc headers in different places, dependent on if its compiled or external #ifdef UPNP_STATIC - #include <miniupnpc/miniupnpc.h> - #include <miniupnpc/upnpcommands.h> - #include <miniupnpc/upnperrors.h> + #include <miniupnpc/miniupnpc.h> + #include <miniupnpc/upnpcommands.h> + #include <miniupnpc/upnperrors.h> #else - #include "miniupnpc.h" - #include "upnpcommands.h" - #include "upnperrors.h" + #include "miniupnpc.h" + #include "upnpcommands.h" + #include "upnperrors.h" #endif #define NET_MAKE_IP(b1,b2,b3,b4) ((LPARAM)(((DWORD)(b1)<<24)+((DWORD)(b2)<<16)+((DWORD)(b3)<<8)+((DWORD)(b4)))) @@ -81,6 +86,16 @@ namespace nodetool " If this option is given the options add-priority-node and seed-node are ignored"}; const command_line::arg_descriptor<std::vector<std::string> > arg_p2p_seed_node = {"seed-node", "Connect to a node to retrieve peer addresses, and disconnect"}; const command_line::arg_descriptor<bool> arg_p2p_hide_my_port = {"hide-my-port", "Do not announce yourself as peerlist candidate", false, true}; + + const command_line::arg_descriptor<bool> arg_no_igd = {"no-igd", "Disable UPnP port mapping"}; + const command_line::arg_descriptor<int64_t> arg_out_peers = {"out-peers", "set max limit of out peers", -1}; + const command_line::arg_descriptor<int> arg_tos_flag = {"tos-flag", "set TOS flag", -1}; + + const command_line::arg_descriptor<int64_t> arg_limit_rate_up = {"limit-rate-up", "set limit-rate-up [kB/s]", -1}; + const command_line::arg_descriptor<int64_t> arg_limit_rate_down = {"limit-rate-down", "set limit-rate-down [kB/s]", -1}; + const command_line::arg_descriptor<uint64_t> arg_limit_rate = {"limit-rate", "set limit-rate [kB/s]", 128}; + + const command_line::arg_descriptor<bool> arg_save_graph = {"save-graph", "Save data for dr monero", false}; } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> @@ -96,6 +111,13 @@ namespace nodetool command_line::add_arg(desc, arg_p2p_add_exclusive_node); command_line::add_arg(desc, arg_p2p_seed_node); command_line::add_arg(desc, arg_p2p_hide_my_port); + command_line::add_arg(desc, arg_no_igd); + command_line::add_arg(desc, arg_out_peers); + command_line::add_arg(desc, arg_tos_flag); + command_line::add_arg(desc, arg_limit_rate_up); + command_line::add_arg(desc, arg_limit_rate_down); + command_line::add_arg(desc, arg_limit_rate); + command_line::add_arg(desc, arg_save_graph); } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> @@ -117,7 +139,6 @@ namespace nodetool //at this moment we have hardcoded config m_config.m_net_config.handshake_interval = P2P_DEFAULT_HANDSHAKE_INTERVAL; - m_config.m_net_config.connections_count = P2P_DEFAULT_CONNECTIONS_COUNT; m_config.m_net_config.packet_max_size = P2P_DEFAULT_PACKET_MAX_SIZE; //20 MB limit m_config.m_net_config.config_id = 0; // initial config m_config.m_net_config.connection_timeout = P2P_DEFAULT_CONNECTION_TIMEOUT; @@ -162,6 +183,7 @@ namespace nodetool m_port = command_line::get_arg(vm, p2p_bind_arg); m_external_port = command_line::get_arg(vm, arg_p2p_external_port); m_allow_local_ip = command_line::get_arg(vm, arg_p2p_allow_local_ip); + m_no_igd = command_line::get_arg(vm, arg_no_igd); if (command_line::has_arg(vm, arg_p2p_add_peer)) { @@ -175,17 +197,24 @@ namespace nodetool m_command_line_peers.push_back(pe); } } + + if(command_line::has_arg(vm, arg_save_graph)) + { + set_save_graph(true); + } if (command_line::has_arg(vm,arg_p2p_add_exclusive_node)) { if (!parse_peers_and_add_to_container(vm, arg_p2p_add_exclusive_node, m_exclusive_peers)) return false; } + if (command_line::has_arg(vm, arg_p2p_add_priority_node)) { if (!parse_peers_and_add_to_container(vm, arg_p2p_add_priority_node, m_priority_peers)) return false; } + if (command_line::has_arg(vm, arg_p2p_seed_node)) { if (!parse_peers_and_add_to_container(vm, arg_p2p_seed_node, m_seed_nodes)) @@ -194,6 +223,21 @@ namespace nodetool if(command_line::has_arg(vm, arg_p2p_hide_my_port)) m_hide_my_port = true; + + if ( !set_max_out_peers(vm, command_line::get_arg(vm, arg_out_peers) ) ) + return false; + + if ( !set_tos_flag(vm, command_line::get_arg(vm, arg_tos_flag) ) ) + return false; + + if ( !set_rate_up_limit(vm, command_line::get_arg(vm, arg_limit_rate_up) ) ) + return false; + + if ( !set_rate_down_limit(vm, command_line::get_arg(vm, arg_limit_rate_down) ) ) + return false; + + if ( !set_rate_limit(vm, command_line::get_arg(vm, arg_limit_rate) ) ) + return false; return true; } @@ -238,44 +282,117 @@ namespace nodetool //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::init(const boost::program_options::variables_map& vm, bool testnet) + bool node_server<t_payload_net_handler>::init(const boost::program_options::variables_map& vm) { + bool testnet = command_line::get_arg(vm, daemon_args::arg_testnet_on); + if (testnet) { - append_net_address(m_seed_nodes, "107.152.187.202:28080"); + memcpy(&m_network_id, &::config::testnet::NETWORK_ID, 16); append_net_address(m_seed_nodes, "197.242.158.240:28080"); append_net_address(m_seed_nodes, "107.152.130.98:28080"); + append_net_address(m_seed_nodes, "5.9.25.103:28080"); + append_net_address(m_seed_nodes, "5.9.55.70:28080"); } else { + memcpy(&m_network_id, &::config::NETWORK_ID, 16); // for each hostname in the seed nodes list, attempt to DNS resolve and // add the result addresses as seed nodes // TODO: at some point add IPv6 support, but that won't be relevant // for some time yet. + + std::vector<std::vector<std::string>> dns_results; + dns_results.resize(m_seed_nodes_list.size()); + + std::list<boost::thread*> dns_threads; + uint64_t result_index = 0; for (const std::string& addr_str : m_seed_nodes_list) { - // TODO: care about dnssec avail/valid - bool avail, valid; - std::vector<std::string> addr_list = tools::DNSResolver::instance().get_ipv4(addr_str, avail, valid); - for (const std::string& a : addr_list) + boost::thread* th = new boost::thread([=, &dns_results, &addr_str] + { + LOG_PRINT_L4("dns_threads[" << result_index << "] created for: " << addr_str) + // TODO: care about dnssec avail/valid + bool avail, valid; + std::vector<std::string> addr_list; + + try + { + addr_list = tools::DNSResolver().get_ipv4(addr_str, avail, valid); + LOG_PRINT_L4("dns_threads[" << result_index << "] DNS resolve done"); + boost::this_thread::interruption_point(); + } + catch(const boost::thread_interrupted&) + { + // thread interruption request + // even if we now have results, finish thread without setting + // result variables, which are now out of scope in main thread + LOG_PRINT_L4("dns_threads[" << result_index << "] interrupted"); + return; + } + + LOG_PRINT_L4("dns_threads[" << result_index << "] addr_str: " << addr_str << " number of results: " << addr_list.size()); + dns_results[result_index] = addr_list; + }); + + dns_threads.push_back(th); + ++result_index; + } + + LOG_PRINT_L4("dns_threads created, now waiting for completion or timeout of " << CRYPTONOTE_DNS_TIMEOUT_MS << "ms"); + boost::chrono::system_clock::time_point deadline = boost::chrono::system_clock::now() + boost::chrono::milliseconds(CRYPTONOTE_DNS_TIMEOUT_MS); + uint64_t i = 0; + for (boost::thread* th : dns_threads) + { + if (! th->try_join_until(deadline)) + { + LOG_PRINT_L4("dns_threads[" << i << "] timed out, sending interrupt"); + th->interrupt(); + } + ++i; + } + + i = 0; + for (const auto& result : dns_results) + { + LOG_PRINT_L4("DNS lookup for " << m_seed_nodes_list[i] << ": " << result.size() << " results"); + // if no results for node, thread's lookup likely timed out + if (result.size()) { - append_net_address(m_seed_nodes, a + ":18080"); + for (const auto& addr_string : result) + { + append_net_address(m_seed_nodes, addr_string + ":18080"); + } } + ++i; } if (!m_seed_nodes.size()) { - append_net_address(m_seed_nodes, "62.210.78.186:18080"); - append_net_address(m_seed_nodes, "195.12.60.154:18080"); - append_net_address(m_seed_nodes, "54.241.246.125:18080"); - append_net_address(m_seed_nodes, "107.170.157.169:18080"); - append_net_address(m_seed_nodes, "54.207.112.216:18080"); - append_net_address(m_seed_nodes, "78.27.112.54:18080"); - append_net_address(m_seed_nodes, "209.222.30.57:18080"); - append_net_address(m_seed_nodes, "80.71.13.55:18080"); - append_net_address(m_seed_nodes, "107.178.112.126:18080"); - append_net_address(m_seed_nodes, "107.158.233.98:18080"); - append_net_address(m_seed_nodes, "64.22.111.2:18080"); + LOG_PRINT_L0("DNS seed node lookup either timed out or failed, falling back to defaults"); + append_net_address(m_seed_nodes, "46.165.232.77:18080"); + append_net_address(m_seed_nodes, "63.141.254.186:18080"); + append_net_address(m_seed_nodes, ":18080"); + append_net_address(m_seed_nodes, "119.81.118.164:18080"); + append_net_address(m_seed_nodes, "60.191.33.112:18080"); + append_net_address(m_seed_nodes, "198.74.231.92:18080"); + append_net_address(m_seed_nodes, "5.9.55.70:18080"); + append_net_address(m_seed_nodes, "119.81.118.165:18080"); + append_net_address(m_seed_nodes, "202.112.0.100:18080"); + append_net_address(m_seed_nodes, "84.106.163.174:18080"); + append_net_address(m_seed_nodes, "178.206.94.87:18080"); + append_net_address(m_seed_nodes, "119.81.118.163:18080"); + append_net_address(m_seed_nodes, "95.37.217.253:18080"); + append_net_address(m_seed_nodes, "161.67.132.39:18080"); + append_net_address(m_seed_nodes, "119.81.48.114:18080"); + append_net_address(m_seed_nodes, "119.81.118.166:18080"); + append_net_address(m_seed_nodes, "93.120.240.209:18080"); + append_net_address(m_seed_nodes, "46.183.145.69:18080"); + append_net_address(m_seed_nodes, "108.170.123.66:18080"); + append_net_address(m_seed_nodes, "5.9.83.204:18080"); + append_net_address(m_seed_nodes, "104.130.19.193:18080"); + append_net_address(m_seed_nodes, "119.81.48.115:18080"); + append_net_address(m_seed_nodes, "80.71.13.36:18080"); } } @@ -316,42 +433,43 @@ namespace nodetool LOG_PRINT_L0("External port defined as " << m_external_port); // Add UPnP port mapping - LOG_PRINT_L0("Attempting to add IGD port mapping."); - int result; - UPNPDev* deviceList = upnpDiscover(1000, NULL, NULL, 0, 0, &result); - UPNPUrls urls; - IGDdatas igdData; - char lanAddress[64]; - result = UPNP_GetValidIGD(deviceList, &urls, &igdData, lanAddress, sizeof lanAddress); - freeUPNPDevlist(deviceList); - if (result != 0) { - if (result == 1) { - std::ostringstream portString; - portString << m_listenning_port; - - // Delete the port mapping before we create it, just in case we have dangling port mapping from the daemon not being shut down correctly - UPNP_DeletePortMapping(urls.controlURL, igdData.first.servicetype, portString.str().c_str(), "TCP", 0); - - int portMappingResult; - portMappingResult = UPNP_AddPortMapping(urls.controlURL, igdData.first.servicetype, portString.str().c_str(), portString.str().c_str(), lanAddress, CRYPTONOTE_NAME, "TCP", 0, "0"); - if (portMappingResult != 0) { - LOG_ERROR("UPNP_AddPortMapping failed, error: " << strupnperror(portMappingResult)); - } else { - LOG_PRINT_GREEN("Added IGD port mapping.", LOG_LEVEL_0); - } - } else if (result == 2) { - LOG_PRINT_L0("IGD was found but reported as not connected."); - } else if (result == 3) { - LOG_PRINT_L0("UPnP device was found but not recoginzed as IGD."); - } else { - LOG_ERROR("UPNP_GetValidIGD returned an unknown result code."); - } - - FreeUPNPUrls(&urls); - } else { - LOG_PRINT_L0("No IGD was found."); - } - + if(m_no_igd == false) { + LOG_PRINT_L0("Attempting to add IGD port mapping."); + int result; + UPNPDev* deviceList = upnpDiscover(1000, NULL, NULL, 0, 0, &result); + UPNPUrls urls; + IGDdatas igdData; + char lanAddress[64]; + result = UPNP_GetValidIGD(deviceList, &urls, &igdData, lanAddress, sizeof lanAddress); + freeUPNPDevlist(deviceList); + if (result != 0) { + if (result == 1) { + std::ostringstream portString; + portString << m_listenning_port; + + // Delete the port mapping before we create it, just in case we have dangling port mapping from the daemon not being shut down correctly + UPNP_DeletePortMapping(urls.controlURL, igdData.first.servicetype, portString.str().c_str(), "TCP", 0); + + int portMappingResult; + portMappingResult = UPNP_AddPortMapping(urls.controlURL, igdData.first.servicetype, portString.str().c_str(), portString.str().c_str(), lanAddress, CRYPTONOTE_NAME, "TCP", 0, "0"); + if (portMappingResult != 0) { + LOG_ERROR("UPNP_AddPortMapping failed, error: " << strupnperror(portMappingResult)); + } else { + LOG_PRINT_GREEN("Added IGD port mapping.", LOG_LEVEL_0); + } + } else if (result == 2) { + LOG_PRINT_L0("IGD was found but reported as not connected."); + } else if (result == 3) { + LOG_PRINT_L0("UPnP device was found but not recoginzed as IGD."); + } else { + LOG_ERROR("UPNP_GetValidIGD returned an unknown result code."); + } + + FreeUPNPUrls(&urls); + } else { + LOG_PRINT_L0("No IGD was found."); + } + } return res; } //----------------------------------------------------------------------------------- @@ -364,6 +482,30 @@ namespace nodetool template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::run() { + // creating thread to log number of connections + mPeersLoggerThread.reset(new std::thread([&]() + { + _note("Thread monitor number of peers - start"); + while (!is_closing) + { // main loop of thread + //number_of_peers = m_net_server.get_config_object().get_connections_count(); + unsigned int number_of_peers = 0; + m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + { + if (!cntxt.m_is_income) ++number_of_peers; + return true; + }); // lambda + + m_current_number_of_out_peers = number_of_peers; + if (epee::net_utils::data_logger::is_dying()) + break; + epee::net_utils::data_logger::get_instance().add_data("peers", number_of_peers); + + std::this_thread::sleep_for(std::chrono::seconds(1)); + } // main loop of thread + _note("Thread monitor number of peers - done"); + })); // lambda + //here you can set worker threads count int thrds_count = 10; @@ -394,6 +536,7 @@ namespace nodetool template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::deinit() { + kill(); m_peerlist.deinit(); m_net_server.deinit_server(); return store_config(); @@ -606,6 +749,16 @@ namespace nodetool template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::try_to_connect_and_handshake_with_new_peer(const net_address& na, bool just_take_peerlist, uint64_t last_seen_stamp, bool white) { + if (m_current_number_of_out_peers == m_config.m_net_config.connections_count) // out peers limit + { + return false; + } + else if (m_current_number_of_out_peers > m_config.m_net_config.connections_count) + { + m_net_server.get_config_object().del_out_connections(1); + m_current_number_of_out_peers --; // atomic variable, update time = 1s + return false; + } LOG_PRINT_L1("Connecting to " << epee::string_tools::get_ip_string_from_int32(na.ip) << ":" << epee::string_tools::num_to_string_fast(na.port) << "(white=" << white << ", last_seen: " << (last_seen_stamp ? epee::misc_utils::get_time_interval_string(time(NULL) - last_seen_stamp):"never") @@ -693,16 +846,22 @@ namespace nodetool ++try_count; - if(is_peer_used(pe)) + _note("Considering connecting (out) to peer: " << pe.id << " " << epee::string_tools::get_ip_string_from_int32(pe.adr.ip) << ":" << boost::lexical_cast<std::string>(pe.adr.port)); + + if(is_peer_used(pe)) { + _note("Peer is used"); continue; + } LOG_PRINT_L1("Selected peer: " << pe.id << " " << epee::string_tools::get_ip_string_from_int32(pe.adr.ip) << ":" << boost::lexical_cast<std::string>(pe.adr.port) << "[white=" << use_white_list << "] last_seen: " << (pe.last_seen ? epee::misc_utils::get_time_interval_string(time(NULL) - pe.last_seen) : "never")); - if(!try_to_connect_and_handshake_with_new_peer(pe.adr, false, pe.last_seen, use_white_list)) + if(!try_to_connect_and_handshake_with_new_peer(pe.adr, false, pe.last_seen, use_white_list)) { + _note("Handshake failed"); continue; + } return true; } @@ -1241,4 +1400,83 @@ namespace nodetool return true; } + + template<class t_payload_net_handler> + bool node_server<t_payload_net_handler>::set_max_out_peers(const boost::program_options::variables_map& vm, int64_t max) + { + if(max == -1) { + m_config.m_net_config.connections_count = P2P_DEFAULT_CONNECTIONS_COUNT; + epee::net_utils::data_logger::get_instance().add_data("peers_limit", m_config.m_net_config.connections_count); + return true; + } + epee::net_utils::data_logger::get_instance().add_data("peers_limit", max); + m_config.m_net_config.connections_count = max; + return true; + } + + template<class t_payload_net_handler> + void node_server<t_payload_net_handler>::delete_connections(size_t count) + { + m_net_server.get_config_object().del_out_connections(count); + } + + template<class t_payload_net_handler> + bool node_server<t_payload_net_handler>::set_tos_flag(const boost::program_options::variables_map& vm, int flag) + { + if(flag==-1){ + return true; + } + epee::net_utils::connection<epee::levin::async_protocol_handler<p2p_connection_context> >::set_tos_flag(flag); + _dbg1("Set ToS flag " << flag); + return true; + } + + template<class t_payload_net_handler> + bool node_server<t_payload_net_handler>::set_rate_up_limit(const boost::program_options::variables_map& vm, int64_t limit) + { + this->islimitup=true; + + if (limit==-1) { + limit=128; + this->islimitup=false; + } + + limit *= 1024; + epee::net_utils::connection<epee::levin::async_protocol_handler<p2p_connection_context> >::set_rate_up_limit( limit ); + LOG_PRINT_L0("Set limit-up to " << limit/1024 << " kB/s"); + return true; + } + + template<class t_payload_net_handler> + bool node_server<t_payload_net_handler>::set_rate_down_limit(const boost::program_options::variables_map& vm, int64_t limit) + { + this->islimitdown=true; + if(limit==-1) { + limit=128; + this->islimitdown=false; + } + limit *= 1024; + epee::net_utils::connection<epee::levin::async_protocol_handler<p2p_connection_context> >::set_rate_down_limit( limit ); + LOG_PRINT_L0("Set limit-down to " << limit/1024 << " kB/s"); + return true; + } + + template<class t_payload_net_handler> + bool node_server<t_payload_net_handler>::set_rate_limit(const boost::program_options::variables_map& vm, uint64_t limit) + { + limit *= 1024; + if(this->islimitdown==false && this->islimitup==false) { + epee::net_utils::connection<epee::levin::async_protocol_handler<p2p_connection_context> >::set_rate_up_limit( limit ); + epee::net_utils::connection<epee::levin::async_protocol_handler<p2p_connection_context> >::set_rate_down_limit( limit ); + LOG_PRINT_L0("Set limit to " << limit/1024 << " kB/s"); + } + else if(this->islimitdown==false && this->islimitup==true ) { + epee::net_utils::connection<epee::levin::async_protocol_handler<p2p_connection_context> >::set_rate_down_limit( limit ); + } + else if(this->islimitdown==true && this->islimitup==false ) { + epee::net_utils::connection<epee::levin::async_protocol_handler<p2p_connection_context> >::set_rate_up_limit( limit ); + } + + return true; + } } diff --git a/src/p2p/net_node_common.h b/src/p2p/net_node_common.h index 6559963e0..a7b8bf6f3 100644 --- a/src/p2p/net_node_common.h +++ b/src/p2p/net_node_common.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/p2p/net_peerlist.h b/src/p2p/net_peerlist.h index e3e387bbc..9407e7bce 100644 --- a/src/p2p/net_peerlist.h +++ b/src/p2p/net_peerlist.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/p2p/net_peerlist_boost_serialization.h b/src/p2p/net_peerlist_boost_serialization.h index 17525e8fe..d0bd44ef0 100644 --- a/src/p2p/net_peerlist_boost_serialization.h +++ b/src/p2p/net_peerlist_boost_serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/p2p/network_throttle-detail.cpp b/src/p2p/network_throttle-detail.cpp new file mode 100644 index 000000000..6fa27b62a --- /dev/null +++ b/src/p2p/network_throttle-detail.cpp @@ -0,0 +1,382 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief implementaion for throttling of connection (count and rate-limit speed etc) + +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/* rfree: implementation for throttle details */ + +#include <boost/asio.hpp> +#include <string> +#include <vector> +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <atomic> + +#include <boost/asio.hpp> +#include <boost/array.hpp> +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <boost/enable_shared_from_this.hpp> +#include <boost/interprocess/detail/atomic.hpp> +#include <boost/thread/thread.hpp> + +#include <memory> + +#include "syncobj.h" + +#include "../../contrib/epee/include/net/net_utils_base.h" +#include "../../contrib/epee/include/misc_log_ex.h" +#include <boost/lambda/bind.hpp> +#include <boost/foreach.hpp> +#include <boost/lambda/lambda.hpp> +#include <boost/uuid/random_generator.hpp> +#include <boost/chrono.hpp> +#include <boost/utility/value_init.hpp> +#include <boost/asio/deadline_timer.hpp> +#include <boost/date_time/posix_time/posix_time.hpp> +#include <boost/thread/thread.hpp> +#include "misc_language.h" +#include "pragma_comp_defs.h" +#include <sstream> +#include <iomanip> +#include <algorithm> + + + +#include <boost/asio/basic_socket.hpp> +#include <boost/asio/ip/unicast.hpp> +#include "../../contrib/epee/include/net/abstract_tcp_server2.h" + +// TODO: +#include "../../src/p2p/network_throttle-detail.hpp" + +#include "../../contrib/otshell_utils/utils.hpp" +#include "data_logger.hpp" +using namespace nOT::nUtils; + +// ################################################################################################ +// ################################################################################################ +// the "header part". Not separeted out for .hpp because point of this modification is +// to rebuild just 1 translation unit while working on this code. +// (But maybe common parts will be separated out later though - if needed) +// ################################################################################################ +// ################################################################################################ + +using namespace nOT::nUtils; + +namespace epee +{ +namespace net_utils +{ + + +/* ============================================================================ */ + +class connection_basic_pimpl { + public: + connection_basic_pimpl(const std::string &name); + + static int m_default_tos; + + network_throttle_bw m_throttle; // per-perr + critical_section m_throttle_lock; + + void _packet(size_t packet_size, int phase, int q_len); // execute a sleep ; phase is not really used now(?) could be used for different kinds of sleep e.g. direct/queue write +}; + + +} // namespace +} // namespace + + + + + + +// ################################################################################################ +// ################################################################################################ +// The implementation part +// ################################################################################################ +// ################################################################################################ + +namespace epee +{ +namespace net_utils +{ + +// ================================================================================================ +// network_throttle +// ================================================================================================ + +network_throttle::~network_throttle() { } + +network_throttle::packet_info::packet_info() + : m_size(0) +{ +} + +network_throttle::network_throttle(const std::string &nameshort, const std::string &name, int window_size) + : m_window_size( (window_size==-1) ? 10 : window_size ), + m_history( m_window_size ), m_nameshort(nameshort) +{ + set_name(name); + m_network_add_cost = 128; + m_network_minimal_segment = 256; + m_network_max_segment = 1024*1024; + m_any_packet_yet = false; + m_slot_size = 1.0; // hard coded in few places + m_target_speed = 16 * 1024; // other defaults are probably defined in the command-line parsing code when this class is used e.g. as main global throttle +} + +void network_throttle::set_name(const std::string &name) +{ + m_name = name; +} + +void network_throttle::set_target_speed( network_speed_kbps target ) +{ + m_target_speed = target * 1024; + _note_c("net/"+m_nameshort, "Setting LIMIT: " << target << " kbps"); + set_real_target_speed(target); +} + +void network_throttle::set_real_target_speed( network_speed_kbps real_target ) +{ + m_real_target_speed = real_target * 1024; +} + +network_speed_kbps network_throttle::get_terget_speed() +{ + return m_real_target_speed / 1024; +} + +void network_throttle::tick() +{ + double time_now = get_time_seconds(); + if (!m_any_packet_yet) m_start_time = time_now; // starting now + + network_time_seconds current_sample_time_slot = time_to_slot( time_now ); // T=13.7 --> 13 (for 1-second smallwindow) + network_time_seconds last_sample_time_slot = time_to_slot( m_last_sample_time ); + + // moving to next position, and filling gaps + // !! during this loop the m_last_sample_time and last_sample_time_slot mean the variable moved in +1 + // TODO optimize when moving few slots at once + while ( (!m_any_packet_yet) || (last_sample_time_slot < current_sample_time_slot)) + { + _dbg3("Moving counter buffer by 1 second " << last_sample_time_slot << " < " << current_sample_time_slot << " (last time " << m_last_sample_time<<")"); + // rotate buffer + for (size_t i=m_history.size()-1; i>=1; --i) m_history[i] = m_history[i-1]; + m_history[0] = packet_info(); + if (! m_any_packet_yet) + { + m_last_sample_time = time_now; + } + m_last_sample_time += 1; last_sample_time_slot = time_to_slot( m_last_sample_time ); // increase and recalculate time, time slot + m_any_packet_yet=true; + } + m_last_sample_time = time_now; // the real exact last time +} + +void network_throttle::handle_trafic_exact(size_t packet_size) +{ + _handle_trafic_exact(packet_size, packet_size); +} + +void network_throttle::_handle_trafic_exact(size_t packet_size, size_t orginal_size) +{ + tick(); + + calculate_times_struct cts ; calculate_times(packet_size, cts , false, -1); + calculate_times_struct cts2; calculate_times(packet_size, cts2, false, 5); + m_history[0].m_size += packet_size; + + std::ostringstream oss; oss << "["; for (auto sample: m_history) oss << sample.m_size << " "; oss << "]" << std::ends; + std::string history_str = oss.str(); + + _dbg2_c( "net/" + m_nameshort , "Throttle " << m_name << ": packet of ~"<<packet_size<<"b " << " (from "<<orginal_size<<" b)" + << " Speed AVG=" << std::setw(4) << ((long int)(cts .average/1024)) <<"[w="<<cts .window<<"]" + << " " << std::setw(4) << ((long int)(cts2.average/1024)) <<"[w="<<cts2.window<<"]" + <<" / " << " Limit="<< ((long int)(m_target_speed/1024)) <<" KiB/sec " + << " " << history_str + ); +} + +void network_throttle::handle_trafic_tcp(size_t packet_size) +{ + size_t all_size = packet_size + m_network_add_cost; + all_size = std::max( m_network_minimal_segment , all_size); + _handle_trafic_exact( all_size , packet_size ); +} + +network_time_seconds network_throttle::get_sleep_time_after_tick(size_t packet_size) { + tick(); + return get_sleep_time(packet_size); +} + +void network_throttle::logger_handle_net(const std::string &filename, double time, size_t size) { + if (! epee::net_utils::data_logger::m_save_graph) + return; + std::mutex mutex; + mutex.lock(); { + std::fstream file; + file.open(filename.c_str(), std::ios::app | std::ios::out ); + file.precision(6); + if(!file.is_open()) + _warn("Can't open file " << filename); + file << static_cast<int>(time) << " " << static_cast<double>(size/1024) << "\n"; + file.close(); + } mutex.unlock(); +} + +// fine tune this to decide about sending speed: +network_time_seconds network_throttle::get_sleep_time(size_t packet_size) const +{ + double D2=0; + calculate_times_struct cts = { 0, 0, 0, 0}; + calculate_times(packet_size, cts, true, m_window_size); D2=cts.delay; + return D2; +} + +// MAIN LOGIC: +void network_throttle::calculate_times(size_t packet_size, calculate_times_struct &cts, bool dbg, double force_window) const +{ + const double the_window_size = std::max( (double)m_window_size , + ((force_window>0) ? force_window : m_window_size) + ); + + if (!m_any_packet_yet) { + cts.window=0; cts.average=0; cts.delay=0; + cts.recomendetDataSize = m_network_minimal_segment; // should be overrided by caller anyway + return ; // no packet yet, I can not decide about sleep time + } + + network_time_seconds window_len = (the_window_size-1) * m_slot_size ; // -1 since current slot is not finished + window_len += (m_last_sample_time - time_to_slot(m_last_sample_time)); // add the time for current slot e.g. 13.7-13 = 0.7 + + auto time_passed = get_time_seconds() - m_start_time; + cts.window = std::max( std::min( window_len , time_passed ) , m_slot_size ) ; // window length resulting from size of history but limited by how long ago history was started, + // also at least slot size (e.g. 1 second) to not be ridiculous + // window_len e.g. 5.7 because takes into account current slot time + + size_t Epast = 0; // summ of traffic till now + for (auto sample : m_history) Epast += sample.m_size; + + const size_t E = Epast; + const size_t Enow = Epast + packet_size ; // including the data we're about to send now + + const double M = m_target_speed; // max + const double D1 = (Epast - M*cts.window) / M; // delay - how long to sleep to get back to target speed + const double D2 = (Enow - M*cts.window) / M; // delay - how long to sleep to get back to target speed (including current packet) + + cts.delay = (D1*0.80 + D2*0.20); // finall sleep depends on both with/without current packet + // update_overheat(); + cts.average = Epast/cts.window; // current avg. speed (for info) + + if (Epast <= 0) { + if (cts.delay>=0) cts.delay = 0; // no traffic in history so we will not wait + } + + double Wgood=-1; + { // how much data we recommend now to download + Wgood = the_window_size + 1; + cts.recomendetDataSize = M*cts.window - E; + } + + if (dbg) { + std::ostringstream oss; oss << "["; for (auto sample: m_history) oss << sample.m_size << " "; oss << "]" << std::ends; + std::string history_str = oss.str(); + _dbg1_c( "net/"+m_nameshort+"_c" , + (cts.delay > 0 ? "SLEEP" : "") + << "dbg " << m_name << ": " + << "speed is A=" << std::setw(8) <<cts.average<<" vs " + << "Max=" << std::setw(8) <<M<<" " + << " so sleep: " + << "D=" << std::setw(8) <<cts.delay<<" sec " + << "E="<< std::setw(8) << E << " (Enow="<<std::setw(8)<<Enow<<") " + << "M=" << std::setw(8) << M <<" W="<< std::setw(8) << cts.window << " " + << "R=" << std::setw(8) << cts.recomendetDataSize << " Wgood" << std::setw(8) << Wgood << " " + << "History: " << std::setw(8) << history_str << " " + << "m_last_sample_time=" << std::setw(8) << m_last_sample_time + ); + + } +} + +double network_throttle::get_time_seconds() const { + #if defined(__APPLE__) + auto point = std::chrono::system_clock::now(); + #else + auto point = std::chrono::steady_clock::now(); + #endif + auto time_from_epoh = point.time_since_epoch(); + auto ms = std::chrono::duration_cast< std::chrono::milliseconds >( time_from_epoh ).count(); + double ms_f = ms; + return ms_f / 1000.; +} + +size_t network_throttle::get_recommended_size_of_planned_transport_window(double force_window) const { + calculate_times_struct cts = { 0, 0, 0, 0}; + network_throttle::calculate_times(0, cts, true, force_window); + cts.recomendetDataSize += m_network_add_cost; + if (cts.recomendetDataSize<0) cts.recomendetDataSize=0; + if (cts.recomendetDataSize>m_network_max_segment) cts.recomendetDataSize=m_network_max_segment; + size_t RI = (long int)cts.recomendetDataSize; + return RI; +} + +size_t network_throttle::get_recommended_size_of_planned_transport() const { + size_t R1=0,R2=0,R3=0; + R1 = get_recommended_size_of_planned_transport_window( -1 ); + R2 = get_recommended_size_of_planned_transport_window(m_window_size / 2); + R3 = get_recommended_size_of_planned_transport_window( 5 ); + auto RM = std::min(R1, std::min(R2,R3)); + + const double a1=20, a2=10, a3=10, am=10; // weight of the various windows in decisssion // TODO 70 => 20 + return (R1*a1 + R2*a2 + R3*a3 + RM*am) / (a1+a2+a3+am); +} + +double network_throttle::get_current_speed() const { + unsigned int bytes_transferred = 0; + if (m_history.size() == 0 || m_slot_size == 0) + return 0; + + auto it = m_history.begin(); + while (it < m_history.end() - 1) + { + bytes_transferred += it->m_size; + it ++; + } + + return bytes_transferred / ((m_history.size() - 1) * m_slot_size); +} + +} // namespace +} // namespace + diff --git a/src/p2p/network_throttle-detail.hpp b/src/p2p/network_throttle-detail.hpp new file mode 100644 index 000000000..063dac850 --- /dev/null +++ b/src/p2p/network_throttle-detail.hpp @@ -0,0 +1,131 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief implementaion for throttling of connection (count and rate-limit speed etc) + +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +/* rfree: throttle details, implementing rate limiting */ + + +#ifndef INCLUDED_src_p2p_throttle_detail_hpp +#define INCLUDED_src_p2p_throttle_detail_hpp + +#include "../../src/p2p/network_throttle.hpp" + +namespace epee +{ +namespace net_utils +{ + + +class network_throttle : public i_network_throttle { + private: + struct packet_info { + size_t m_size; // octets sent. Summary for given small-window (e.g. for all packaged in 1 second) + packet_info(); + }; + + + network_speed_kbps m_target_speed; + network_speed_kbps m_real_target_speed; + size_t m_network_add_cost; // estimated add cost of headers + size_t m_network_minimal_segment; // estimated minimal cost of sending 1 byte to round up to + size_t m_network_max_segment; // recommended max size of 1 TCP transmission + + const size_t m_window_size; // the number of samples to average over + network_time_seconds m_slot_size; // the size of one slot. TODO: now hardcoded for 1 second e.g. in time_to_slot() + // TODO for big window size, for performance better the substract on change of m_last_sample_time instead of recalculating average of eg >100 elements + + std::vector< packet_info > m_history; // the history of bw usage + network_time_seconds m_last_sample_time; // time of last history[0] - so we know when to rotate the buffer + network_time_seconds m_start_time; // when we were created + bool m_any_packet_yet; // did we yet got any packet to count + + double m_overheat; // last overheat + double m_overheat_time; // time in seconds after epoch + + std::string m_name; // my name for debug and logs + std::string m_nameshort; // my name for debug and logs (used in log file name) + + // each sample is now 1 second + public: + network_throttle(const std::string &nameshort, const std::string &name, int window_size=-1); + virtual ~network_throttle(); + virtual void set_name(const std::string &name); + virtual void set_target_speed( network_speed_kbps target ); + virtual void set_real_target_speed( network_speed_kbps real_target ); // only for throttle_out + virtual network_speed_kbps get_terget_speed(); + + // add information about events: + virtual void handle_trafic_exact(size_t packet_size); ///< count the new traffic/packet; the size is exact considering all network costs + virtual void handle_trafic_tcp(size_t packet_size); ///< count the new traffic/packet; the size is as TCP, we will consider MTU etc + + virtual void tick(); ///< poke and update timers/history (recalculates, moves the history if needed, checks the real clock etc) + + virtual double get_time_seconds() const ; ///< timer that we use, time in seconds, monotionic + + // time calculations: + virtual void calculate_times(size_t packet_size, calculate_times_struct &cts, bool dbg, double force_window) const; ///< MAIN LOGIC (see base class for info) + + virtual network_time_seconds get_sleep_time_after_tick(size_t packet_size); ///< increase the timer if needed, and get the package size + virtual network_time_seconds get_sleep_time(size_t packet_size) const; ///< gets the Delay (recommended Delay time) from calc. (not safe: only if time didnt change?) TODO + + virtual size_t get_recommended_size_of_planned_transport() const; ///< what should be the size (bytes) of next data block to be transported + virtual size_t get_recommended_size_of_planned_transport_window(double force_window) const; ///< ditto, but for given windows time frame + virtual double get_current_speed() const; + + private: + virtual network_time_seconds time_to_slot(network_time_seconds t) const { return std::floor( t ); } // convert exact time eg 13.7 to rounded time for slot number in history 13 + virtual void _handle_trafic_exact(size_t packet_size, size_t orginal_size); + virtual void logger_handle_net(const std::string &filename, double time, size_t size); +}; + +/*** + * The complete set of traffic throttle for one typical connection +*/ +struct network_throttle_bw { + public: + network_throttle m_in; ///< for incomming traffic (this we can not controll directly as it depends of what others send to us - usually) + network_throttle m_inreq; ///< for requesting incomming traffic (this is exact usually) + network_throttle m_out; ///< for outgoing traffic that we just sent (this is exact usually) + + public: + network_throttle_bw(const std::string &name1); +}; + + + +} // namespace net_utils +} // namespace epee + + +#endif + + diff --git a/src/p2p/network_throttle.cpp b/src/p2p/network_throttle.cpp new file mode 100644 index 000000000..7bc89881d --- /dev/null +++ b/src/p2p/network_throttle.cpp @@ -0,0 +1,120 @@ +/** +@file +@author rfree (current maintainer in monero.cc project) +@brief interface for throttling of connection (count and rate-limit speed etc) +@details <PRE> + +Throttling work by: +1) taking note of all traffic (hooks added e.g. to connection class) and measuring speed +2) depending on that information we sleep before sending out data (or send smaller portions of data) +3) depending on the information we can also sleep before sending requests or ask for smaller sets of data to download + +</PRE> + +@image html images/net/rate1-down-1k.png +@image html images/net/rate1-down-full.png +@image html images/net/rate1-up-10k.png +@image html images/net/rate1-up-full.png +@image html images/net/rate2-down-100k.png +@image html images/net/rate2-down-10k.png +@image html images/net/rate2-down-50k.png +@image html images/net/rate2-down-full.png +@image html images/net/rate2-up-100k.png +@image html images/net/rate2-up-10k.png +@image html images/net/rate3-up-10k.png + + +*/ + +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "../../src/p2p/network_throttle-detail.hpp" + +namespace epee +{ +namespace net_utils +{ + +// ================================================================================================ +// network_throttle_manager +// ================================================================================================ + +// ================================================================================================ +// static: +std::mutex network_throttle_manager::m_lock_get_global_throttle_in; +std::mutex network_throttle_manager::m_lock_get_global_throttle_inreq; +std::mutex network_throttle_manager::m_lock_get_global_throttle_out; + +int network_throttle_manager::xxx; + + +// ================================================================================================ +// methods: +i_network_throttle & network_throttle_manager::get_global_throttle_in() { + std::call_once(m_once_get_global_throttle_in, [] { m_obj_get_global_throttle_in.reset(new network_throttle("in/all","<<< global-IN",10)); } ); + return * m_obj_get_global_throttle_in; +} +std::once_flag network_throttle_manager::m_once_get_global_throttle_in; +std::unique_ptr<i_network_throttle> network_throttle_manager::m_obj_get_global_throttle_in; + + + +i_network_throttle & network_throttle_manager::get_global_throttle_inreq() { + std::call_once(m_once_get_global_throttle_inreq, [] { m_obj_get_global_throttle_inreq.reset(new network_throttle("inreq/all", "<== global-IN-REQ",10)); } ); + return * m_obj_get_global_throttle_inreq; +} +std::once_flag network_throttle_manager::m_once_get_global_throttle_inreq; +std::unique_ptr<i_network_throttle> network_throttle_manager::m_obj_get_global_throttle_inreq; + + +i_network_throttle & network_throttle_manager::get_global_throttle_out() { + std::call_once(m_once_get_global_throttle_out, [] { m_obj_get_global_throttle_out.reset(new network_throttle("out/all", ">>> global-OUT",10)); } ); + return * m_obj_get_global_throttle_out; +} +std::once_flag network_throttle_manager::m_once_get_global_throttle_out; +std::unique_ptr<i_network_throttle> network_throttle_manager::m_obj_get_global_throttle_out; + + + + +network_throttle_bw::network_throttle_bw(const std::string &name1) + : m_in("in/"+name1, name1+"-DOWNLOAD"), m_inreq("inreq/"+name1, name1+"-DOWNLOAD-REQUESTS"), m_out("out/"+name1, name1+"-UPLOAD") +{ } + + + + +} // namespace +} // namespace + + + + + diff --git a/src/p2p/network_throttle.hpp b/src/p2p/network_throttle.hpp new file mode 100644 index 000000000..add4daa86 --- /dev/null +++ b/src/p2p/network_throttle.hpp @@ -0,0 +1,185 @@ +/// @file +/// @author rfree (current maintainer in monero.cc project) +/// @brief interface for throttling of connection (count and rate-limit speed etc) + +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +/* rfree: throttle basic interface */ +/* rfree: also includes the manager for singeton/global such objects */ + + +#ifndef INCLUDED_p2p_network_throttle_hpp +#define INCLUDED_p2p_network_throttle_hpp + +#include <boost/asio.hpp> +#include <string> +#include <vector> +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <atomic> + +#include <boost/asio.hpp> +#include <boost/array.hpp> +#include <boost/noncopyable.hpp> +#include <boost/shared_ptr.hpp> +#include <boost/enable_shared_from_this.hpp> +#include <boost/interprocess/detail/atomic.hpp> +#include <boost/thread/thread.hpp> + +#include "syncobj.h" + +#include "../../contrib/epee/include/net/net_utils_base.h" +#include "../../contrib/epee/include/misc_log_ex.h" +#include <boost/lambda/bind.hpp> +#include <boost/foreach.hpp> +#include <boost/lambda/lambda.hpp> +#include <boost/uuid/random_generator.hpp> +#include <boost/chrono.hpp> +#include <boost/utility/value_init.hpp> +#include <boost/asio/deadline_timer.hpp> +#include <boost/date_time/posix_time/posix_time.hpp> +#include <boost/thread/thread.hpp> +#include "misc_language.h" +#include "pragma_comp_defs.h" +#include <sstream> +#include <iomanip> +#include <algorithm> + +#include <memory> +#include <mutex> +#include <fstream> + +namespace epee +{ +namespace net_utils +{ + +// just typedefs to in code define the units used. TODO later it will be enforced that casts to other numericals are only explicit to avoid mistakes? use boost::chrono? +typedef double network_speed_kbps; +typedef double network_time_seconds; +typedef double network_MB; + +class i_network_throttle; + +/*** +@brief All information about given throttle - speed calculations +*/ +struct calculate_times_struct { + double average; + double window; + double delay; + double recomendetDataSize; +}; +typedef calculate_times_struct calculate_times_struct; + + +namespace cryptonote { class cryptonote_protocol_handler_base; } // a friend class // TODO friend not working + +/*** +@brief Access to simple throttles, with singlton to access global network limits +*/ +class network_throttle_manager { + // provides global (singleton) in/inreq/out throttle access + + // [[note1]] see also http://www.nuonsoft.com/blog/2012/10/21/implementing-a-thread-safe-singleton-with-c11/ + // [[note2]] _inreq is the requested in traffic - we anticipate we will get in-bound traffic soon as result of what we do (e.g. that we sent network downloads requests) + + //protected: + public: // XXX + // [[note1]] + static std::once_flag m_once_get_global_throttle_in; + static std::once_flag m_once_get_global_throttle_inreq; // [[note2]] + static std::once_flag m_once_get_global_throttle_out; + static std::unique_ptr<i_network_throttle> m_obj_get_global_throttle_in; + static std::unique_ptr<i_network_throttle> m_obj_get_global_throttle_inreq; + static std::unique_ptr<i_network_throttle> m_obj_get_global_throttle_out; + + static std::mutex m_lock_get_global_throttle_in; + static std::mutex m_lock_get_global_throttle_inreq; + static std::mutex m_lock_get_global_throttle_out; + + friend class cryptonote::cryptonote_protocol_handler_base; // FRIEND - to directly access global throttle-s. !! REMEMBER TO USE LOCKS! + friend class connection_basic; // FRIEND - to directly access global throttle-s. !! REMEMBER TO USE LOCKS! + friend class connection_basic_pimpl; // ditto + + static int xxx; + + public: + static i_network_throttle & get_global_throttle_in(); ///< singleton ; for friend class ; caller MUST use proper locks! like m_lock_get_global_throttle_in + static i_network_throttle & get_global_throttle_inreq(); ///< ditto ; use lock ... use m_lock_get_global_throttle_inreq obviously + static i_network_throttle & get_global_throttle_out(); ///< ditto ; use lock ... use m_lock_get_global_throttle_out obviously +}; + + + +/*** +@brief interface for the throttle, see the derivated class +*/ +class i_network_throttle { + public: + virtual void set_name(const std::string &name)=0; + virtual void set_target_speed( network_speed_kbps target )=0; + virtual void set_real_target_speed(network_speed_kbps real_target)=0; + virtual network_speed_kbps get_terget_speed()=0; + + virtual void handle_trafic_exact(size_t packet_size) =0; // count the new traffic/packet; the size is exact considering all network costs + virtual void handle_trafic_tcp(size_t packet_size) =0; // count the new traffic/packet; the size is as TCP, we will consider MTU etc + virtual void tick() =0; // poke and update timers/history + + // time calculations: + + virtual void calculate_times(size_t packet_size, calculate_times_struct &cts, bool dbg, double force_window) const =0; // assuming sending new package (or 0), calculate: + // Average, Window, Delay, Recommended data size ; also gets dbg=debug flag, and forced widnow size if >0 or -1 for not forcing window size + + // Average speed, Window size, recommended Delay to sleep now, Recommended size of data to send now + + virtual network_time_seconds get_sleep_time(size_t packet_size) const =0; // gets the D (recommended Delay time) from calc + virtual network_time_seconds get_sleep_time_after_tick(size_t packet_size) =0; // ditto, but first tick the timer + + virtual size_t get_recommended_size_of_planned_transport() const =0; // what should be the recommended limit of data size that we can transport over current network_throttle in near future + + virtual double get_time_seconds() const =0; // a timer + virtual void logger_handle_net(const std::string &filename, double time, size_t size)=0; + + +}; + + +// ... more in the -advanced.h file + + +} // namespace net_utils +} // namespace epee + + +#endif + + + diff --git a/src/p2p/p2p_protocol_defs.h b/src/p2p/p2p_protocol_defs.h index 44b8c5b50..e157f3188 100644 --- a/src/p2p/p2p_protocol_defs.h +++ b/src/p2p/p2p_protocol_defs.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/p2p/stdafx.h b/src/p2p/stdafx.h index ede53d517..da1216924 100644 --- a/src/p2p/stdafx.h +++ b/src/p2p/stdafx.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/platform/mingw/alloca.h b/src/platform/mingw/alloca.h index 32589db06..868db1af4 100644 --- a/src/platform/mingw/alloca.h +++ b/src/platform/mingw/alloca.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/platform/msc/alloca.h b/src/platform/msc/alloca.h index 82186553b..c212d64b9 100644 --- a/src/platform/msc/alloca.h +++ b/src/platform/msc/alloca.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/platform/msc/inline_c.h b/src/platform/msc/inline_c.h index a016b783e..fa4fd48f3 100644 --- a/src/platform/msc/inline_c.h +++ b/src/platform/msc/inline_c.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/platform/msc/stdbool.h b/src/platform/msc/stdbool.h index 1044d2539..e3c932ccf 100644 --- a/src/platform/msc/stdbool.h +++ b/src/platform/msc/stdbool.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/platform/msc/sys/param.h b/src/platform/msc/sys/param.h index 07a036a6e..0db360ab0 100644 --- a/src/platform/msc/sys/param.h +++ b/src/platform/msc/sys/param.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt index 5417a0ec1..cb8a8426c 100644 --- a/src/rpc/CMakeLists.txt +++ b/src/rpc/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # @@ -45,6 +45,7 @@ bitmonero_add_library(rpc target_link_libraries(rpc LINK_PRIVATE cryptonote_core + cryptonote_protocol ${Boost_CHRONO_LIBRARY} ${Boost_REGEX_LIBRARY} ${Boost_SYSTEM_LIBRARY} diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 036cb64ff..561161950 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -40,29 +40,10 @@ using namespace epee; #include "misc_language.h" #include "crypto/hash.h" #include "core_rpc_server_error_codes.h" +#include "daemon/command_line_args.h" namespace cryptonote { - namespace - { - const command_line::arg_descriptor<std::string> arg_rpc_bind_ip = { - "rpc-bind-ip" - , "IP for RPC server" - , "127.0.0.1" - }; - - const command_line::arg_descriptor<std::string> arg_rpc_bind_port = { - "rpc-bind-port" - , "Port for RPC server" - , std::to_string(config::RPC_DEFAULT_PORT) - }; - - const command_line::arg_descriptor<std::string> arg_testnet_rpc_bind_port = { - "testnet-rpc-bind-port" - , "Port for testnet RPC server" - , std::to_string(config::testnet::RPC_DEFAULT_PORT) - }; - } //----------------------------------------------------------------------------------- void core_rpc_server::init_options(boost::program_options::options_description& desc) @@ -75,11 +56,9 @@ namespace cryptonote core_rpc_server::core_rpc_server( core& cr , nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<cryptonote::core> >& p2p - , bool testnet ) : m_core(cr) , m_p2p(p2p) - , m_testnet(testnet) {} //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::handle_command_line( @@ -97,6 +76,8 @@ namespace cryptonote const boost::program_options::variables_map& vm ) { + m_testnet = command_line::get_arg(vm, daemon_args::arg_testnet_on); + m_net_server.set_threads_prefix("RPC"); bool r = handle_command_line(vm); CHECK_AND_ASSERT_MES(r, false, "Failed to process command line in core_rpc_server"); @@ -124,7 +105,7 @@ namespace cryptonote #define CHECK_CORE_READY() do { if(!check_core_ready()){res.status = CORE_RPC_STATUS_BUSY;return true;} } while(0) //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_height(const COMMAND_RPC_GET_HEIGHT::request& req, COMMAND_RPC_GET_HEIGHT::response& res, connection_context& cntx) + bool core_rpc_server::on_get_height(const COMMAND_RPC_GET_HEIGHT::request& req, COMMAND_RPC_GET_HEIGHT::response& res) { CHECK_CORE_BUSY(); res.height = m_core.get_current_blockchain_height(); @@ -132,7 +113,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_info(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res, connection_context& cntx) + bool core_rpc_server::on_get_info(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res) { CHECK_CORE_BUSY(); res.height = m_core.get_current_blockchain_height(); @@ -150,7 +131,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_blocks(const COMMAND_RPC_GET_BLOCKS_FAST::request& req, COMMAND_RPC_GET_BLOCKS_FAST::response& res, connection_context& cntx) + bool core_rpc_server::on_get_blocks(const COMMAND_RPC_GET_BLOCKS_FAST::request& req, COMMAND_RPC_GET_BLOCKS_FAST::response& res) { CHECK_CORE_BUSY(); std::list<std::pair<block, std::list<transaction> > > bs; @@ -175,7 +156,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_random_outs(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res, connection_context& cntx) + bool core_rpc_server::on_get_random_outs(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) { CHECK_CORE_BUSY(); res.status = "Failed"; @@ -204,7 +185,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_indexes(const COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request& req, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response& res, connection_context& cntx) + bool core_rpc_server::on_get_indexes(const COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request& req, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response& res) { CHECK_CORE_BUSY(); bool r = m_core.get_tx_outputs_gindexs(req.txid, res.o_indexes); @@ -218,7 +199,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_transactions(const COMMAND_RPC_GET_TRANSACTIONS::request& req, COMMAND_RPC_GET_TRANSACTIONS::response& res, connection_context& cntx) + bool core_rpc_server::on_get_transactions(const COMMAND_RPC_GET_TRANSACTIONS::request& req, COMMAND_RPC_GET_TRANSACTIONS::response& res) { CHECK_CORE_BUSY(); std::vector<crypto::hash> vh; @@ -260,7 +241,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_send_raw_tx(const COMMAND_RPC_SEND_RAW_TX::request& req, COMMAND_RPC_SEND_RAW_TX::response& res, connection_context& cntx) + bool core_rpc_server::on_send_raw_tx(const COMMAND_RPC_SEND_RAW_TX::request& req, COMMAND_RPC_SEND_RAW_TX::response& res) { CHECK_CORE_READY(); @@ -303,7 +284,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_start_mining(const COMMAND_RPC_START_MINING::request& req, COMMAND_RPC_START_MINING::response& res, connection_context& cntx) + bool core_rpc_server::on_start_mining(const COMMAND_RPC_START_MINING::request& req, COMMAND_RPC_START_MINING::response& res) { CHECK_CORE_READY(); account_public_address adr; @@ -325,7 +306,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_stop_mining(const COMMAND_RPC_STOP_MINING::request& req, COMMAND_RPC_STOP_MINING::response& res, connection_context& cntx) + bool core_rpc_server::on_stop_mining(const COMMAND_RPC_STOP_MINING::request& req, COMMAND_RPC_STOP_MINING::response& res) { if(!m_core.get_miner().stop()) { @@ -336,7 +317,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_mining_status(const COMMAND_RPC_MINING_STATUS::request& req, COMMAND_RPC_MINING_STATUS::response& res, connection_context& cntx) + bool core_rpc_server::on_mining_status(const COMMAND_RPC_MINING_STATUS::request& req, COMMAND_RPC_MINING_STATUS::response& res) { CHECK_CORE_READY(); @@ -354,7 +335,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_save_bc(const COMMAND_RPC_SAVE_BC::request& req, COMMAND_RPC_SAVE_BC::response& res, connection_context& cntx) + bool core_rpc_server::on_save_bc(const COMMAND_RPC_SAVE_BC::request& req, COMMAND_RPC_SAVE_BC::response& res) { CHECK_CORE_BUSY(); if( !m_core.get_blockchain_storage().store_blockchain() ) @@ -366,7 +347,76 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_getblockcount(const COMMAND_RPC_GETBLOCKCOUNT::request& req, COMMAND_RPC_GETBLOCKCOUNT::response& res, connection_context& cntx) + bool core_rpc_server::on_get_peer_list(const COMMAND_RPC_GET_PEER_LIST::request& req, COMMAND_RPC_GET_PEER_LIST::response& res) + { + /* + std::list<nodetool::peerlist_entry> white_list; + std::list<nodetool::peerlist_entry> gray_list; + m_p2p.get_peerlist(white_list, gray_list); + + for (auto & entry : white_list) + { + res.white_list.emplace_back(entry.id, entry.adr.ip, entry.adr.port, entry.last_seen); + } + + for (auto & entry : gray_list) + { + res.gray_list.emplace_back(entry.id, entry.adr.ip, entry.adr.port, entry.last_seen); + } + + */ + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_set_log_hash_rate(const COMMAND_RPC_SET_LOG_HASH_RATE::request& req, COMMAND_RPC_SET_LOG_HASH_RATE::response& res) + { + if(m_core.get_miner().is_mining()) + { + m_core.get_miner().do_print_hashrate(req.visible); + res.status = CORE_RPC_STATUS_OK; + } + else + { + res.status = CORE_RPC_STATUS_NOT_MINING; + } + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_set_log_level(const COMMAND_RPC_SET_LOG_LEVEL::request& req, COMMAND_RPC_SET_LOG_LEVEL::response& res) + { + if (req.level < LOG_LEVEL_MIN || req.level > LOG_LEVEL_MAX) + { + res.status = "Error: log level not valid"; + } + else + { + epee::log_space::log_singletone::get_set_log_detalisation_level(true, req.level); + int otshell_utils_log_level = 100 - (req.level * 20); + gCurrentLogger.setDebugLevel(otshell_utils_log_level); + res.status = CORE_RPC_STATUS_OK; + } + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_get_transaction_pool(const COMMAND_RPC_GET_TRANSACTION_POOL::request& req, COMMAND_RPC_GET_TRANSACTION_POOL::response& res) + { + CHECK_CORE_BUSY(); + m_core.get_pool_transactions_and_spent_keys_info(res.transactions, res.spent_key_images); + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_stop_daemon(const COMMAND_RPC_STOP_DAEMON::request& req, COMMAND_RPC_STOP_DAEMON::response& res) + { + // FIXME: replace back to original m_p2p.send_stop_signal() after + // investigating why that isn't working quite right. + m_p2p.send_stop_signal(); + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_getblockcount(const COMMAND_RPC_GETBLOCKCOUNT::request& req, COMMAND_RPC_GETBLOCKCOUNT::response& res) { CHECK_CORE_BUSY(); res.count = m_core.get_current_blockchain_height(); @@ -374,7 +424,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_getblockhash(const COMMAND_RPC_GETBLOCKHASH::request& req, COMMAND_RPC_GETBLOCKHASH::response& res, epee::json_rpc::error& error_resp, connection_context& cntx) + bool core_rpc_server::on_getblockhash(const COMMAND_RPC_GETBLOCKHASH::request& req, COMMAND_RPC_GETBLOCKHASH::response& res, epee::json_rpc::error& error_resp) { if(!check_core_busy()) { @@ -415,7 +465,7 @@ namespace cryptonote return 0; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_getblocktemplate(const COMMAND_RPC_GETBLOCKTEMPLATE::request& req, COMMAND_RPC_GETBLOCKTEMPLATE::response& res, epee::json_rpc::error& error_resp, connection_context& cntx) + bool core_rpc_server::on_getblocktemplate(const COMMAND_RPC_GETBLOCKTEMPLATE::request& req, COMMAND_RPC_GETBLOCKTEMPLATE::response& res, epee::json_rpc::error& error_resp) { if(!check_core_ready()) { @@ -475,12 +525,13 @@ namespace cryptonote LOG_ERROR("Failed to calculate offset for "); return false; } + res.prev_hash = string_tools::pod_to_hex(b.prev_id); res.blocktemplate_blob = string_tools::buff_to_hex_nodelimer(block_blob); res.status = CORE_RPC_STATUS_OK; return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_submitblock(const COMMAND_RPC_SUBMITBLOCK::request& req, COMMAND_RPC_SUBMITBLOCK::response& res, epee::json_rpc::error& error_resp, connection_context& cntx) + bool core_rpc_server::on_submitblock(const COMMAND_RPC_SUBMITBLOCK::request& req, COMMAND_RPC_SUBMITBLOCK::response& res, epee::json_rpc::error& error_resp) { CHECK_CORE_READY(); if(req.size()!=1) @@ -552,7 +603,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_last_block_header(const COMMAND_RPC_GET_LAST_BLOCK_HEADER::request& req, COMMAND_RPC_GET_LAST_BLOCK_HEADER::response& res, epee::json_rpc::error& error_resp, connection_context& cntx) + bool core_rpc_server::on_get_last_block_header(const COMMAND_RPC_GET_LAST_BLOCK_HEADER::request& req, COMMAND_RPC_GET_LAST_BLOCK_HEADER::response& res, epee::json_rpc::error& error_resp) { if(!check_core_busy()) { @@ -588,7 +639,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_block_header_by_hash(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::response& res, epee::json_rpc::error& error_resp, connection_context& cntx){ + bool core_rpc_server::on_get_block_header_by_hash(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::response& res, epee::json_rpc::error& error_resp){ if(!check_core_busy()) { error_resp.code = CORE_RPC_ERROR_CODE_CORE_BUSY; @@ -629,7 +680,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_block_header_by_height(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::response& res, epee::json_rpc::error& error_resp, connection_context& cntx){ + bool core_rpc_server::on_get_block_header_by_height(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::response& res, epee::json_rpc::error& error_resp){ if(!check_core_busy()) { error_resp.code = CORE_RPC_ERROR_CODE_CORE_BUSY; @@ -662,7 +713,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_connections(const COMMAND_RPC_GET_CONNECTIONS::request& req, COMMAND_RPC_GET_CONNECTIONS::response& res, epee::json_rpc::error& error_resp, connection_context& cntx) + bool core_rpc_server::on_get_connections(const COMMAND_RPC_GET_CONNECTIONS::request& req, COMMAND_RPC_GET_CONNECTIONS::response& res, epee::json_rpc::error& error_resp) { if(!check_core_busy()) { @@ -678,7 +729,7 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_info_json(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res, epee::json_rpc::error& error_resp, connection_context& cntx) + bool core_rpc_server::on_get_info_json(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res, epee::json_rpc::error& error_resp) { if(!check_core_busy()) { @@ -702,4 +753,62 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ -} + bool core_rpc_server::on_fast_exit(const COMMAND_RPC_FAST_EXIT::request& req, COMMAND_RPC_FAST_EXIT::response& res) + { + cryptonote::core::set_fast_exit(); + m_p2p.deinit(); + m_core.deinit(); + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_out_peers(const COMMAND_RPC_OUT_PEERS::request& req, COMMAND_RPC_OUT_PEERS::response& res) + { + // TODO + /*if (m_p2p.get_outgoing_connections_count() > req.out_peers) + { + m_p2p.m_config.m_net_config.connections_count = req.out_peers; + if (m_p2p.get_outgoing_connections_count() > req.out_peers) + { + int count = m_p2p.get_outgoing_connections_count() - req.out_peers; + m_p2p.delete_connections(count); + } + } + + else + m_p2p.m_config.m_net_config.connections_count = req.out_peers; + */ + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_start_save_graph(const COMMAND_RPC_START_SAVE_GRAPH::request& req, COMMAND_RPC_START_SAVE_GRAPH::response& res) + { + m_p2p.set_save_graph(true); + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_stop_save_graph(const COMMAND_RPC_STOP_SAVE_GRAPH::request& req, COMMAND_RPC_STOP_SAVE_GRAPH::response& res) + { + m_p2p.set_save_graph(false); + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + + const command_line::arg_descriptor<std::string> core_rpc_server::arg_rpc_bind_ip = { + "rpc-bind-ip" + , "IP for RPC server" + , "127.0.0.1" + }; + + const command_line::arg_descriptor<std::string> core_rpc_server::arg_rpc_bind_port = { + "rpc-bind-port" + , "Port for RPC server" + , std::to_string(config::RPC_DEFAULT_PORT) + }; + + const command_line::arg_descriptor<std::string> core_rpc_server::arg_testnet_rpc_bind_port = { + "testnet-rpc-bind-port" + , "Port for testnet RPC server" + , std::to_string(config::testnet::RPC_DEFAULT_PORT) + }; + +} // namespace cryptonote diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 02bc533c9..cee8df25d 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -39,6 +39,10 @@ #include "p2p/net_node.h" #include "cryptonote_protocol/cryptonote_protocol_handler.h" +// yes, epee doesn't properly use its full namespace when calling its +// functions from macros. *sigh* +using namespace epee; + namespace cryptonote { /************************************************************************/ @@ -47,19 +51,22 @@ namespace cryptonote class core_rpc_server: public epee::http_server_impl_base<core_rpc_server> { public: + + static const command_line::arg_descriptor<std::string> arg_rpc_bind_ip; + static const command_line::arg_descriptor<std::string> arg_rpc_bind_port; + static const command_line::arg_descriptor<std::string> arg_testnet_rpc_bind_port; + typedef epee::net_utils::connection_context_base connection_context; core_rpc_server( core& cr , nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<cryptonote::core> >& p2p - , bool testnet ); static void init_options(boost::program_options::options_description& desc); bool init( const boost::program_options::variables_map& vm ); - private: CHAIN_HTTP_TO_MAP2(connection_context); //forward http requests to uri map @@ -74,7 +81,16 @@ namespace cryptonote MAP_URI_AUTO_JON2("/stop_mining", on_stop_mining, COMMAND_RPC_STOP_MINING) MAP_URI_AUTO_JON2("/mining_status", on_mining_status, COMMAND_RPC_MINING_STATUS) MAP_URI_AUTO_JON2("/save_bc", on_save_bc, COMMAND_RPC_SAVE_BC) + MAP_URI_AUTO_JON2("/get_peer_list", on_get_peer_list, COMMAND_RPC_GET_PEER_LIST) + MAP_URI_AUTO_JON2("/set_log_hash_rate", on_set_log_hash_rate, COMMAND_RPC_SET_LOG_HASH_RATE) + MAP_URI_AUTO_JON2("/set_log_level", on_set_log_level, COMMAND_RPC_SET_LOG_LEVEL) + MAP_URI_AUTO_JON2("/get_transaction_pool", on_get_transaction_pool, COMMAND_RPC_GET_TRANSACTION_POOL) + MAP_URI_AUTO_JON2("/stop_daemon", on_stop_daemon, COMMAND_RPC_STOP_DAEMON) MAP_URI_AUTO_JON2("/getinfo", on_get_info, COMMAND_RPC_GET_INFO) + MAP_URI_AUTO_JON2("/fast_exit", on_fast_exit, COMMAND_RPC_FAST_EXIT) + MAP_URI_AUTO_JON2("/out_peers", on_out_peers, COMMAND_RPC_OUT_PEERS) + MAP_URI_AUTO_JON2("/start_save_graph", on_start_save_graph, COMMAND_RPC_START_SAVE_GRAPH) + MAP_URI_AUTO_JON2("/stop_save_graph", on_stop_save_graph, COMMAND_RPC_STOP_SAVE_GRAPH) BEGIN_JSON_RPC_MAP("/json_rpc") MAP_JON_RPC("getblockcount", on_getblockcount, COMMAND_RPC_GETBLOCKCOUNT) MAP_JON_RPC_WE("on_getblockhash", on_getblockhash, COMMAND_RPC_GETBLOCKHASH) @@ -88,29 +104,41 @@ namespace cryptonote END_JSON_RPC_MAP() END_URI_MAP2() - bool on_get_height(const COMMAND_RPC_GET_HEIGHT::request& req, COMMAND_RPC_GET_HEIGHT::response& res, connection_context& cntx); - bool on_get_blocks(const COMMAND_RPC_GET_BLOCKS_FAST::request& req, COMMAND_RPC_GET_BLOCKS_FAST::response& res, connection_context& cntx); - bool on_get_transactions(const COMMAND_RPC_GET_TRANSACTIONS::request& req, COMMAND_RPC_GET_TRANSACTIONS::response& res, connection_context& cntx); - bool on_get_indexes(const COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request& req, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response& res, connection_context& cntx); - bool on_send_raw_tx(const COMMAND_RPC_SEND_RAW_TX::request& req, COMMAND_RPC_SEND_RAW_TX::response& res, connection_context& cntx); - bool on_start_mining(const COMMAND_RPC_START_MINING::request& req, COMMAND_RPC_START_MINING::response& res, connection_context& cntx); - bool on_stop_mining(const COMMAND_RPC_STOP_MINING::request& req, COMMAND_RPC_STOP_MINING::response& res, connection_context& cntx); - bool on_mining_status(const COMMAND_RPC_MINING_STATUS::request& req, COMMAND_RPC_MINING_STATUS::response& res, connection_context& cntx); - bool on_get_random_outs(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res, connection_context& cntx); - bool on_get_info(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res, connection_context& cntx); - bool on_save_bc(const COMMAND_RPC_SAVE_BC::request& req, COMMAND_RPC_SAVE_BC::response& res, connection_context& cntx); + bool on_get_height(const COMMAND_RPC_GET_HEIGHT::request& req, COMMAND_RPC_GET_HEIGHT::response& res); + bool on_get_blocks(const COMMAND_RPC_GET_BLOCKS_FAST::request& req, COMMAND_RPC_GET_BLOCKS_FAST::response& res); + bool on_get_transactions(const COMMAND_RPC_GET_TRANSACTIONS::request& req, COMMAND_RPC_GET_TRANSACTIONS::response& res); + bool on_get_indexes(const COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request& req, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response& res); + bool on_send_raw_tx(const COMMAND_RPC_SEND_RAW_TX::request& req, COMMAND_RPC_SEND_RAW_TX::response& res); + bool on_start_mining(const COMMAND_RPC_START_MINING::request& req, COMMAND_RPC_START_MINING::response& res); + bool on_stop_mining(const COMMAND_RPC_STOP_MINING::request& req, COMMAND_RPC_STOP_MINING::response& res); + bool on_mining_status(const COMMAND_RPC_MINING_STATUS::request& req, COMMAND_RPC_MINING_STATUS::response& res); + bool on_get_random_outs(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res); + bool on_get_info(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res); + bool on_save_bc(const COMMAND_RPC_SAVE_BC::request& req, COMMAND_RPC_SAVE_BC::response& res); + bool on_get_peer_list(const COMMAND_RPC_GET_PEER_LIST::request& req, COMMAND_RPC_GET_PEER_LIST::response& res); + bool on_set_log_hash_rate(const COMMAND_RPC_SET_LOG_HASH_RATE::request& req, COMMAND_RPC_SET_LOG_HASH_RATE::response& res); + bool on_set_log_level(const COMMAND_RPC_SET_LOG_LEVEL::request& req, COMMAND_RPC_SET_LOG_LEVEL::response& res); + bool on_get_transaction_pool(const COMMAND_RPC_GET_TRANSACTION_POOL::request& req, COMMAND_RPC_GET_TRANSACTION_POOL::response& res); + bool on_stop_daemon(const COMMAND_RPC_STOP_DAEMON::request& req, COMMAND_RPC_STOP_DAEMON::response& res); + bool on_fast_exit(const COMMAND_RPC_FAST_EXIT::request& req, COMMAND_RPC_FAST_EXIT::response& res); + bool on_out_peers(const COMMAND_RPC_OUT_PEERS::request& req, COMMAND_RPC_OUT_PEERS::response& res); + bool on_start_save_graph(const COMMAND_RPC_START_SAVE_GRAPH::request& req, COMMAND_RPC_START_SAVE_GRAPH::response& res); + bool on_stop_save_graph(const COMMAND_RPC_STOP_SAVE_GRAPH::request& req, COMMAND_RPC_STOP_SAVE_GRAPH::response& res); //json_rpc - bool on_getblockcount(const COMMAND_RPC_GETBLOCKCOUNT::request& req, COMMAND_RPC_GETBLOCKCOUNT::response& res, connection_context& cntx); - bool on_getblockhash(const COMMAND_RPC_GETBLOCKHASH::request& req, COMMAND_RPC_GETBLOCKHASH::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); - bool on_getblocktemplate(const COMMAND_RPC_GETBLOCKTEMPLATE::request& req, COMMAND_RPC_GETBLOCKTEMPLATE::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); - bool on_submitblock(const COMMAND_RPC_SUBMITBLOCK::request& req, COMMAND_RPC_SUBMITBLOCK::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); - bool on_get_last_block_header(const COMMAND_RPC_GET_LAST_BLOCK_HEADER::request& req, COMMAND_RPC_GET_LAST_BLOCK_HEADER::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); - bool on_get_block_header_by_hash(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); - bool on_get_block_header_by_height(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); - bool on_get_connections(const COMMAND_RPC_GET_CONNECTIONS::request& req, COMMAND_RPC_GET_CONNECTIONS::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); - bool on_get_info_json(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res, epee::json_rpc::error& error_resp, connection_context& cntx); + bool on_getblockcount(const COMMAND_RPC_GETBLOCKCOUNT::request& req, COMMAND_RPC_GETBLOCKCOUNT::response& res); + bool on_getblockhash(const COMMAND_RPC_GETBLOCKHASH::request& req, COMMAND_RPC_GETBLOCKHASH::response& res, epee::json_rpc::error& error_resp); + bool on_getblocktemplate(const COMMAND_RPC_GETBLOCKTEMPLATE::request& req, COMMAND_RPC_GETBLOCKTEMPLATE::response& res, epee::json_rpc::error& error_resp); + bool on_submitblock(const COMMAND_RPC_SUBMITBLOCK::request& req, COMMAND_RPC_SUBMITBLOCK::response& res, epee::json_rpc::error& error_resp); + bool on_get_last_block_header(const COMMAND_RPC_GET_LAST_BLOCK_HEADER::request& req, COMMAND_RPC_GET_LAST_BLOCK_HEADER::response& res, epee::json_rpc::error& error_resp); + bool on_get_block_header_by_hash(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::response& res, epee::json_rpc::error& error_resp); + bool on_get_block_header_by_height(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::response& res, epee::json_rpc::error& error_resp); + bool on_get_connections(const COMMAND_RPC_GET_CONNECTIONS::request& req, COMMAND_RPC_GET_CONNECTIONS::response& res, epee::json_rpc::error& error_resp); + bool on_get_info_json(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res, epee::json_rpc::error& error_resp); //----------------------- + +private: + bool handle_command_line( const boost::program_options::variables_map& vm ); diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index d711e1732..b6a2edd0b 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -39,6 +39,7 @@ namespace cryptonote //----------------------------------------------- #define CORE_RPC_STATUS_OK "OK" #define CORE_RPC_STATUS_BUSY "BUSY" +#define CORE_RPC_STATUS_NOT_MINING "NOT MINING" struct COMMAND_RPC_GET_HEIGHT { @@ -383,6 +384,7 @@ namespace cryptonote uint64_t difficulty; uint64_t height; uint64_t reserved_offset; + std::string prev_hash; blobdata blocktemplate_blob; std::string status; @@ -390,6 +392,7 @@ namespace cryptonote KV_SERIALIZE(difficulty) KV_SERIALIZE(height) KV_SERIALIZE(reserved_offset) + KV_SERIALIZE(prev_hash) KV_SERIALIZE(blocktemplate_blob) KV_SERIALIZE(status) END_KV_SERIALIZE_MAP() @@ -438,7 +441,7 @@ namespace cryptonote KV_SERIALIZE(reward) END_KV_SERIALIZE_MAP() }; - + struct COMMAND_RPC_GET_LAST_BLOCK_HEADER { struct request @@ -508,6 +511,148 @@ namespace cryptonote }; + struct peer { + uint64_t id; + uint32_t ip; + uint16_t port; + uint64_t last_seen; + + peer() = default; + + peer(uint64_t id, uint32_t ip, uint16_t port, uint64_t last_seen) + : id(id), ip(ip), port(port), last_seen(last_seen) + {} + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(id) + KV_SERIALIZE(ip) + KV_SERIALIZE(port) + KV_SERIALIZE(last_seen) + END_KV_SERIALIZE_MAP() + }; + + struct COMMAND_RPC_GET_PEER_LIST + { + struct request + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + std::vector<peer> white_list; + std::vector<peer> gray_list; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + KV_SERIALIZE(white_list) + KV_SERIALIZE(gray_list) + END_KV_SERIALIZE_MAP() + }; + }; + + struct COMMAND_RPC_SET_LOG_HASH_RATE + { + struct request + { + bool visible; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(visible) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + END_KV_SERIALIZE_MAP() + }; + }; + + struct COMMAND_RPC_SET_LOG_LEVEL + { + struct request + { + int8_t level; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(level) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + END_KV_SERIALIZE_MAP() + }; + }; + + struct tx_info + { + std::string id_hash; + std::string tx_json; // TODO - expose this data directly + uint64_t blob_size; + uint64_t fee; + std::string max_used_block_id_hash; + uint64_t max_used_block_height; + bool kept_by_block; + uint64_t last_failed_height; + std::string last_failed_id_hash; + uint64_t receive_time; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(id_hash) + KV_SERIALIZE(tx_json) + KV_SERIALIZE(blob_size) + KV_SERIALIZE(fee) + KV_SERIALIZE(max_used_block_id_hash) + KV_SERIALIZE(max_used_block_height) + KV_SERIALIZE(kept_by_block) + KV_SERIALIZE(last_failed_height) + KV_SERIALIZE(last_failed_id_hash) + KV_SERIALIZE(receive_time) + END_KV_SERIALIZE_MAP() + }; + + struct spent_key_image_info + { + std::string id_hash; + std::vector<std::string> txs_hashes; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(id_hash) + KV_SERIALIZE(txs_hashes) + END_KV_SERIALIZE_MAP() + }; + + struct COMMAND_RPC_GET_TRANSACTION_POOL + { + struct request + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + std::vector<tx_info> transactions; + std::vector<spent_key_image_info> spent_key_images; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + KV_SERIALIZE(transactions) + KV_SERIALIZE(spent_key_images) + END_KV_SERIALIZE_MAP() + }; + }; + struct COMMAND_RPC_GET_CONNECTIONS { struct request @@ -528,5 +673,122 @@ namespace cryptonote }; }; + + struct COMMAND_RPC_GET_BLOCK_HEADERS_RANGE + { + struct request + { + uint64_t start_height; + uint64_t end_height; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(start_height) + KV_SERIALIZE(end_height) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + std::vector<block_header_responce> headers; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + KV_SERIALIZE(headers) + END_KV_SERIALIZE_MAP() + }; + }; + + struct COMMAND_RPC_STOP_DAEMON + { + struct request + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + END_KV_SERIALIZE_MAP() + }; + }; + + struct COMMAND_RPC_FAST_EXIT + { + struct request + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + END_KV_SERIALIZE_MAP() + }; + }; + + struct COMMAND_RPC_OUT_PEERS + { + struct request + { + uint64_t out_peers; + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(out_peers) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + END_KV_SERIALIZE_MAP() + }; + }; + + struct COMMAND_RPC_START_SAVE_GRAPH + { + struct request + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + END_KV_SERIALIZE_MAP() + }; + }; + + struct COMMAND_RPC_STOP_SAVE_GRAPH + { + struct request + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + END_KV_SERIALIZE_MAP() + }; + }; } diff --git a/src/rpc/core_rpc_server_error_codes.h b/src/rpc/core_rpc_server_error_codes.h index ab6ca9aef..91659eb2a 100644 --- a/src/rpc/core_rpc_server_error_codes.h +++ b/src/rpc/core_rpc_server_error_codes.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/serialization/binary_archive.h b/src/serialization/binary_archive.h index 56c94684e..5cd4988e4 100644 --- a/src/serialization/binary_archive.h +++ b/src/serialization/binary_archive.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -42,8 +42,8 @@ #include "warnings.h" /* I have no clue what these lines means */ -PUSH_WARNINGS; -DISABLE_VS_WARNINGS(4244); +PUSH_WARNINGS +DISABLE_VS_WARNINGS(4244) //TODO: fix size_t warning in x32 platform diff --git a/src/serialization/binary_utils.h b/src/serialization/binary_utils.h index f2cfadd4e..4fe782326 100644 --- a/src/serialization/binary_utils.h +++ b/src/serialization/binary_utils.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/serialization/crypto.h b/src/serialization/crypto.h index 8986d8081..f18e85b12 100644 --- a/src/serialization/crypto.h +++ b/src/serialization/crypto.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/serialization/debug_archive.h b/src/serialization/debug_archive.h index 127e57ed6..4d697c14c 100644 --- a/src/serialization/debug_archive.h +++ b/src/serialization/debug_archive.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/serialization/json_archive.h b/src/serialization/json_archive.h index bd8b16130..49bbf95b6 100644 --- a/src/serialization/json_archive.h +++ b/src/serialization/json_archive.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/serialization/json_utils.h b/src/serialization/json_utils.h index 64f9f946f..462d1f8e7 100644 --- a/src/serialization/json_utils.h +++ b/src/serialization/json_utils.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/serialization/serialization.h b/src/serialization/serialization.h index 9e534cbc6..c94c99336 100644 --- a/src/serialization/serialization.h +++ b/src/serialization/serialization.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/serialization/string.h b/src/serialization/string.h index 077f657d2..0f1b0d4e0 100644 --- a/src/serialization/string.h +++ b/src/serialization/string.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/serialization/variant.h b/src/serialization/variant.h index 9b5d54237..9efa9c346 100644 --- a/src/serialization/variant.h +++ b/src/serialization/variant.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/serialization/vector.h b/src/serialization/vector.h index a099ce070..3787f00e6 100644 --- a/src/serialization/vector.h +++ b/src/serialization/vector.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/simplewallet/CMakeLists.txt b/src/simplewallet/CMakeLists.txt index 14f877907..a33ed0f32 100644 --- a/src/simplewallet/CMakeLists.txt +++ b/src/simplewallet/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # @@ -50,6 +50,7 @@ target_link_libraries(simplewallet crypto common mnemonics + p2p ${UNBOUND_LIBRARY} ${UPNP_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} diff --git a/src/simplewallet/password_container.cpp b/src/simplewallet/password_container.cpp index 1c4aee453..e5fb933bb 100644 --- a/src/simplewallet/password_container.cpp +++ b/src/simplewallet/password_container.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/simplewallet/password_container.h b/src/simplewallet/password_container.h index b39fa1fe4..8a4191c7a 100644 --- a/src/simplewallet/password_container.h +++ b/src/simplewallet/password_container.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 4f7df2d3a..9ac80fa9e 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -67,6 +67,9 @@ namespace po = boost::program_options; #define EXTENDED_LOGS_FILE "wallet_details.log" +unsigned int epee::g_test_dbg_lock_sleep = 0; + +#define DEFAULT_MIX 3 namespace { @@ -81,6 +84,7 @@ namespace const command_line::arg_descriptor<int> arg_daemon_port = {"daemon-port", "Use daemon instance at port <arg> instead of 8081", 0}; const command_line::arg_descriptor<uint32_t> arg_log_level = {"set_log", "", 0, true}; const command_line::arg_descriptor<bool> arg_testnet = {"testnet", "Used to deploy test nets. The daemon must be launched with --testnet flag", false}; + const command_line::arg_descriptor<bool> arg_restricted = {"restricted-rpc", "Restricts RPC to view only commands", false}; const command_line::arg_descriptor< std::vector<std::string> > arg_command = {"command", ""}; @@ -231,6 +235,36 @@ bool simple_wallet::seed(const std::vector<std::string> &args/* = std::vector<st return true; } +bool simple_wallet::seed_set_language(const std::vector<std::string> &args/* = std::vector<std::string>()*/) +{ + bool success = false; + if (!m_wallet->is_deterministic()) + { + fail_msg_writer() << "This wallet is non-deterministic and doesn't have a seed."; + return true; + } + tools::password_container pwd_container; + success = pwd_container.read_password(); + if (!success) + { + fail_msg_writer() << "failed to read wallet password"; + return true; + } + + /* verify password before using so user doesn't accidentally set a new password for rewritten wallet */ + success = m_wallet->verify_password(pwd_container.password()); + if (!success) + { + fail_msg_writer() << "invalid password"; + return true; + } + + std::string mnemonic_language = get_mnemonic_language(); + m_wallet->set_seed_language(mnemonic_language); + m_wallet->rewrite(m_wallet_file, pwd_container.password()); + return true; +} + bool simple_wallet::help(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { success_msg_writer() << get_commands_str(); @@ -249,15 +283,45 @@ simple_wallet::simple_wallet() m_cmd_binder.set_handler("incoming_transfers", boost::bind(&simple_wallet::show_incoming_transfers, this, _1), "incoming_transfers [available|unavailable] - Show incoming transfers - all of them or filter them by availability"); m_cmd_binder.set_handler("payments", boost::bind(&simple_wallet::show_payments, this, _1), "payments <payment_id_1> [<payment_id_2> ... <payment_id_N>] - Show payments <payment_id_1>, ... <payment_id_N>"); m_cmd_binder.set_handler("bc_height", boost::bind(&simple_wallet::show_blockchain_height, this, _1), "Show blockchain height"); - m_cmd_binder.set_handler("transfer", boost::bind(&simple_wallet::transfer, this, _1), "transfer <mixin_count> <addr_1> <amount_1> [<addr_2> <amount_2> ... <addr_N> <amount_N>] [payment_id] - Transfer <amount_1>,... <amount_N> to <address_1>,... <address_N>, respectively. <mixin_count> is the number of transactions yours is indistinguishable from (from 0 to maximum available)"); + m_cmd_binder.set_handler("transfer", boost::bind(&simple_wallet::transfer, this, _1), "transfer [<mixin_count>] <addr_1> <amount_1> [<addr_2> <amount_2> ... <addr_N> <amount_N>] [payment_id] - Transfer <amount_1>,... <amount_N> to <address_1>,... <address_N>, respectively. <mixin_count> is the number of transactions yours is indistinguishable from (from 0 to maximum available)"); m_cmd_binder.set_handler("set_log", boost::bind(&simple_wallet::set_log, this, _1), "set_log <level> - Change current log detalization level, <level> is a number 0-4"); m_cmd_binder.set_handler("address", boost::bind(&simple_wallet::print_address, this, _1), "Show current wallet public address"); m_cmd_binder.set_handler("save", boost::bind(&simple_wallet::save, this, _1), "Save wallet synchronized data"); m_cmd_binder.set_handler("viewkey", boost::bind(&simple_wallet::viewkey, this, _1), "Get viewkey"); m_cmd_binder.set_handler("seed", boost::bind(&simple_wallet::seed, this, _1), "Get deterministic seed"); + m_cmd_binder.set_handler("set", boost::bind(&simple_wallet::set_variable, this, _1), "available options: seed language - Set wallet seed langage"); m_cmd_binder.set_handler("help", boost::bind(&simple_wallet::help, this, _1), "Show this help"); } //---------------------------------------------------------------------------------------------------- +bool simple_wallet::set_variable(const std::vector<std::string> &args) +{ + if (args.empty()) + { + fail_msg_writer() << "set: needs an argument. available options: seed"; + return true; + } + else + { + if (args[0] == "seed") + { + if (args.size() == 1) + { + fail_msg_writer() << "set seed: needs an argument. available options: language"; + return true; + } + else if (args[1] == "language") + { + std::vector<std::string> local_args = args; + local_args.erase(local_args.begin(), local_args.begin()+2); + seed_set_language(local_args); + return true; + } + } + } + fail_msg_writer() << "set: unrecognized argument(s)"; + return true; +} +//---------------------------------------------------------------------------------------------------- bool simple_wallet::set_log(const std::vector<std::string> &args) { if(args.size() != 1) @@ -285,14 +349,28 @@ bool simple_wallet::ask_wallet_create_if_needed() { std::string wallet_path; - wallet_path = command_line::input_line( - "Specify wallet file name (e.g., wallet.bin). If the wallet doesn't exist, it will be created.\n" - "Wallet file name: " - ); + bool valid_path = false; + do { + wallet_path = command_line::input_line( + "Specify wallet file name (e.g., wallet.bin). If the wallet doesn't exist, it will be created.\n" + "Wallet file name: " + ); + valid_path = tools::wallet2::wallet_valid_path_format(wallet_path); + if (!valid_path) + { + fail_msg_writer() << "wallet file path not valid: " << wallet_path; + } + } + while (!valid_path); bool keys_file_exists; bool wallet_file_exists; tools::wallet2::wallet_exists(wallet_path, keys_file_exists, wallet_file_exists); + LOG_PRINT_L3("wallet_path: " << wallet_path << ""); + LOG_PRINT_L3("keys_file_exists: " << std::boolalpha << keys_file_exists << std::noboolalpha + << " wallet_file_exists: " << std::boolalpha << wallet_file_exists << std::noboolalpha); + + LOG_PRINT_L1("Loading wallet..."); // add logic to error out if new wallet requested but named wallet file exists if (keys_file_exists || wallet_file_exists) @@ -575,6 +653,12 @@ bool simple_wallet::new_wallet(const std::string &wallet_file, const std::string //---------------------------------------------------------------------------------------------------- bool simple_wallet::open_wallet(const string &wallet_file, const std::string& password, bool testnet) { + if (!tools::wallet2::wallet_valid_path_format(wallet_file)) + { + fail_msg_writer() << "wallet file path not valid: " << wallet_file; + return false; + } + m_wallet_file = wallet_file; m_wallet.reset(new tools::wallet2(testnet)); m_wallet->callback(this); @@ -986,19 +1070,24 @@ bool simple_wallet::transfer(const std::vector<std::string> &args_) return true; std::vector<std::string> local_args = args_; - if(local_args.size() < 3) - { - fail_msg_writer() << "wrong number of arguments, expected at least 3, got " << local_args.size(); - return true; - } size_t fake_outs_count; - if(!epee::string_tools::get_xtype_from_string(fake_outs_count, local_args[0])) + if(local_args.size() > 0) { + if(!epee::string_tools::get_xtype_from_string(fake_outs_count, local_args[0])) + { + fake_outs_count = DEFAULT_MIX; + } + else + { + local_args.erase(local_args.begin()); + } + } + + if(local_args.size() < 2) { - fail_msg_writer() << "mixin_count should be non-negative integer, got " << local_args[0]; - return true; + fail_msg_writer() << "wrong number of arguments"; + return true; } - local_args.erase(local_args.begin()); std::vector<uint8_t> extra; if (1 == local_args.size() % 2) @@ -1256,6 +1345,7 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_params, arg_non_deterministic ); command_line::add_arg(desc_params, arg_electrum_seed ); command_line::add_arg(desc_params, arg_testnet); + command_line::add_arg(desc_params, arg_restricted); tools::wallet_rpc_server::init_options(desc_params); po::positional_options_description positional_options; @@ -1326,6 +1416,7 @@ int main(int argc, char* argv[]) } bool testnet = command_line::get_arg(vm, arg_testnet); + bool restricted = command_line::get_arg(vm, arg_restricted); std::string wallet_file = command_line::get_arg(vm, arg_wallet_file); std::string wallet_password = command_line::get_arg(vm, arg_password); std::string daemon_address = command_line::get_arg(vm, arg_daemon_address); @@ -1338,7 +1429,7 @@ int main(int argc, char* argv[]) if (daemon_address.empty()) daemon_address = std::string("http://") + daemon_host + ":" + std::to_string(daemon_port); - tools::wallet2 wal(testnet); + tools::wallet2 wal(testnet,restricted); try { LOG_PRINT_L0("Loading wallet..."); diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index ebc85ed17..a8fe78414 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -81,6 +81,17 @@ namespace cryptonote bool viewkey(const std::vector<std::string> &args = std::vector<std::string>()); bool seed(const std::vector<std::string> &args = std::vector<std::string>()); + + /*! + * \brief Sets seed language. + * + * interactive + * - prompts for password so wallet can be rewritten + * - calls get_mnemonic_language() which prompts for language + * + * \return success status + */ + bool seed_set_language(const std::vector<std::string> &args = std::vector<std::string>()); bool help(const std::vector<std::string> &args = std::vector<std::string>()); bool start_mining(const std::vector<std::string> &args); bool stop_mining(const std::vector<std::string> &args); @@ -96,6 +107,7 @@ namespace cryptonote ); bool print_address(const std::vector<std::string> &args = std::vector<std::string>()); bool save(const std::vector<std::string> &args); + bool set_variable(const std::vector<std::string> &args); bool set_log(const std::vector<std::string> &args); uint64_t get_daemon_blockchain_height(std::string& err); diff --git a/src/version.cmake b/src/version.cmake index 14ce8843d..ef4eb1fc9 100644 --- a/src/version.cmake +++ b/src/version.cmake @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # @@ -80,4 +80,4 @@ else() endif() configure_file("src/version.h.in" "${TO}") -endif()
\ No newline at end of file +endif() diff --git a/src/version.h.in b/src/version.h.in index a09131cdb..6dd62faf9 100644 --- a/src/version.h.in +++ b/src/version.h.in @@ -1,3 +1,3 @@ #define MONERO_VERSION_TAG "@VERSIONTAG@" -#define MONERO_VERSION "0.8.8.6" +#define MONERO_VERSION "0.8.8.7" #define MONERO_VERSION_FULL MONERO_VERSION "-" MONERO_VERSION_TAG diff --git a/src/wallet/CMakeLists.txt b/src/wallet/CMakeLists.txt index af3ec0fb8..6cf496f0d 100644 --- a/src/wallet/CMakeLists.txt +++ b/src/wallet/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 163e19df4..6698e7296 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -227,24 +227,25 @@ void wallet2::process_new_transaction(const cryptonote::transaction& tx, uint64_ } tx_extra_nonce extra_nonce; + crypto::hash payment_id = null_hash; if (find_tx_extra_field_by_type(tx_extra_fields, extra_nonce)) { - crypto::hash payment_id; if(get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) { - uint64_t received = (tx_money_spent_in_ins < tx_money_got_in_outs) ? tx_money_got_in_outs - tx_money_spent_in_ins : 0; - if (0 < received && null_hash != payment_id) - { - payment_details payment; - payment.m_tx_hash = cryptonote::get_transaction_hash(tx); - payment.m_amount = received; - payment.m_block_height = height; - payment.m_unlock_time = tx.unlock_time; - m_payments.emplace(payment_id, payment); - LOG_PRINT_L2("Payment found: " << payment_id << " / " << payment.m_tx_hash << " / " << payment.m_amount); - } + // We got a payment ID to go with this tx } } + uint64_t received = (tx_money_spent_in_ins < tx_money_got_in_outs) ? tx_money_got_in_outs - tx_money_spent_in_ins : 0; + if (0 < received) + { + payment_details payment; + payment.m_tx_hash = cryptonote::get_transaction_hash(tx); + payment.m_amount = received; + payment.m_block_height = height; + payment.m_unlock_time = tx.unlock_time; + m_payments.emplace(payment_id, payment); + LOG_PRINT_L2("Payment found: " << payment_id << " / " << payment.m_tx_hash << " / " << payment.m_amount); + } } //---------------------------------------------------------------------------------------------------- void wallet2::process_unconfirmed(const cryptonote::transaction& tx) @@ -571,6 +572,54 @@ void wallet2::load_keys(const std::string& keys_file_name, const std::string& pa } /*! + * \brief verify password for default wallet keys file. + * \param password Password to verify + * + * for verification only + * should not mutate state, unlike load_keys() + * can be used prior to rewriting wallet keys file, to ensure user has entered the correct password + * + */ +bool wallet2::verify_password(const std::string& password) +{ + const std::string keys_file_name = m_keys_file; + wallet2::keys_file_data keys_file_data; + std::string buf; + bool r = epee::file_io_utils::load_file_to_string(keys_file_name, buf); + THROW_WALLET_EXCEPTION_IF(!r, error::file_read_error, keys_file_name); + + // Decrypt the contents + r = ::serialization::parse_binary(buf, keys_file_data); + THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "internal error: failed to deserialize \"" + keys_file_name + '\"'); + crypto::chacha8_key key; + crypto::generate_chacha8_key(password, key); + std::string account_data; + account_data.resize(keys_file_data.account_data.size()); + crypto::chacha8(keys_file_data.account_data.data(), keys_file_data.account_data.size(), key, keys_file_data.iv, &account_data[0]); + + // The contents should be JSON if the wallet follows the new format. + rapidjson::Document json; + if (json.Parse(account_data.c_str(), keys_file_data.account_data.size()).HasParseError()) + { + // old format before JSON wallet key file format + } + else + { + account_data = std::string(json["key_data"].GetString(), json["key_data"].GetString() + + json["key_data"].GetStringLength()); + } + + cryptonote::account_base account_data_check; + + r = epee::serialization::load_t_from_binary(account_data_check, account_data); + const cryptonote::account_keys& keys = account_data_check.get_keys(); + + r = r && verify_keys(keys.m_view_secret_key, keys.m_account_address.m_view_public_key); + r = r && verify_keys(keys.m_spend_secret_key, keys.m_account_address.m_spend_public_key); + return r; +} + +/*! * \brief Generates a wallet or restores one. * \param wallet_ Name of wallet file * \param password Password of wallet file @@ -616,11 +665,9 @@ void wallet2::rewrite(const std::string& wallet_name, const std::string& passwor { prepare_file_names(wallet_name); boost::system::error_code ignored_ec; - THROW_WALLET_EXCEPTION_IF(!boost::filesystem::exists(m_wallet_file, ignored_ec), error::file_not_found, m_wallet_file); THROW_WALLET_EXCEPTION_IF(!boost::filesystem::exists(m_keys_file, ignored_ec), error::file_not_found, m_keys_file); bool r = store_keys(m_keys_file, password); THROW_WALLET_EXCEPTION_IF(!r, error::file_save_error, m_keys_file); - store(); } //---------------------------------------------------------------------------------------------------- void wallet2::wallet_exists(const std::string& file_path, bool& keys_file_exists, bool& wallet_file_exists) @@ -633,6 +680,11 @@ void wallet2::wallet_exists(const std::string& file_path, bool& keys_file_exists wallet_file_exists = boost::filesystem::exists(wallet_file, ignore); } //---------------------------------------------------------------------------------------------------- +bool wallet2::wallet_valid_path_format(const std::string& file_path) +{ + return !file_path.empty(); +} +//---------------------------------------------------------------------------------------------------- bool wallet2::parse_payment_id(const std::string& payment_id_str, crypto::hash& payment_id) { cryptonote::blobdata payment_id_data; @@ -686,14 +738,16 @@ void wallet2::load(const std::string& wallet_, const std::string& password) { LOG_PRINT_L0("file not found: " << m_wallet_file << ", starting with empty blockchain"); m_account_public_address = m_account.get_keys().m_account_address; - return; } - bool r = tools::unserialize_obj_from_file(*this, m_wallet_file); - THROW_WALLET_EXCEPTION_IF(!r, error::file_read_error, m_wallet_file); - THROW_WALLET_EXCEPTION_IF( - m_account_public_address.m_spend_public_key != m_account.get_keys().m_account_address.m_spend_public_key || - m_account_public_address.m_view_public_key != m_account.get_keys().m_account_address.m_view_public_key, - error::wallet_files_doesnt_correspond, m_keys_file, m_wallet_file); + else + { + bool r = tools::unserialize_obj_from_file(*this, m_wallet_file); + THROW_WALLET_EXCEPTION_IF(!r, error::file_read_error, m_wallet_file); + THROW_WALLET_EXCEPTION_IF( + m_account_public_address.m_spend_public_key != m_account.get_keys().m_account_address.m_spend_public_key || + m_account_public_address.m_view_public_key != m_account.get_keys().m_account_address.m_view_public_key, + error::wallet_files_doesnt_correspond, m_keys_file, m_wallet_file); + } cryptonote::block genesis; generate_genesis(genesis); @@ -763,6 +817,17 @@ void wallet2::get_payments(const crypto::hash& payment_id, std::list<wallet2::pa }); } //---------------------------------------------------------------------------------------------------- +void wallet2::get_payments(std::list<std::pair<crypto::hash,wallet2::payment_details>>& payments, uint64_t min_height) const +{ + auto range = std::make_pair(m_payments.begin(), m_payments.end()); + std::for_each(range.first, range.second, [&payments, &min_height](const payment_container::value_type& x) { + if (min_height < x.second.m_block_height) + { + payments.push_back(x); + } + }); +} +//---------------------------------------------------------------------------------------------------- bool wallet2::is_transfer_unlocked(const transfer_details& td) const { if(!is_tx_spendtime_unlocked(td.m_tx.unlock_time)) diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index f22c5d79d..1febfba39 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -82,7 +82,7 @@ namespace tools { wallet2(const wallet2&) : m_run(true), m_callback(0), m_testnet(false) {}; public: - wallet2(bool testnet = false) : m_run(true), m_callback(0), m_testnet(testnet), is_old_file_format(false) {}; + wallet2(bool testnet = false, bool restricted = false) : m_run(true), m_callback(0), m_testnet(testnet), m_restricted(restricted), is_old_file_format(false) {}; struct transfer_details { uint64_t m_block_height; @@ -153,6 +153,11 @@ namespace tools void rewrite(const std::string& wallet_name, const std::string& password); void load(const std::string& wallet, const std::string& password); void store(); + + /*! + * \brief verifies given password is correct for default wallet keys file + */ + bool verify_password(const std::string& password); cryptonote::account_base& get_account(){return m_account;} // upper_transaction_size_limit as defined below is set to @@ -191,6 +196,7 @@ namespace tools bool refresh(size_t & blocks_fetched, bool& received_money, bool& ok); bool testnet() { return m_testnet; } + bool restricted() const { return m_restricted; } uint64_t balance(); uint64_t unlocked_balance(); @@ -206,6 +212,7 @@ namespace tools bool check_connection(); void get_transfers(wallet2::transfer_container& incoming_transfers) const; void get_payments(const crypto::hash& payment_id, std::list<wallet2::payment_details>& payments, uint64_t min_height = 0) const; + void get_payments(std::list<std::pair<crypto::hash,wallet2::payment_details>>& payments, uint64_t min_height) const; uint64_t get_blockchain_current_height() const { return m_local_bc_height; } template <class t_archive> inline void serialize(t_archive &a, const unsigned int ver) @@ -224,7 +231,19 @@ namespace tools a & m_payments; } + /*! + * \brief Check if wallet keys and bin files exist + * \param file_path Wallet file path + * \param keys_file_exists Whether keys file exists + * \param wallet_file_exists Whether bin file exists + */ static void wallet_exists(const std::string& file_path, bool& keys_file_exists, bool& wallet_file_exists); + /*! + * \brief Check if wallet file path is valid format + * \param file_path Wallet file path + * \return Whether path is valid format + */ + static bool wallet_valid_path_format(const std::string& file_path); static bool parse_payment_id(const std::string& payment_id_str, crypto::hash& payment_id); @@ -279,6 +298,7 @@ namespace tools i_wallet2_callback* m_callback; bool m_testnet; + bool m_restricted; std::string seed_language; /*!< Language of the mnemonics (seed). */ bool is_old_file_format; /*!< Whether the wallet file is of an old file format */ }; diff --git a/src/wallet/wallet_errors.h b/src/wallet/wallet_errors.h index ebec931e4..bf520c36c 100644 --- a/src/wallet/wallet_errors.h +++ b/src/wallet/wallet_errors.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index 66cf64166..e39d9ba7b 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -85,7 +85,7 @@ namespace tools return epee::http_server_impl_base<wallet_rpc_server, connection_context>::init(m_port, m_bind_ip); } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_getbalance(const wallet_rpc::COMMAND_RPC_GET_BALANCE::request& req, wallet_rpc::COMMAND_RPC_GET_BALANCE::response& res, epee::json_rpc::error& er, connection_context& cntx) + bool wallet_rpc_server::on_getbalance(const wallet_rpc::COMMAND_RPC_GET_BALANCE::request& req, wallet_rpc::COMMAND_RPC_GET_BALANCE::response& res, epee::json_rpc::error& er) { try { @@ -101,7 +101,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_getaddress(const wallet_rpc::COMMAND_RPC_GET_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS::response& res, epee::json_rpc::error& er, connection_context& cntx) + bool wallet_rpc_server::on_getaddress(const wallet_rpc::COMMAND_RPC_GET_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS::response& res, epee::json_rpc::error& er) { try { @@ -161,12 +161,19 @@ namespace tools } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_transfer(const wallet_rpc::COMMAND_RPC_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_TRANSFER::response& res, epee::json_rpc::error& er, connection_context& cntx) + bool wallet_rpc_server::on_transfer(const wallet_rpc::COMMAND_RPC_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_TRANSFER::response& res, epee::json_rpc::error& er) { std::vector<cryptonote::tx_destination_entry> dsts; std::vector<uint8_t> extra; + if (m_wallet.restricted()) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + // validate the transfer requested and populate dsts & extra if (!validate_transfer(req.destinations, req.payment_id, dsts, extra, er)) { @@ -212,12 +219,19 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_transfer_split(const wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::request& req, wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::response& res, epee::json_rpc::error& er, connection_context& cntx) + bool wallet_rpc_server::on_transfer_split(const wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::request& req, wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::response& res, epee::json_rpc::error& er) { std::vector<cryptonote::tx_destination_entry> dsts; std::vector<uint8_t> extra; + if (m_wallet.restricted()) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + // validate the transfer requested and populate dsts & extra; RPC_TRANSFER::request and RPC_TRANSFER_SPLIT::request are identical types. if (!validate_transfer(req.destinations, req.payment_id, dsts, extra, er)) { @@ -259,8 +273,15 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_store(const wallet_rpc::COMMAND_RPC_STORE::request& req, wallet_rpc::COMMAND_RPC_STORE::response& res, epee::json_rpc::error& er, connection_context& cntx) + bool wallet_rpc_server::on_store(const wallet_rpc::COMMAND_RPC_STORE::request& req, wallet_rpc::COMMAND_RPC_STORE::response& res, epee::json_rpc::error& er) { + if (m_wallet.restricted()) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + try { m_wallet.store(); @@ -274,7 +295,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_get_payments(const wallet_rpc::COMMAND_RPC_GET_PAYMENTS::request& req, wallet_rpc::COMMAND_RPC_GET_PAYMENTS::response& res, epee::json_rpc::error& er, connection_context& cntx) + bool wallet_rpc_server::on_get_payments(const wallet_rpc::COMMAND_RPC_GET_PAYMENTS::request& req, wallet_rpc::COMMAND_RPC_GET_PAYMENTS::response& res, epee::json_rpc::error& er) { crypto::hash payment_id; cryptonote::blobdata payment_id_blob; @@ -311,10 +332,30 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_get_bulk_payments(const wallet_rpc::COMMAND_RPC_GET_BULK_PAYMENTS::request& req, wallet_rpc::COMMAND_RPC_GET_BULK_PAYMENTS::response& res, epee::json_rpc::error& er, connection_context& cntx) + bool wallet_rpc_server::on_get_bulk_payments(const wallet_rpc::COMMAND_RPC_GET_BULK_PAYMENTS::request& req, wallet_rpc::COMMAND_RPC_GET_BULK_PAYMENTS::response& res, epee::json_rpc::error& er) { res.payments.clear(); + /* If the payment ID list is empty, we get payments to any payment ID (or lack thereof) */ + if (req.payment_ids.empty()) + { + std::list<std::pair<crypto::hash,wallet2::payment_details>> payment_list; + m_wallet.get_payments(payment_list, req.min_block_height); + + for (auto & payment : payment_list) + { + wallet_rpc::payment_details rpc_payment; + rpc_payment.payment_id = epee::string_tools::pod_to_hex(payment.first); + rpc_payment.tx_hash = epee::string_tools::pod_to_hex(payment.second.m_tx_hash); + rpc_payment.amount = payment.second.m_amount; + rpc_payment.block_height = payment.second.m_block_height; + rpc_payment.unlock_time = payment.second.m_unlock_time; + res.payments.push_back(std::move(rpc_payment)); + } + + return true; + } + for (auto & payment_id_str : req.payment_ids) { crypto::hash payment_id; @@ -356,7 +397,7 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_incoming_transfers(const wallet_rpc::COMMAND_RPC_INCOMING_TRANSFERS::request& req, wallet_rpc::COMMAND_RPC_INCOMING_TRANSFERS::response& res, epee::json_rpc::error& er, connection_context& cntx) + bool wallet_rpc_server::on_incoming_transfers(const wallet_rpc::COMMAND_RPC_INCOMING_TRANSFERS::request& req, wallet_rpc::COMMAND_RPC_INCOMING_TRANSFERS::response& res, epee::json_rpc::error& er) { if(req.transfer_type.compare("all") != 0 && req.transfer_type.compare("available") != 0 && req.transfer_type.compare("unavailable") != 0) { @@ -390,11 +431,13 @@ namespace tools { transfers_found = true; } + auto txBlob = t_serializable_object_to_blob(td.m_tx); wallet_rpc::transfer_details rpc_transfers; rpc_transfers.amount = td.amount(); rpc_transfers.spent = td.m_spent; rpc_transfers.global_index = td.m_global_output_index; rpc_transfers.tx_hash = boost::lexical_cast<std::string>(cryptonote::get_transaction_hash(td.m_tx)); + rpc_transfers.tx_size = txBlob.size(); res.transfers.push_back(rpc_transfers); } } @@ -407,8 +450,15 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool wallet_rpc_server::on_query_key(const wallet_rpc::COMMAND_RPC_QUERY_KEY::request& req, wallet_rpc::COMMAND_RPC_QUERY_KEY::response& res, epee::json_rpc::error& er, connection_context& cntx) + bool wallet_rpc_server::on_query_key(const wallet_rpc::COMMAND_RPC_QUERY_KEY::request& req, wallet_rpc::COMMAND_RPC_QUERY_KEY::response& res, epee::json_rpc::error& er) { + if (m_wallet.restricted()) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + if (req.key_type.compare("mnemonic") == 0) { if (!m_wallet.get_seed(res.key)) diff --git a/src/wallet/wallet_rpc_server.h b/src/wallet/wallet_rpc_server.h index 6f52e87ba..cfdc1c147 100644 --- a/src/wallet/wallet_rpc_server.h +++ b/src/wallet/wallet_rpc_server.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -74,20 +74,20 @@ namespace tools END_URI_MAP2() //json_rpc - bool on_getbalance(const wallet_rpc::COMMAND_RPC_GET_BALANCE::request& req, wallet_rpc::COMMAND_RPC_GET_BALANCE::response& res, epee::json_rpc::error& er, connection_context& cntx); - bool on_getaddress(const wallet_rpc::COMMAND_RPC_GET_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS::response& res, epee::json_rpc::error& er, connection_context& cntx); + bool on_getbalance(const wallet_rpc::COMMAND_RPC_GET_BALANCE::request& req, wallet_rpc::COMMAND_RPC_GET_BALANCE::response& res, epee::json_rpc::error& er); + bool on_getaddress(const wallet_rpc::COMMAND_RPC_GET_ADDRESS::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS::response& res, epee::json_rpc::error& er); bool validate_transfer(const std::list<wallet_rpc::transfer_destination> destinations, const std::string payment_id, std::vector<cryptonote::tx_destination_entry>& dsts, std::vector<uint8_t>& extra, epee::json_rpc::error& er); - bool on_transfer(const wallet_rpc::COMMAND_RPC_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_TRANSFER::response& res, epee::json_rpc::error& er, connection_context& cntx); - bool on_transfer_split(const wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::request& req, wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::response& res, epee::json_rpc::error& er, connection_context& cntx); - bool on_store(const wallet_rpc::COMMAND_RPC_STORE::request& req, wallet_rpc::COMMAND_RPC_STORE::response& res, epee::json_rpc::error& er, connection_context& cntx); - bool on_get_payments(const wallet_rpc::COMMAND_RPC_GET_PAYMENTS::request& req, wallet_rpc::COMMAND_RPC_GET_PAYMENTS::response& res, epee::json_rpc::error& er, connection_context& cntx); - bool on_get_bulk_payments(const wallet_rpc::COMMAND_RPC_GET_BULK_PAYMENTS::request& req, wallet_rpc::COMMAND_RPC_GET_BULK_PAYMENTS::response& res, epee::json_rpc::error& er, connection_context& cntx); - bool on_incoming_transfers(const wallet_rpc::COMMAND_RPC_INCOMING_TRANSFERS::request& req, wallet_rpc::COMMAND_RPC_INCOMING_TRANSFERS::response& res, epee::json_rpc::error& er, connection_context& cntx); + bool on_transfer(const wallet_rpc::COMMAND_RPC_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_TRANSFER::response& res, epee::json_rpc::error& er); + bool on_transfer_split(const wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::request& req, wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::response& res, epee::json_rpc::error& er); + bool on_store(const wallet_rpc::COMMAND_RPC_STORE::request& req, wallet_rpc::COMMAND_RPC_STORE::response& res, epee::json_rpc::error& er); + bool on_get_payments(const wallet_rpc::COMMAND_RPC_GET_PAYMENTS::request& req, wallet_rpc::COMMAND_RPC_GET_PAYMENTS::response& res, epee::json_rpc::error& er); + bool on_get_bulk_payments(const wallet_rpc::COMMAND_RPC_GET_BULK_PAYMENTS::request& req, wallet_rpc::COMMAND_RPC_GET_BULK_PAYMENTS::response& res, epee::json_rpc::error& er); + bool on_incoming_transfers(const wallet_rpc::COMMAND_RPC_INCOMING_TRANSFERS::request& req, wallet_rpc::COMMAND_RPC_INCOMING_TRANSFERS::response& res, epee::json_rpc::error& er); bool handle_command_line(const boost::program_options::variables_map& vm); //json rpc v2 - bool on_query_key(const wallet_rpc::COMMAND_RPC_QUERY_KEY::request& req, wallet_rpc::COMMAND_RPC_QUERY_KEY::response& res, epee::json_rpc::error& er, connection_context& cntx); + bool on_query_key(const wallet_rpc::COMMAND_RPC_QUERY_KEY::request& req, wallet_rpc::COMMAND_RPC_QUERY_KEY::response& res, epee::json_rpc::error& er); wallet2& m_wallet; std::string m_port; diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h index 48f8ec2a5..35783a189 100644 --- a/src/wallet/wallet_rpc_server_commands_defs.h +++ b/src/wallet/wallet_rpc_server_commands_defs.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -228,12 +228,14 @@ namespace wallet_rpc bool spent; uint64_t global_index; std::string tx_hash; + uint64_t tx_size; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(amount) KV_SERIALIZE(spent) KV_SERIALIZE(global_index) KV_SERIALIZE(tx_hash) + KV_SERIALIZE(tx_size) END_KV_SERIALIZE_MAP() }; diff --git a/src/wallet/wallet_rpc_server_error_codes.h b/src/wallet/wallet_rpc_server_error_codes.h index 75f5e2b9c..28642c19d 100644 --- a/src/wallet/wallet_rpc_server_error_codes.h +++ b/src/wallet/wallet_rpc_server_error_codes.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -37,3 +37,4 @@ #define WALLET_RPC_ERROR_CODE_GENERIC_TRANSFER_ERROR -4 #define WALLET_RPC_ERROR_CODE_WRONG_PAYMENT_ID -5 #define WALLET_RPC_ERROR_CODE_TRANSFER_TYPE -6 +#define WALLET_RPC_ERROR_CODE_DENIED -7 diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ae20b8754..459ed3ed8 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # diff --git a/tests/core_proxy/CMakeLists.txt b/tests/core_proxy/CMakeLists.txt index ab6f7d043..7d40d72b6 100644 --- a/tests/core_proxy/CMakeLists.txt +++ b/tests/core_proxy/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # @@ -38,6 +38,8 @@ add_executable(core_proxy target_link_libraries(core_proxy LINK_PRIVATE cryptonote_core + cryptonote_protocol + p2p ${UPNP_LIBRARIES} ${Boost_CHRONO_LIBRARY} ${Boost_FILESYSTEM_LIBRARY} diff --git a/tests/core_proxy/core_proxy.cpp b/tests/core_proxy/core_proxy.cpp index 3a44e91eb..b942ed5c5 100644 --- a/tests/core_proxy/core_proxy.cpp +++ b/tests/core_proxy/core_proxy.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -62,6 +62,8 @@ using namespace crypto; BOOST_CLASS_VERSION(nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<tests::proxy_core> >, 1); +unsigned int epee::g_test_dbg_lock_sleep = 0; + int main(int argc, char* argv[]) { @@ -106,7 +108,6 @@ int main(int argc, char* argv[]) cryptonote::t_cryptonote_protocol_handler<tests::proxy_core> cprotocol(pr_core, NULL); nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<tests::proxy_core> > p2psrv { cprotocol - , std::move(config::NETWORK_ID) }; cprotocol.set_p2p_endpoint(&p2psrv); //pr_core.set_cryptonote_protocol(&cprotocol); @@ -115,7 +116,7 @@ int main(int argc, char* argv[]) //initialize objects LOG_PRINT_L0("Initializing p2p server..."); - bool res = p2psrv.init(vm, false); + bool res = p2psrv.init(vm); CHECK_AND_ASSERT_MES(res, 1, "Failed to initialize p2p server."); LOG_PRINT_L0("P2p server initialized OK"); @@ -148,6 +149,7 @@ int main(int argc, char* argv[]) LOG_PRINT("Node stopped.", LOG_LEVEL_0); + epee::net_utils::data_logger::get_instance().kill_instance(); return 0; CATCH_ENTRY_L0("main", 1); diff --git a/tests/core_proxy/core_proxy.h b/tests/core_proxy/core_proxy.h index bfbd03661..09582da40 100644 --- a/tests/core_proxy/core_proxy.h +++ b/tests/core_proxy/core_proxy.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -34,6 +34,7 @@ #include "cryptonote_core/cryptonote_basic_impl.h" #include "cryptonote_core/verification_context.h" +#include "cryptonote_core/blockchain_storage.h" #include <unordered_map> namespace tests @@ -81,5 +82,8 @@ namespace tests bool on_idle(){return true;} bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, cryptonote::NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp){return true;} bool handle_get_objects(cryptonote::NOTIFY_REQUEST_GET_OBJECTS::request& arg, cryptonote::NOTIFY_RESPONSE_GET_OBJECTS::request& rsp, cryptonote::cryptonote_connection_context& context){return true;} + cryptonote::blockchain_storage &get_blockchain_storage() { throw std::runtime_error("Called invalid member function: please never call get_blockchain_storage on the TESTING class proxy_core."); } + bool get_test_drop_download() {return true;} + bool get_test_drop_download_height() {return true;} }; } diff --git a/tests/core_tests/CMakeLists.txt b/tests/core_tests/CMakeLists.txt index 72c079554..2b9e0cf81 100644 --- a/tests/core_tests/CMakeLists.txt +++ b/tests/core_tests/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # @@ -60,6 +60,8 @@ add_executable(coretests target_link_libraries(coretests LINK_PRIVATE cryptonote_core + p2p + ${Boost_CHRONO_LIBRARY} ${Boost_FILESYSTEM_LIBRARY} ${Boost_SYSTEM_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} diff --git a/tests/core_tests/block_reward.cpp b/tests/core_tests/block_reward.cpp index 795c58027..789c6fffa 100644 --- a/tests/core_tests/block_reward.cpp +++ b/tests/core_tests/block_reward.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/core_tests/block_reward.h b/tests/core_tests/block_reward.h index b3bf92c25..274b46867 100644 --- a/tests/core_tests/block_reward.h +++ b/tests/core_tests/block_reward.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/core_tests/block_validation.cpp b/tests/core_tests/block_validation.cpp index a8d5d741e..97d6634c0 100644 --- a/tests/core_tests/block_validation.cpp +++ b/tests/core_tests/block_validation.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/core_tests/block_validation.h b/tests/core_tests/block_validation.h index 8ebf0d975..5da746d0b 100644 --- a/tests/core_tests/block_validation.h +++ b/tests/core_tests/block_validation.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/core_tests/chain_split_1.cpp b/tests/core_tests/chain_split_1.cpp index 3893aa396..9fd75d411 100644 --- a/tests/core_tests/chain_split_1.cpp +++ b/tests/core_tests/chain_split_1.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/core_tests/chain_split_1.h b/tests/core_tests/chain_split_1.h index ccd5f6b46..4a1503550 100644 --- a/tests/core_tests/chain_split_1.h +++ b/tests/core_tests/chain_split_1.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/core_tests/chain_switch_1.cpp b/tests/core_tests/chain_switch_1.cpp index ce54cbb71..ffc41a916 100644 --- a/tests/core_tests/chain_switch_1.cpp +++ b/tests/core_tests/chain_switch_1.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/core_tests/chain_switch_1.h b/tests/core_tests/chain_switch_1.h index 0783929c6..7e8b64184 100644 --- a/tests/core_tests/chain_switch_1.h +++ b/tests/core_tests/chain_switch_1.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/core_tests/chaingen.cpp b/tests/core_tests/chaingen.cpp index 024975c5a..9e7576c12 100644 --- a/tests/core_tests/chaingen.cpp +++ b/tests/core_tests/chaingen.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/core_tests/chaingen.h b/tests/core_tests/chaingen.h index d2990e009..d187f36ca 100644 --- a/tests/core_tests/chaingen.h +++ b/tests/core_tests/chaingen.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -487,7 +487,9 @@ inline bool do_replay_events(std::vector<test_event_entry>& events) cryptonote::cryptonote_protocol_stub pr; //TODO: stub only for this kind of test, make real validation of relayed objects cryptonote::core c(&pr); - if (!c.init(vm, false)) + // FIXME: make sure that vm has arg_testnet_on set to true or false if + // this test needs for it to be so. + if (!c.init(vm)) { std::cout << concolor::magenta << "Failed to init core" << concolor::normal << std::endl; return false; diff --git a/tests/core_tests/chaingen001.cpp b/tests/core_tests/chaingen001.cpp index 88f93769a..29d3ce393 100644 --- a/tests/core_tests/chaingen001.cpp +++ b/tests/core_tests/chaingen001.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/core_tests/chaingen_main.cpp b/tests/core_tests/chaingen_main.cpp index 465fe6933..a9ef8b00b 100644 --- a/tests/core_tests/chaingen_main.cpp +++ b/tests/core_tests/chaingen_main.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -44,6 +44,8 @@ namespace const command_line::arg_descriptor<bool> arg_test_transactions = {"test_transactions", ""}; } +unsigned int epee::g_test_dbg_lock_sleep = 0; + int main(int argc, char* argv[]) { TRY_ENTRY(); diff --git a/tests/core_tests/chaingen_tests_list.h b/tests/core_tests/chaingen_tests_list.h index dd71e7e41..6e39b7443 100644 --- a/tests/core_tests/chaingen_tests_list.h +++ b/tests/core_tests/chaingen_tests_list.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/core_tests/double_spend.cpp b/tests/core_tests/double_spend.cpp index 1b209a7e7..40913a834 100644 --- a/tests/core_tests/double_spend.cpp +++ b/tests/core_tests/double_spend.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/core_tests/double_spend.h b/tests/core_tests/double_spend.h index 1086b8a4f..0a2e60575 100644 --- a/tests/core_tests/double_spend.h +++ b/tests/core_tests/double_spend.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/core_tests/double_spend.inl b/tests/core_tests/double_spend.inl index 9840adf86..cfd002d37 100644 --- a/tests/core_tests/double_spend.inl +++ b/tests/core_tests/double_spend.inl @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/core_tests/integer_overflow.cpp b/tests/core_tests/integer_overflow.cpp index 2188a4f63..120dc0caf 100644 --- a/tests/core_tests/integer_overflow.cpp +++ b/tests/core_tests/integer_overflow.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/core_tests/integer_overflow.h b/tests/core_tests/integer_overflow.h index 6af229cbe..31e044776 100644 --- a/tests/core_tests/integer_overflow.h +++ b/tests/core_tests/integer_overflow.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/core_tests/ring_signature_1.cpp b/tests/core_tests/ring_signature_1.cpp index ed9b16953..15b6531c4 100644 --- a/tests/core_tests/ring_signature_1.cpp +++ b/tests/core_tests/ring_signature_1.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/core_tests/ring_signature_1.h b/tests/core_tests/ring_signature_1.h index 7f14f997d..0a174742b 100644 --- a/tests/core_tests/ring_signature_1.h +++ b/tests/core_tests/ring_signature_1.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/core_tests/transaction_tests.cpp b/tests/core_tests/transaction_tests.cpp index d46d48b07..bd11a76bd 100644 --- a/tests/core_tests/transaction_tests.cpp +++ b/tests/core_tests/transaction_tests.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/core_tests/transaction_tests.h b/tests/core_tests/transaction_tests.h index daca4a545..9042a0f62 100644 --- a/tests/core_tests/transaction_tests.h +++ b/tests/core_tests/transaction_tests.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/core_tests/tx_validation.cpp b/tests/core_tests/tx_validation.cpp index a4323ce31..c642f5455 100644 --- a/tests/core_tests/tx_validation.cpp +++ b/tests/core_tests/tx_validation.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/core_tests/tx_validation.h b/tests/core_tests/tx_validation.h index bd62ec7ea..a83e99c4c 100644 --- a/tests/core_tests/tx_validation.h +++ b/tests/core_tests/tx_validation.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/crypto/CMakeLists.txt b/tests/crypto/CMakeLists.txt index 83982eda1..a6f56827d 100644 --- a/tests/crypto/CMakeLists.txt +++ b/tests/crypto/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # diff --git a/tests/crypto/crypto-ops-data.c b/tests/crypto/crypto-ops-data.c index c755a693e..205c0e7c0 100644 --- a/tests/crypto/crypto-ops-data.c +++ b/tests/crypto/crypto-ops-data.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/crypto/crypto-ops.c b/tests/crypto/crypto-ops.c index 6cc296310..b5771e7c1 100644 --- a/tests/crypto/crypto-ops.c +++ b/tests/crypto/crypto-ops.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/crypto/crypto-tests.h b/tests/crypto/crypto-tests.h index 0da9a989c..a85cf9893 100644 --- a/tests/crypto/crypto-tests.h +++ b/tests/crypto/crypto-tests.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/crypto/crypto.cpp b/tests/crypto/crypto.cpp index 4bfb26caf..7478b5a60 100644 --- a/tests/crypto/crypto.cpp +++ b/tests/crypto/crypto.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/crypto/hash.c b/tests/crypto/hash.c index 82e23c113..01cd53399 100644 --- a/tests/crypto/hash.c +++ b/tests/crypto/hash.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/crypto/main.cpp b/tests/crypto/main.cpp index f16070c42..9bde85a83 100644 --- a/tests/crypto/main.cpp +++ b/tests/crypto/main.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/crypto/random.c b/tests/crypto/random.c index 12d5c7163..f6c860b7b 100644 --- a/tests/crypto/random.c +++ b/tests/crypto/random.c @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/cryptolib.pl b/tests/cryptolib.pl index 72765e9f3..022489a16 100644 --- a/tests/cryptolib.pl +++ b/tests/cryptolib.pl @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # diff --git a/tests/cryptotest.pl b/tests/cryptotest.pl index 0fc6c8b66..c5a48dbfa 100644 --- a/tests/cryptotest.pl +++ b/tests/cryptotest.pl @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # diff --git a/tests/daemon_tests/CMakeLists.txt b/tests/daemon_tests/CMakeLists.txt index b7a93481c..fe0221fa7 100644 --- a/tests/daemon_tests/CMakeLists.txt +++ b/tests/daemon_tests/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # diff --git a/tests/daemon_tests/transfers.cpp b/tests/daemon_tests/transfers.cpp index ff73e1931..1e94c93ce 100644 --- a/tests/daemon_tests/transfers.cpp +++ b/tests/daemon_tests/transfers.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/data/account-007af2d7c5ffd8f69005debae820207820805e28c7d7a16714591143f56fb51e2b91ad0c1a535567e6292b321773df5e5aaace00fe767c4f09de452838575357ca9f b/tests/data/account-007af2d7c5ffd8f69005debae820207820805e28c7d7a16714591143f56fb51e2b91ad0c1a535567e6292b321773df5e5aaace00fe767c4f09de452838575357ca9f Binary files differindex 12f4a0f10..7a3e6d58c 100644 --- a/tests/data/account-007af2d7c5ffd8f69005debae820207820805e28c7d7a16714591143f56fb51e2b91ad0c1a535567e6292b321773df5e5aaace00fe767c4f09de452838575357ca9f +++ b/tests/data/account-007af2d7c5ffd8f69005debae820207820805e28c7d7a16714591143f56fb51e2b91ad0c1a535567e6292b321773df5e5aaace00fe767c4f09de452838575357ca9f diff --git a/tests/data/account-00aff84db50d6a54dd56051379f6c336fdd330d1cb11e7523bbf71f30b1ae760fa47ace8679b6486f79429980fd2331715a631f5729db284eb1fc6f108aeb7a7f4fe b/tests/data/account-00aff84db50d6a54dd56051379f6c336fdd330d1cb11e7523bbf71f30b1ae760fa47ace8679b6486f79429980fd2331715a631f5729db284eb1fc6f108aeb7a7f4fe Binary files differindex 54122832c..58afb2e3c 100644 --- a/tests/data/account-00aff84db50d6a54dd56051379f6c336fdd330d1cb11e7523bbf71f30b1ae760fa47ace8679b6486f79429980fd2331715a631f5729db284eb1fc6f108aeb7a7f4fe +++ b/tests/data/account-00aff84db50d6a54dd56051379f6c336fdd330d1cb11e7523bbf71f30b1ae760fa47ace8679b6486f79429980fd2331715a631f5729db284eb1fc6f108aeb7a7f4fe diff --git a/tests/difficulty/CMakeLists.txt b/tests/difficulty/CMakeLists.txt index 20600a15a..1d3f78f09 100644 --- a/tests/difficulty/CMakeLists.txt +++ b/tests/difficulty/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # diff --git a/tests/difficulty/difficulty.cpp b/tests/difficulty/difficulty.cpp index 44840d0d5..efe7a2068 100644 --- a/tests/difficulty/difficulty.cpp +++ b/tests/difficulty/difficulty.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/difficulty/generate-data b/tests/difficulty/generate-data index 804a76399..4aa3ac7b1 100755 --- a/tests/difficulty/generate-data +++ b/tests/difficulty/generate-data @@ -1,6 +1,6 @@ #!/usr/bin/python3 -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # @@ -79,4 +79,4 @@ for i in range(1000): time += randint(-diff // power - 10, 3 * diff // power + 10) assert 0 <= time <= UINT_MAX print(time, diff) - diff = gen.send(time)
\ No newline at end of file + diff = gen.send(time) diff --git a/tests/functional_tests/CMakeLists.txt b/tests/functional_tests/CMakeLists.txt index 71b7c6e01..80351693f 100644 --- a/tests/functional_tests/CMakeLists.txt +++ b/tests/functional_tests/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # diff --git a/tests/functional_tests/main.cpp b/tests/functional_tests/main.cpp index 68a93ac69..a653e7b1b 100644 --- a/tests/functional_tests/main.cpp +++ b/tests/functional_tests/main.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -55,6 +55,8 @@ namespace const command_line::arg_descriptor<size_t> arg_test_repeat_count = {"test_repeat_count", "", 1}; } +unsigned int epee::g_test_dbg_lock_sleep = 0; + int main(int argc, char* argv[]) { TRY_ENTRY(); diff --git a/tests/functional_tests/transactions_flow_test.cpp b/tests/functional_tests/transactions_flow_test.cpp index 806433ab6..5d88b65f2 100644 --- a/tests/functional_tests/transactions_flow_test.cpp +++ b/tests/functional_tests/transactions_flow_test.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/functional_tests/transactions_flow_test.h b/tests/functional_tests/transactions_flow_test.h index 7f7d5c2c9..0d2f28700 100644 --- a/tests/functional_tests/transactions_flow_test.h +++ b/tests/functional_tests/transactions_flow_test.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/functional_tests/transactions_generation_from_blockchain.cpp b/tests/functional_tests/transactions_generation_from_blockchain.cpp index c16167f65..6612d550c 100644 --- a/tests/functional_tests/transactions_generation_from_blockchain.cpp +++ b/tests/functional_tests/transactions_generation_from_blockchain.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/functional_tests/transactions_generation_from_blockchain.h b/tests/functional_tests/transactions_generation_from_blockchain.h index da5d68200..8dba13144 100644 --- a/tests/functional_tests/transactions_generation_from_blockchain.h +++ b/tests/functional_tests/transactions_generation_from_blockchain.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/gtest/CHANGES b/tests/gtest/CHANGES new file mode 100644 index 000000000..055213242 --- /dev/null +++ b/tests/gtest/CHANGES @@ -0,0 +1,157 @@ +Changes for 1.7.0: + +* New feature: death tests are supported on OpenBSD and in iOS + simulator now. +* New feature: Google Test now implements a protocol to allow + a test runner to detect that a test program has exited + prematurely and report it as a failure (before it would be + falsely reported as a success if the exit code is 0). +* New feature: Test::RecordProperty() can now be used outside of the + lifespan of a test method, in which case it will be attributed to + the current test case or the test program in the XML report. +* New feature (potentially breaking): --gtest_list_tests now prints + the type parameters and value parameters for each test. +* Improvement: char pointers and char arrays are now escaped properly + in failure messages. +* Improvement: failure summary in XML reports now includes file and + line information. +* Improvement: the <testsuites> XML element now has a timestamp attribute. +* Improvement: When --gtest_filter is specified, XML report now doesn't + contain information about tests that are filtered out. +* Fixed the bug where long --gtest_filter flag values are truncated in + death tests. +* Potentially breaking change: RUN_ALL_TESTS() is now implemented as a + function instead of a macro in order to work better with Clang. +* Compatibility fixes with C++ 11 and various platforms. +* Bug/warning fixes. + +Changes for 1.6.0: + +* New feature: ADD_FAILURE_AT() for reporting a test failure at the + given source location -- useful for writing testing utilities. +* New feature: the universal value printer is moved from Google Mock + to Google Test. +* New feature: type parameters and value parameters are reported in + the XML report now. +* A gtest_disable_pthreads CMake option. +* Colored output works in GNU Screen sessions now. +* Parameters of value-parameterized tests are now printed in the + textual output. +* Failures from ad hoc test assertions run before RUN_ALL_TESTS() are + now correctly reported. +* Arguments of ASSERT_XY and EXPECT_XY no longer need to support << to + ostream. +* More complete handling of exceptions. +* GTEST_ASSERT_XY can be used instead of ASSERT_XY in case the latter + name is already used by another library. +* --gtest_catch_exceptions is now true by default, allowing a test + program to continue after an exception is thrown. +* Value-parameterized test fixtures can now derive from Test and + WithParamInterface<T> separately, easing conversion of legacy tests. +* Death test messages are clearly marked to make them more + distinguishable from other messages. +* Compatibility fixes for Android, Google Native Client, MinGW, HP UX, + PowerPC, Lucid autotools, libCStd, Sun C++, Borland C++ Builder (Code Gear), + IBM XL C++ (Visual Age C++), and C++0x. +* Bug fixes and implementation clean-ups. +* Potentially incompatible changes: disables the harmful 'make install' + command in autotools. + +Changes for 1.5.0: + + * New feature: assertions can be safely called in multiple threads + where the pthreads library is available. + * New feature: predicates used inside EXPECT_TRUE() and friends + can now generate custom failure messages. + * New feature: Google Test can now be compiled as a DLL. + * New feature: fused source files are included. + * New feature: prints help when encountering unrecognized Google Test flags. + * Experimental feature: CMake build script (requires CMake 2.6.4+). + * Experimental feature: the Pump script for meta programming. + * double values streamed to an assertion are printed with enough precision + to differentiate any two different values. + * Google Test now works on Solaris and AIX. + * Build and test script improvements. + * Bug fixes and implementation clean-ups. + + Potentially breaking changes: + + * Stopped supporting VC++ 7.1 with exceptions disabled. + * Dropped support for 'make install'. + +Changes for 1.4.0: + + * New feature: the event listener API + * New feature: test shuffling + * New feature: the XML report format is closer to junitreport and can + be parsed by Hudson now. + * New feature: when a test runs under Visual Studio, its failures are + integrated in the IDE. + * New feature: /MD(d) versions of VC++ projects. + * New feature: elapsed time for the tests is printed by default. + * New feature: comes with a TR1 tuple implementation such that Boost + is no longer needed for Combine(). + * New feature: EXPECT_DEATH_IF_SUPPORTED macro and friends. + * New feature: the Xcode project can now produce static gtest + libraries in addition to a framework. + * Compatibility fixes for Solaris, Cygwin, minGW, Windows Mobile, + Symbian, gcc, and C++Builder. + * Bug fixes and implementation clean-ups. + +Changes for 1.3.0: + + * New feature: death tests on Windows, Cygwin, and Mac. + * New feature: ability to use Google Test assertions in other testing + frameworks. + * New feature: ability to run disabled test via + --gtest_also_run_disabled_tests. + * New feature: the --help flag for printing the usage. + * New feature: access to Google Test flag values in user code. + * New feature: a script that packs Google Test into one .h and one + .cc file for easy deployment. + * New feature: support for distributing test functions to multiple + machines (requires support from the test runner). + * Bug fixes and implementation clean-ups. + +Changes for 1.2.1: + + * Compatibility fixes for Linux IA-64 and IBM z/OS. + * Added support for using Boost and other TR1 implementations. + * Changes to the build scripts to support upcoming release of Google C++ + Mocking Framework. + * Added Makefile to the distribution package. + * Improved build instructions in README. + +Changes for 1.2.0: + + * New feature: value-parameterized tests. + * New feature: the ASSERT/EXPECT_(NON)FATAL_FAILURE(_ON_ALL_THREADS) + macros. + * Changed the XML report format to match JUnit/Ant's. + * Added tests to the Xcode project. + * Added scons/SConscript for building with SCons. + * Added src/gtest-all.cc for building Google Test from a single file. + * Fixed compatibility with Solaris and z/OS. + * Enabled running Python tests on systems with python 2.3 installed, + e.g. Mac OS X 10.4. + * Bug fixes. + +Changes for 1.1.0: + + * New feature: type-parameterized tests. + * New feature: exception assertions. + * New feature: printing elapsed time of tests. + * Improved the robustness of death tests. + * Added an Xcode project and samples. + * Adjusted the output format on Windows to be understandable by Visual Studio. + * Minor bug fixes. + +Changes for 1.0.1: + + * Added project files for Visual Studio 7.1. + * Fixed issues with compiling on Mac OS X. + * Fixed issues with compiling on Cygwin. + +Changes for 1.0.0: + + * Initial Open Source release of Google Test diff --git a/tests/gtest/CMakeLists.txt b/tests/gtest/CMakeLists.txt index 0fe26540b..bd78cfe67 100644 --- a/tests/gtest/CMakeLists.txt +++ b/tests/gtest/CMakeLists.txt @@ -59,6 +59,16 @@ include_directories( # Where Google Test's libraries can be found. link_directories(${gtest_BINARY_DIR}/src) +# Summary of tuple support for Microsoft Visual Studio: +# Compiler version(MS) version(cmake) Support +# ---------- ----------- -------------- ----------------------------- +# <= VS 2010 <= 10 <= 1600 Use Google Tests's own tuple. +# VS 2012 11 1700 std::tr1::tuple + _VARIADIC_MAX=10 +# VS 2013 12 1800 std::tr1::tuple +if (MSVC AND MSVC_VERSION EQUAL 1700) + add_definitions(/D _VARIADIC_MAX=10) +endif() + ######################################################################## # # Defines the gtest & gtest_main libraries. User tests should link @@ -77,7 +87,7 @@ target_link_libraries(gtest_main gtest) # # They are not built by default. To build them, set the # gtest_build_samples option to ON. You can do it by running ccmake -# or specifying the -Dbuild_gtest_samples=ON flag when running cmake. +# or specifying the -Dgtest_build_samples=ON flag when running cmake. if (gtest_build_samples) cxx_executable(sample1_unittest samples gtest_main samples/sample1.cc) @@ -124,6 +134,8 @@ if (gtest_build_tests) test/gtest-param-test2_test.cc) cxx_test(gtest-port_test gtest_main) cxx_test(gtest_pred_impl_unittest gtest_main) + cxx_test(gtest_premature_exit_test gtest + test/gtest_premature_exit_test.cc) cxx_test(gtest-printers_test gtest_main) cxx_test(gtest_prod_test gtest_main test/production.cc) @@ -140,10 +152,13 @@ if (gtest_build_tests) ############################################################ # C++ tests built with non-standard compiler flags. - cxx_library(gtest_no_exception "${cxx_no_exception}" - src/gtest-all.cc) - cxx_library(gtest_main_no_exception "${cxx_no_exception}" - src/gtest-all.cc src/gtest_main.cc) + # MSVC 7.1 does not support STL with exceptions disabled. + if (NOT MSVC OR MSVC_VERSION GREATER 1310) + cxx_library(gtest_no_exception "${cxx_no_exception}" + src/gtest-all.cc) + cxx_library(gtest_main_no_exception "${cxx_no_exception}" + src/gtest-all.cc src/gtest_main.cc) + endif() cxx_library(gtest_main_no_rtti "${cxx_no_rtti}" src/gtest-all.cc src/gtest_main.cc) @@ -166,12 +181,10 @@ if (gtest_build_tests) PROPERTIES COMPILE_DEFINITIONS "GTEST_LINKED_AS_SHARED_LIBRARY=1") - if (NOT MSVC OR NOT MSVC_VERSION EQUAL 1600) - # The C++ Standard specifies tuple_element<int, class>. - # Yet MSVC 10's <utility> declares tuple_element<size_t, class>. - # That declaration conflicts with our own standard-conforming - # tuple implementation. Therefore using our own tuple with - # MSVC 10 doesn't compile. + if (NOT MSVC OR MSVC_VERSION LESS 1600) # 1600 is Visual Studio 2010. + # Visual Studio 2010, 2012, and 2013 define symbols in std::tr1 that + # conflict with our own definitions. Therefore using our own tuple does not + # work on those compilers. cxx_library(gtest_main_use_own_tuple "${cxx_use_own_tuple}" src/gtest-all.cc src/gtest_main.cc) @@ -189,11 +202,15 @@ if (gtest_build_tests) cxx_executable(gtest_break_on_failure_unittest_ test gtest) py_test(gtest_break_on_failure_unittest) - cxx_executable_with_flags( - gtest_catch_exceptions_no_ex_test_ - "${cxx_no_exception}" - gtest_main_no_exception - test/gtest_catch_exceptions_test_.cc) + # Visual Studio .NET 2003 does not support STL with exceptions disabled. + if (NOT MSVC OR MSVC_VERSION GREATER 1310) # 1310 is Visual Studio .NET 2003 + cxx_executable_with_flags( + gtest_catch_exceptions_no_ex_test_ + "${cxx_no_exception}" + gtest_main_no_exception + test/gtest_catch_exceptions_test_.cc) + endif() + cxx_executable_with_flags( gtest_catch_exceptions_ex_test_ "${cxx_exception}" @@ -222,11 +239,14 @@ if (gtest_build_tests) cxx_executable(gtest_shuffle_test_ test gtest) py_test(gtest_shuffle_test) - cxx_executable(gtest_throw_on_failure_test_ test gtest_no_exception) - set_target_properties(gtest_throw_on_failure_test_ - PROPERTIES - COMPILE_FLAGS "${cxx_no_exception}") - py_test(gtest_throw_on_failure_test) + # MSVC 7.1 does not support STL with exceptions disabled. + if (NOT MSVC OR MSVC_VERSION GREATER 1310) + cxx_executable(gtest_throw_on_failure_test_ test gtest_no_exception) + set_target_properties(gtest_throw_on_failure_test_ + PROPERTIES + COMPILE_FLAGS "${cxx_no_exception}") + py_test(gtest_throw_on_failure_test) + endif() cxx_executable(gtest_uninitialized_test_ test gtest) py_test(gtest_uninitialized_test) diff --git a/tests/gtest/CONTRIBUTORS b/tests/gtest/CONTRIBUTORS new file mode 100644 index 000000000..feae2fc04 --- /dev/null +++ b/tests/gtest/CONTRIBUTORS @@ -0,0 +1,37 @@ +# This file contains a list of people who've made non-trivial +# contribution to the Google C++ Testing Framework project. People +# who commit code to the project are encouraged to add their names +# here. Please keep the list sorted by first names. + +Ajay Joshi <jaj@google.com> +Balázs Dán <balazs.dan@gmail.com> +Bharat Mediratta <bharat@menalto.com> +Chandler Carruth <chandlerc@google.com> +Chris Prince <cprince@google.com> +Chris Taylor <taylorc@google.com> +Dan Egnor <egnor@google.com> +Eric Roman <eroman@chromium.org> +Hady Zalek <hady.zalek@gmail.com> +Jeffrey Yasskin <jyasskin@google.com> +Jói Sigurðsson <joi@google.com> +Keir Mierle <mierle@gmail.com> +Keith Ray <keith.ray@gmail.com> +Kenton Varda <kenton@google.com> +Manuel Klimek <klimek@google.com> +Markus Heule <markus.heule@gmail.com> +Mika Raento <mikie@iki.fi> +Miklós Fazekas <mfazekas@szemafor.com> +Pasi Valminen <pasi.valminen@gmail.com> +Patrick Hanna <phanna@google.com> +Patrick Riley <pfr@google.com> +Peter Kaminski <piotrk@google.com> +Preston Jackson <preston.a.jackson@gmail.com> +Rainer Klaffenboeck <rainer.klaffenboeck@dynatrace.com> +Russ Cox <rsc@google.com> +Russ Rufer <russ@pentad.com> +Sean Mcafee <eefacm@gmail.com> +Sigurður Ãsgeirsson <siggi@google.com> +Tracy Bialik <tracy@pentad.com> +Vadim Berman <vadimb@google.com> +Vlad Losev <vladl@google.com> +Zhanyong Wan <wan@google.com> diff --git a/tests/gtest/LICENSE b/tests/gtest/LICENSE new file mode 100644 index 000000000..1941a11f8 --- /dev/null +++ b/tests/gtest/LICENSE @@ -0,0 +1,28 @@ +Copyright 2008, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/tests/gtest/README b/tests/gtest/README new file mode 100644 index 000000000..404bf3b83 --- /dev/null +++ b/tests/gtest/README @@ -0,0 +1,435 @@ +Google C++ Testing Framework +============================ + +http://code.google.com/p/googletest/ + +Overview +-------- + +Google's framework for writing C++ tests on a variety of platforms +(Linux, Mac OS X, Windows, Windows CE, Symbian, etc). Based on the +xUnit architecture. Supports automatic test discovery, a rich set of +assertions, user-defined assertions, death tests, fatal and non-fatal +failures, various options for running the tests, and XML test report +generation. + +Please see the project page above for more information as well as the +mailing list for questions, discussions, and development. There is +also an IRC channel on OFTC (irc.oftc.net) #gtest available. Please +join us! + +Requirements for End Users +-------------------------- + +Google Test is designed to have fairly minimal requirements to build +and use with your projects, but there are some. Currently, we support +Linux, Windows, Mac OS X, and Cygwin. We will also make our best +effort to support other platforms (e.g. Solaris, AIX, and z/OS). +However, since core members of the Google Test project have no access +to these platforms, Google Test may have outstanding issues there. If +you notice any problems on your platform, please notify +googletestframework@googlegroups.com. Patches for fixing them are +even more welcome! + +### Linux Requirements ### + +These are the base requirements to build and use Google Test from a source +package (as described below): + * GNU-compatible Make or gmake + * POSIX-standard shell + * POSIX(-2) Regular Expressions (regex.h) + * A C++98-standard-compliant compiler + +### Windows Requirements ### + + * Microsoft Visual C++ 7.1 or newer + +### Cygwin Requirements ### + + * Cygwin 1.5.25-14 or newer + +### Mac OS X Requirements ### + + * Mac OS X 10.4 Tiger or newer + * Developer Tools Installed + +Also, you'll need CMake 2.6.4 or higher if you want to build the +samples using the provided CMake script, regardless of the platform. + +Requirements for Contributors +----------------------------- + +We welcome patches. If you plan to contribute a patch, you need to +build Google Test and its own tests from an SVN checkout (described +below), which has further requirements: + + * Python version 2.3 or newer (for running some of the tests and + re-generating certain source files from templates) + * CMake 2.6.4 or newer + +Getting the Source +------------------ + +There are two primary ways of getting Google Test's source code: you +can download a stable source release in your preferred archive format, +or directly check out the source from our Subversion (SVN) repository. +The SVN checkout requires a few extra steps and some extra software +packages on your system, but lets you track the latest development and +make patches much more easily, so we highly encourage it. + +### Source Package ### + +Google Test is released in versioned source packages which can be +downloaded from the download page [1]. Several different archive +formats are provided, but the only difference is the tools used to +manipulate them, and the size of the resulting file. Download +whichever you are most comfortable with. + + [1] http://code.google.com/p/googletest/downloads/list + +Once the package is downloaded, expand it using whichever tools you +prefer for that type. This will result in a new directory with the +name "gtest-X.Y.Z" which contains all of the source code. Here are +some examples on Linux: + + tar -xvzf gtest-X.Y.Z.tar.gz + tar -xvjf gtest-X.Y.Z.tar.bz2 + unzip gtest-X.Y.Z.zip + +### SVN Checkout ### + +To check out the main branch (also known as the "trunk") of Google +Test, run the following Subversion command: + + svn checkout http://googletest.googlecode.com/svn/trunk/ gtest-svn + +Setting up the Build +-------------------- + +To build Google Test and your tests that use it, you need to tell your +build system where to find its headers and source files. The exact +way to do it depends on which build system you use, and is usually +straightforward. + +### Generic Build Instructions ### + +Suppose you put Google Test in directory ${GTEST_DIR}. To build it, +create a library build target (or a project as called by Visual Studio +and Xcode) to compile + + ${GTEST_DIR}/src/gtest-all.cc + +with ${GTEST_DIR}/include in the system header search path and ${GTEST_DIR} +in the normal header search path. Assuming a Linux-like system and gcc, +something like the following will do: + + g++ -isystem ${GTEST_DIR}/include -I${GTEST_DIR} \ + -pthread -c ${GTEST_DIR}/src/gtest-all.cc + ar -rv libgtest.a gtest-all.o + +(We need -pthread as Google Test uses threads.) + +Next, you should compile your test source file with +${GTEST_DIR}/include in the system header search path, and link it +with gtest and any other necessary libraries: + + g++ -isystem ${GTEST_DIR}/include -pthread path/to/your_test.cc libgtest.a \ + -o your_test + +As an example, the make/ directory contains a Makefile that you can +use to build Google Test on systems where GNU make is available +(e.g. Linux, Mac OS X, and Cygwin). It doesn't try to build Google +Test's own tests. Instead, it just builds the Google Test library and +a sample test. You can use it as a starting point for your own build +script. + +If the default settings are correct for your environment, the +following commands should succeed: + + cd ${GTEST_DIR}/make + make + ./sample1_unittest + +If you see errors, try to tweak the contents of make/Makefile to make +them go away. There are instructions in make/Makefile on how to do +it. + +### Using CMake ### + +Google Test comes with a CMake build script (CMakeLists.txt) that can +be used on a wide range of platforms ("C" stands for cross-platform.). +If you don't have CMake installed already, you can download it for +free from http://www.cmake.org/. + +CMake works by generating native makefiles or build projects that can +be used in the compiler environment of your choice. The typical +workflow starts with: + + mkdir mybuild # Create a directory to hold the build output. + cd mybuild + cmake ${GTEST_DIR} # Generate native build scripts. + +If you want to build Google Test's samples, you should replace the +last command with + + cmake -Dgtest_build_samples=ON ${GTEST_DIR} + +If you are on a *nix system, you should now see a Makefile in the +current directory. Just type 'make' to build gtest. + +If you use Windows and have Visual Studio installed, a gtest.sln file +and several .vcproj files will be created. You can then build them +using Visual Studio. + +On Mac OS X with Xcode installed, a .xcodeproj file will be generated. + +### Legacy Build Scripts ### + +Before settling on CMake, we have been providing hand-maintained build +projects/scripts for Visual Studio, Xcode, and Autotools. While we +continue to provide them for convenience, they are not actively +maintained any more. We highly recommend that you follow the +instructions in the previous two sections to integrate Google Test +with your existing build system. + +If you still need to use the legacy build scripts, here's how: + +The msvc\ folder contains two solutions with Visual C++ projects. +Open the gtest.sln or gtest-md.sln file using Visual Studio, and you +are ready to build Google Test the same way you build any Visual +Studio project. Files that have names ending with -md use DLL +versions of Microsoft runtime libraries (the /MD or the /MDd compiler +option). Files without that suffix use static versions of the runtime +libraries (the /MT or the /MTd option). Please note that one must use +the same option to compile both gtest and the test code. If you use +Visual Studio 2005 or above, we recommend the -md version as /MD is +the default for new projects in these versions of Visual Studio. + +On Mac OS X, open the gtest.xcodeproj in the xcode/ folder using +Xcode. Build the "gtest" target. The universal binary framework will +end up in your selected build directory (selected in the Xcode +"Preferences..." -> "Building" pane and defaults to xcode/build). +Alternatively, at the command line, enter: + + xcodebuild + +This will build the "Release" configuration of gtest.framework in your +default build location. See the "xcodebuild" man page for more +information about building different configurations and building in +different locations. + +If you wish to use the Google Test Xcode project with Xcode 4.x and +above, you need to either: + * update the SDK configuration options in xcode/Config/General.xconfig. + Comment options SDKROOT, MACOS_DEPLOYMENT_TARGET, and GCC_VERSION. If + you choose this route you lose the ability to target earlier versions + of MacOS X. + * Install an SDK for an earlier version. This doesn't appear to be + supported by Apple, but has been reported to work + (http://stackoverflow.com/questions/5378518). + +Tweaking Google Test +-------------------- + +Google Test can be used in diverse environments. The default +configuration may not work (or may not work well) out of the box in +some environments. However, you can easily tweak Google Test by +defining control macros on the compiler command line. Generally, +these macros are named like GTEST_XYZ and you define them to either 1 +or 0 to enable or disable a certain feature. + +We list the most frequently used macros below. For a complete list, +see file include/gtest/internal/gtest-port.h. + +### Choosing a TR1 Tuple Library ### + +Some Google Test features require the C++ Technical Report 1 (TR1) +tuple library, which is not yet available with all compilers. The +good news is that Google Test implements a subset of TR1 tuple that's +enough for its own need, and will automatically use this when the +compiler doesn't provide TR1 tuple. + +Usually you don't need to care about which tuple library Google Test +uses. However, if your project already uses TR1 tuple, you need to +tell Google Test to use the same TR1 tuple library the rest of your +project uses, or the two tuple implementations will clash. To do +that, add + + -DGTEST_USE_OWN_TR1_TUPLE=0 + +to the compiler flags while compiling Google Test and your tests. If +you want to force Google Test to use its own tuple library, just add + + -DGTEST_USE_OWN_TR1_TUPLE=1 + +to the compiler flags instead. + +If you don't want Google Test to use tuple at all, add + + -DGTEST_HAS_TR1_TUPLE=0 + +and all features using tuple will be disabled. + +### Multi-threaded Tests ### + +Google Test is thread-safe where the pthread library is available. +After #include "gtest/gtest.h", you can check the GTEST_IS_THREADSAFE +macro to see whether this is the case (yes if the macro is #defined to +1, no if it's undefined.). + +If Google Test doesn't correctly detect whether pthread is available +in your environment, you can force it with + + -DGTEST_HAS_PTHREAD=1 + +or + + -DGTEST_HAS_PTHREAD=0 + +When Google Test uses pthread, you may need to add flags to your +compiler and/or linker to select the pthread library, or you'll get +link errors. If you use the CMake script or the deprecated Autotools +script, this is taken care of for you. If you use your own build +script, you'll need to read your compiler and linker's manual to +figure out what flags to add. + +### As a Shared Library (DLL) ### + +Google Test is compact, so most users can build and link it as a +static library for the simplicity. You can choose to use Google Test +as a shared library (known as a DLL on Windows) if you prefer. + +To compile *gtest* as a shared library, add + + -DGTEST_CREATE_SHARED_LIBRARY=1 + +to the compiler flags. You'll also need to tell the linker to produce +a shared library instead - consult your linker's manual for how to do +it. + +To compile your *tests* that use the gtest shared library, add + + -DGTEST_LINKED_AS_SHARED_LIBRARY=1 + +to the compiler flags. + +Note: while the above steps aren't technically necessary today when +using some compilers (e.g. GCC), they may become necessary in the +future, if we decide to improve the speed of loading the library (see +http://gcc.gnu.org/wiki/Visibility for details). Therefore you are +recommended to always add the above flags when using Google Test as a +shared library. Otherwise a future release of Google Test may break +your build script. + +### Avoiding Macro Name Clashes ### + +In C++, macros don't obey namespaces. Therefore two libraries that +both define a macro of the same name will clash if you #include both +definitions. In case a Google Test macro clashes with another +library, you can force Google Test to rename its macro to avoid the +conflict. + +Specifically, if both Google Test and some other code define macro +FOO, you can add + + -DGTEST_DONT_DEFINE_FOO=1 + +to the compiler flags to tell Google Test to change the macro's name +from FOO to GTEST_FOO. Currently FOO can be FAIL, SUCCEED, or TEST. +For example, with -DGTEST_DONT_DEFINE_TEST=1, you'll need to write + + GTEST_TEST(SomeTest, DoesThis) { ... } + +instead of + + TEST(SomeTest, DoesThis) { ... } + +in order to define a test. + +Upgrating from an Earlier Version +--------------------------------- + +We strive to keep Google Test releases backward compatible. +Sometimes, though, we have to make some breaking changes for the +users' long-term benefits. This section describes what you'll need to +do if you are upgrading from an earlier version of Google Test. + +### Upgrading from 1.3.0 or Earlier ### + +You may need to explicitly enable or disable Google Test's own TR1 +tuple library. See the instructions in section "Choosing a TR1 Tuple +Library". + +### Upgrading from 1.4.0 or Earlier ### + +The Autotools build script (configure + make) is no longer officially +supportted. You are encouraged to migrate to your own build system or +use CMake. If you still need to use Autotools, you can find +instructions in the README file from Google Test 1.4.0. + +On platforms where the pthread library is available, Google Test uses +it in order to be thread-safe. See the "Multi-threaded Tests" section +for what this means to your build script. + +If you use Microsoft Visual C++ 7.1 with exceptions disabled, Google +Test will no longer compile. This should affect very few people, as a +large portion of STL (including <string>) doesn't compile in this mode +anyway. We decided to stop supporting it in order to greatly simplify +Google Test's implementation. + +Developing Google Test +---------------------- + +This section discusses how to make your own changes to Google Test. + +### Testing Google Test Itself ### + +To make sure your changes work as intended and don't break existing +functionality, you'll want to compile and run Google Test's own tests. +For that you can use CMake: + + mkdir mybuild + cd mybuild + cmake -Dgtest_build_tests=ON ${GTEST_DIR} + +Make sure you have Python installed, as some of Google Test's tests +are written in Python. If the cmake command complains about not being +able to find Python ("Could NOT find PythonInterp (missing: +PYTHON_EXECUTABLE)"), try telling it explicitly where your Python +executable can be found: + + cmake -DPYTHON_EXECUTABLE=path/to/python -Dgtest_build_tests=ON ${GTEST_DIR} + +Next, you can build Google Test and all of its own tests. On *nix, +this is usually done by 'make'. To run the tests, do + + make test + +All tests should pass. + +### Regenerating Source Files ### + +Some of Google Test's source files are generated from templates (not +in the C++ sense) using a script. A template file is named FOO.pump, +where FOO is the name of the file it will generate. For example, the +file include/gtest/internal/gtest-type-util.h.pump is used to generate +gtest-type-util.h in the same directory. + +Normally you don't need to worry about regenerating the source files, +unless you need to modify them. In that case, you should modify the +corresponding .pump files instead and run the pump.py Python script to +regenerate them. You can find pump.py in the scripts/ directory. +Read the Pump manual [2] for how to use it. + + [2] http://code.google.com/p/googletest/wiki/PumpManual + +### Contributing a Patch ### + +We welcome patches. Please read the Google Test developer's guide [3] +for how you can contribute. In particular, make sure you have signed +the Contributor License Agreement, or we won't be able to accept the +patch. + + [3] http://code.google.com/p/googletest/wiki/GoogleTestDevGuide + +Happy testing! diff --git a/tests/gtest/cmake/internal_utils.cmake b/tests/gtest/cmake/internal_utils.cmake index 7efc2ac79..93e6dbb7c 100644 --- a/tests/gtest/cmake/internal_utils.cmake +++ b/tests/gtest/cmake/internal_utils.cmake @@ -37,7 +37,7 @@ macro(fix_default_compiler_settings_) # We prefer more strict warning checking for building Google Test. # Replaces /W3 with /W4 in defaults. - string(REPLACE "/W3" "-W4" ${flag_var} "${${flag_var}}") + string(REPLACE "/W3" "/W4" ${flag_var} "${${flag_var}}") endforeach() endif() endmacro() @@ -55,7 +55,32 @@ macro(config_compiler_and_linker) if (MSVC) # Newlines inside flags variables break CMake's NMake generator. # TODO(vladl@google.com): Add -RTCs and -RTCu to debug builds. - set(cxx_base_flags "-GS -W4 -WX -wd4127 -wd4251 -wd4275 -nologo -J -Zi") + set(cxx_base_flags "-GS -W4 -WX -wd4251 -wd4275 -nologo -J -Zi") + if (MSVC_VERSION LESS 1400) # 1400 is Visual Studio 2005 + # Suppress spurious warnings MSVC 7.1 sometimes issues. + # Forcing value to bool. + set(cxx_base_flags "${cxx_base_flags} -wd4800") + # Copy constructor and assignment operator could not be generated. + set(cxx_base_flags "${cxx_base_flags} -wd4511 -wd4512") + # Compatibility warnings not applicable to Google Test. + # Resolved overload was found by argument-dependent lookup. + set(cxx_base_flags "${cxx_base_flags} -wd4675") + endif() + if (MSVC_VERSION LESS 1500) # 1500 is Visual Studio 2008 + # Conditional expression is constant. + # When compiling with /W4, we get several instances of C4127 + # (Conditional expression is constant). In our code, we disable that + # warning on a case-by-case basis. However, on Visual Studio 2005, + # the warning fires on std::list. Therefore on that compiler and earlier, + # we disable the warning project-wide. + set(cxx_base_flags "${cxx_base_flags} -wd4127") + endif() + if (NOT (MSVC_VERSION LESS 1700)) # 1700 is Visual Studio 2012. + # Suppress "unreachable code" warning on VS 2012 and later. + # http://stackoverflow.com/questions/3232669 explains the issue. + set(cxx_base_flags "${cxx_base_flags} -wd4702") + endif() + set(cxx_base_flags "${cxx_base_flags} -D_UNICODE -DUNICODE -DWIN32 -D_WIN32") set(cxx_base_flags "${cxx_base_flags} -DSTRICT -DWIN32_LEAN_AND_MEAN") set(cxx_exception_flags "-EHsc -D_HAS_EXCEPTIONS=1") @@ -69,7 +94,8 @@ macro(config_compiler_and_linker) # whether RTTI is enabled. Therefore we define GTEST_HAS_RTTI # explicitly. set(cxx_no_rtti_flags "-fno-rtti -DGTEST_HAS_RTTI=0") - set(cxx_strict_flags "-Wextra") + set(cxx_strict_flags + "-Wextra -Wno-unused-parameter -Wno-missing-field-initializers") elseif (CMAKE_CXX_COMPILER_ID STREQUAL "SunPro") set(cxx_exception_flags "-features=except") # Sun Pro doesn't provide macros to indicate whether exceptions and diff --git a/tests/gtest/include/gtest/gtest-death-test.h b/tests/gtest/include/gtest/gtest-death-test.h index a27883f0a..957a69c6a 100644 --- a/tests/gtest/include/gtest/gtest-death-test.h +++ b/tests/gtest/include/gtest/gtest-death-test.h @@ -51,6 +51,17 @@ GTEST_DECLARE_string_(death_test_style); #if GTEST_HAS_DEATH_TEST +namespace internal { + +// Returns a Boolean value indicating whether the caller is currently +// executing in the context of the death test child process. Tools such as +// Valgrind heap checkers may need this to modify their behavior in death +// tests. IMPORTANT: This is an internal utility. Using it may break the +// implementation of death tests. User code MUST NOT use it. +GTEST_API_ bool InDeathTestChild(); + +} // namespace internal + // The following macros are useful for writing death tests. // Here's what happens when an ASSERT_DEATH* or EXPECT_DEATH* is @@ -75,7 +86,7 @@ GTEST_DECLARE_string_(death_test_style); // for (int i = 0; i < 5; i++) { // EXPECT_DEATH(server.ProcessRequest(i), // "Invalid request .* in ProcessRequest()") -// << "Failed to die on request " << i); +// << "Failed to die on request " << i; // } // // ASSERT_EXIT(server.ExitNow(), ::testing::ExitedWithCode(0), "Exiting"); @@ -245,10 +256,10 @@ class GTEST_API_ KilledBySignal { # ifdef NDEBUG # define EXPECT_DEBUG_DEATH(statement, regex) \ - do { statement; } while (::testing::internal::AlwaysFalse()) + GTEST_EXECUTE_STATEMENT_(statement, regex) # define ASSERT_DEBUG_DEATH(statement, regex) \ - do { statement; } while (::testing::internal::AlwaysFalse()) + GTEST_EXECUTE_STATEMENT_(statement, regex) # else diff --git a/tests/gtest/include/gtest/gtest-message.h b/tests/gtest/include/gtest/gtest-message.h index 9b7142f32..fe879bca7 100644 --- a/tests/gtest/include/gtest/gtest-message.h +++ b/tests/gtest/include/gtest/gtest-message.h @@ -48,8 +48,11 @@ #include <limits> -#include "gtest/internal/gtest-string.h" -#include "gtest/internal/gtest-internal.h" +#include "gtest/internal/gtest-port.h" + +// Ensures that there is at least one operator<< in the global namespace. +// See Message& operator<<(...) below for why. +void operator<<(const testing::internal::Secret&, int); namespace testing { @@ -87,15 +90,7 @@ class GTEST_API_ Message { public: // Constructs an empty Message. - // We allocate the stringstream separately because otherwise each use of - // ASSERT/EXPECT in a procedure adds over 200 bytes to the procedure's - // stack frame leading to huge stack frames in some cases; gcc does not reuse - // the stack space. - Message() : ss_(new ::std::stringstream) { - // By default, we want there to be enough precision when printing - // a double to a Message. - *ss_ << std::setprecision(std::numeric_limits<double>::digits10 + 2); - } + Message(); // Copy constructor. Message(const Message& msg) : ss_(new ::std::stringstream) { // NOLINT @@ -118,7 +113,22 @@ class GTEST_API_ Message { // Streams a non-pointer value to this object. template <typename T> inline Message& operator <<(const T& val) { - ::GTestStreamToHelper(ss_.get(), val); + // Some libraries overload << for STL containers. These + // overloads are defined in the global namespace instead of ::std. + // + // C++'s symbol lookup rule (i.e. Koenig lookup) says that these + // overloads are visible in either the std namespace or the global + // namespace, but not other namespaces, including the testing + // namespace which Google Test's Message class is in. + // + // To allow STL containers (and other types that has a << operator + // defined in the global namespace) to be used in Google Test + // assertions, testing::Message must access the custom << operator + // from the global namespace. With this using declaration, + // overloads of << defined in the global namespace and those + // visible via Koenig lookup are both exposed in this function. + using ::operator <<; + *ss_ << val; return *this; } @@ -140,7 +150,7 @@ class GTEST_API_ Message { if (pointer == NULL) { *ss_ << "(null)"; } else { - ::GTestStreamToHelper(ss_.get(), pointer); + *ss_ << pointer; } return *this; } @@ -164,12 +174,8 @@ class GTEST_API_ Message { // These two overloads allow streaming a wide C string to a Message // using the UTF-8 encoding. - Message& operator <<(const wchar_t* wide_c_str) { - return *this << internal::String::ShowWideCString(wide_c_str); - } - Message& operator <<(wchar_t* wide_c_str) { - return *this << internal::String::ShowWideCString(wide_c_str); - } + Message& operator <<(const wchar_t* wide_c_str); + Message& operator <<(wchar_t* wide_c_str); #if GTEST_HAS_STD_WSTRING // Converts the given wide string to a narrow string using the UTF-8 @@ -183,13 +189,11 @@ class GTEST_API_ Message { Message& operator <<(const ::wstring& wstr); #endif // GTEST_HAS_GLOBAL_WSTRING - // Gets the text streamed to this object so far as a String. + // Gets the text streamed to this object so far as an std::string. // Each '\0' character in the buffer is replaced with "\\0". // // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. - internal::String GetString() const { - return internal::StringStreamToString(ss_.get()); - } + std::string GetString() const; private: @@ -199,16 +203,20 @@ class GTEST_API_ Message { // decide between class template specializations for T and T*, so a // tr1::type_traits-like is_pointer works, and we can overload on that. template <typename T> - inline void StreamHelper(internal::true_type /*dummy*/, T* pointer) { + inline void StreamHelper(internal::true_type /*is_pointer*/, T* pointer) { if (pointer == NULL) { *ss_ << "(null)"; } else { - ::GTestStreamToHelper(ss_.get(), pointer); + *ss_ << pointer; } } template <typename T> - inline void StreamHelper(internal::false_type /*dummy*/, const T& value) { - ::GTestStreamToHelper(ss_.get(), value); + inline void StreamHelper(internal::false_type /*is_pointer*/, + const T& value) { + // See the comments in Message& operator <<(const T&) above for why + // we need this using statement. + using ::operator <<; + *ss_ << value; } #endif // GTEST_OS_SYMBIAN @@ -225,6 +233,18 @@ inline std::ostream& operator <<(std::ostream& os, const Message& sb) { return os << sb.GetString(); } +namespace internal { + +// Converts a streamable value to an std::string. A NULL pointer is +// converted to "(null)". When the input value is a ::string, +// ::std::string, ::wstring, or ::std::wstring object, each NUL +// character in it is replaced with "\\0". +template <typename T> +std::string StreamableToString(const T& streamable) { + return (Message() << streamable).GetString(); +} + +} // namespace internal } // namespace testing #endif // GTEST_INCLUDE_GTEST_GTEST_MESSAGE_H_ diff --git a/tests/gtest/include/gtest/gtest-param-test.h b/tests/gtest/include/gtest/gtest-param-test.h index 6407cfd68..d6702c8f1 100644 --- a/tests/gtest/include/gtest/gtest-param-test.h +++ b/tests/gtest/include/gtest/gtest-param-test.h @@ -1257,7 +1257,7 @@ inline internal::ParamGenerator<bool> Bool() { // Boolean flags: // // class FlagDependentTest -// : public testing::TestWithParam<tuple(bool, bool)> > { +// : public testing::TestWithParam<tuple<bool, bool> > { // virtual void SetUp() { // // Assigns external_flag_1 and external_flag_2 values from the tuple. // tie(external_flag_1, external_flag_2) = GetParam(); diff --git a/tests/gtest/include/gtest/gtest-param-test.h.pump b/tests/gtest/include/gtest/gtest-param-test.h.pump index 401cb513a..2dc9303b5 100644 --- a/tests/gtest/include/gtest/gtest-param-test.h.pump +++ b/tests/gtest/include/gtest/gtest-param-test.h.pump @@ -414,7 +414,7 @@ inline internal::ParamGenerator<bool> Bool() { // Boolean flags: // // class FlagDependentTest -// : public testing::TestWithParam<tuple(bool, bool)> > { +// : public testing::TestWithParam<tuple<bool, bool> > { // virtual void SetUp() { // // Assigns external_flag_1 and external_flag_2 values from the tuple. // tie(external_flag_1, external_flag_2) = GetParam(); diff --git a/tests/gtest/include/gtest/gtest-printers.h b/tests/gtest/include/gtest/gtest-printers.h index 9cbab3ff4..18ee7bc64 100644 --- a/tests/gtest/include/gtest/gtest-printers.h +++ b/tests/gtest/include/gtest/gtest-printers.h @@ -103,6 +103,10 @@ #include "gtest/internal/gtest-port.h" #include "gtest/internal/gtest-internal.h" +#if GTEST_HAS_STD_TUPLE_ +# include <tuple> +#endif + namespace testing { // Definitions in the 'internal' and 'internal2' name spaces are @@ -480,14 +484,16 @@ inline void PrintTo(const ::std::wstring& s, ::std::ostream* os) { } #endif // GTEST_HAS_STD_WSTRING -#if GTEST_HAS_TR1_TUPLE -// Overload for ::std::tr1::tuple. Needed for printing function arguments, -// which are packed as tuples. - +#if GTEST_HAS_TR1_TUPLE || GTEST_HAS_STD_TUPLE_ // Helper function for printing a tuple. T must be instantiated with // a tuple type. template <typename T> void PrintTupleTo(const T& t, ::std::ostream* os); +#endif // GTEST_HAS_TR1_TUPLE || GTEST_HAS_STD_TUPLE_ + +#if GTEST_HAS_TR1_TUPLE +// Overload for ::std::tr1::tuple. Needed for printing function arguments, +// which are packed as tuples. // Overloaded PrintTo() for tuples of various arities. We support // tuples of up-to 10 fields. The following implementation works @@ -561,6 +567,13 @@ void PrintTo( } #endif // GTEST_HAS_TR1_TUPLE +#if GTEST_HAS_STD_TUPLE_ +template <typename... Types> +void PrintTo(const ::std::tuple<Types...>& t, ::std::ostream* os) { + PrintTupleTo(t, os); +} +#endif // GTEST_HAS_STD_TUPLE_ + // Overload for std::pair. template <typename T1, typename T2> void PrintTo(const ::std::pair<T1, T2>& value, ::std::ostream* os) { @@ -580,10 +593,7 @@ class UniversalPrinter { public: // MSVC warns about adding const to a function type, so we want to // disable the warning. -#ifdef _MSC_VER -# pragma warning(push) // Saves the current warning state. -# pragma warning(disable:4180) // Temporarily disables warning 4180. -#endif // _MSC_VER + GTEST_DISABLE_MSC_WARNINGS_PUSH_(4180) // Note: we deliberately don't call this PrintTo(), as that name // conflicts with ::testing::internal::PrintTo in the body of the @@ -600,9 +610,7 @@ class UniversalPrinter { PrintTo(value, os); } -#ifdef _MSC_VER -# pragma warning(pop) // Restores the warning state. -#endif // _MSC_VER + GTEST_DISABLE_MSC_WARNINGS_POP_() }; // UniversalPrintArray(begin, len, os) prints an array of 'len' @@ -630,9 +638,12 @@ void UniversalPrintArray(const T* begin, size_t len, ::std::ostream* os) { } } // This overload prints a (const) char array compactly. -GTEST_API_ void UniversalPrintArray(const char* begin, - size_t len, - ::std::ostream* os); +GTEST_API_ void UniversalPrintArray( + const char* begin, size_t len, ::std::ostream* os); + +// This overload prints a (const) wchar_t array compactly. +GTEST_API_ void UniversalPrintArray( + const wchar_t* begin, size_t len, ::std::ostream* os); // Implements printing an array type T[N]. template <typename T, size_t N> @@ -651,10 +662,7 @@ class UniversalPrinter<T&> { public: // MSVC warns about adding const to a function type, so we want to // disable the warning. -#ifdef _MSC_VER -# pragma warning(push) // Saves the current warning state. -# pragma warning(disable:4180) // Temporarily disables warning 4180. -#endif // _MSC_VER + GTEST_DISABLE_MSC_WARNINGS_PUSH_(4180) static void Print(const T& value, ::std::ostream* os) { // Prints the address of the value. We use reinterpret_cast here @@ -665,27 +673,78 @@ class UniversalPrinter<T&> { UniversalPrint(value, os); } -#ifdef _MSC_VER -# pragma warning(pop) // Restores the warning state. -#endif // _MSC_VER + GTEST_DISABLE_MSC_WARNINGS_POP_() }; // Prints a value tersely: for a reference type, the referenced value // (but not the address) is printed; for a (const) char pointer, the // NUL-terminated string (but not the pointer) is printed. + template <typename T> -void UniversalTersePrint(const T& value, ::std::ostream* os) { - UniversalPrint(value, os); -} -inline void UniversalTersePrint(const char* str, ::std::ostream* os) { - if (str == NULL) { - *os << "NULL"; - } else { - UniversalPrint(string(str), os); +class UniversalTersePrinter { + public: + static void Print(const T& value, ::std::ostream* os) { + UniversalPrint(value, os); } -} -inline void UniversalTersePrint(char* str, ::std::ostream* os) { - UniversalTersePrint(static_cast<const char*>(str), os); +}; +template <typename T> +class UniversalTersePrinter<T&> { + public: + static void Print(const T& value, ::std::ostream* os) { + UniversalPrint(value, os); + } +}; +template <typename T, size_t N> +class UniversalTersePrinter<T[N]> { + public: + static void Print(const T (&value)[N], ::std::ostream* os) { + UniversalPrinter<T[N]>::Print(value, os); + } +}; +template <> +class UniversalTersePrinter<const char*> { + public: + static void Print(const char* str, ::std::ostream* os) { + if (str == NULL) { + *os << "NULL"; + } else { + UniversalPrint(string(str), os); + } + } +}; +template <> +class UniversalTersePrinter<char*> { + public: + static void Print(char* str, ::std::ostream* os) { + UniversalTersePrinter<const char*>::Print(str, os); + } +}; + +#if GTEST_HAS_STD_WSTRING +template <> +class UniversalTersePrinter<const wchar_t*> { + public: + static void Print(const wchar_t* str, ::std::ostream* os) { + if (str == NULL) { + *os << "NULL"; + } else { + UniversalPrint(::std::wstring(str), os); + } + } +}; +#endif + +template <> +class UniversalTersePrinter<wchar_t*> { + public: + static void Print(wchar_t* str, ::std::ostream* os) { + UniversalTersePrinter<const wchar_t*>::Print(str, os); + } +}; + +template <typename T> +void UniversalTersePrint(const T& value, ::std::ostream* os) { + UniversalTersePrinter<T>::Print(value, os); } // Prints a value using the type inferred by the compiler. The @@ -694,19 +753,71 @@ inline void UniversalTersePrint(char* str, ::std::ostream* os) { // NUL-terminated string. template <typename T> void UniversalPrint(const T& value, ::std::ostream* os) { - UniversalPrinter<T>::Print(value, os); + // A workarond for the bug in VC++ 7.1 that prevents us from instantiating + // UniversalPrinter with T directly. + typedef T T1; + UniversalPrinter<T1>::Print(value, os); } -#if GTEST_HAS_TR1_TUPLE typedef ::std::vector<string> Strings; +// TuplePolicy<TupleT> must provide: +// - tuple_size +// size of tuple TupleT. +// - get<size_t I>(const TupleT& t) +// static function extracting element I of tuple TupleT. +// - tuple_element<size_t I>::type +// type of element I of tuple TupleT. +template <typename TupleT> +struct TuplePolicy; + +#if GTEST_HAS_TR1_TUPLE +template <typename TupleT> +struct TuplePolicy { + typedef TupleT Tuple; + static const size_t tuple_size = ::std::tr1::tuple_size<Tuple>::value; + + template <size_t I> + struct tuple_element : ::std::tr1::tuple_element<I, Tuple> {}; + + template <size_t I> + static typename AddReference< + const typename ::std::tr1::tuple_element<I, Tuple>::type>::type get( + const Tuple& tuple) { + return ::std::tr1::get<I>(tuple); + } +}; +template <typename TupleT> +const size_t TuplePolicy<TupleT>::tuple_size; +#endif // GTEST_HAS_TR1_TUPLE + +#if GTEST_HAS_STD_TUPLE_ +template <typename... Types> +struct TuplePolicy< ::std::tuple<Types...> > { + typedef ::std::tuple<Types...> Tuple; + static const size_t tuple_size = ::std::tuple_size<Tuple>::value; + + template <size_t I> + struct tuple_element : ::std::tuple_element<I, Tuple> {}; + + template <size_t I> + static const typename ::std::tuple_element<I, Tuple>::type& get( + const Tuple& tuple) { + return ::std::get<I>(tuple); + } +}; +template <typename... Types> +const size_t TuplePolicy< ::std::tuple<Types...> >::tuple_size; +#endif // GTEST_HAS_STD_TUPLE_ + +#if GTEST_HAS_TR1_TUPLE || GTEST_HAS_STD_TUPLE_ // This helper template allows PrintTo() for tuples and // UniversalTersePrintTupleFieldsToStrings() to be defined by // induction on the number of tuple fields. The idea is that // TuplePrefixPrinter<N>::PrintPrefixTo(t, os) prints the first N // fields in tuple t, and can be defined in terms of // TuplePrefixPrinter<N - 1>. - +// // The inductive case. template <size_t N> struct TuplePrefixPrinter { @@ -714,9 +825,14 @@ struct TuplePrefixPrinter { template <typename Tuple> static void PrintPrefixTo(const Tuple& t, ::std::ostream* os) { TuplePrefixPrinter<N - 1>::PrintPrefixTo(t, os); - *os << ", "; - UniversalPrinter<typename ::std::tr1::tuple_element<N - 1, Tuple>::type> - ::Print(::std::tr1::get<N - 1>(t), os); + GTEST_INTENTIONAL_CONST_COND_PUSH_() + if (N > 1) { + GTEST_INTENTIONAL_CONST_COND_POP_() + *os << ", "; + } + UniversalPrinter< + typename TuplePolicy<Tuple>::template tuple_element<N - 1>::type> + ::Print(TuplePolicy<Tuple>::template get<N - 1>(t), os); } // Tersely prints the first N fields of a tuple to a string vector, @@ -725,12 +841,12 @@ struct TuplePrefixPrinter { static void TersePrintPrefixToStrings(const Tuple& t, Strings* strings) { TuplePrefixPrinter<N - 1>::TersePrintPrefixToStrings(t, strings); ::std::stringstream ss; - UniversalTersePrint(::std::tr1::get<N - 1>(t), &ss); + UniversalTersePrint(TuplePolicy<Tuple>::template get<N - 1>(t), &ss); strings->push_back(ss.str()); } }; -// Base cases. +// Base case. template <> struct TuplePrefixPrinter<0> { template <typename Tuple> @@ -739,34 +855,13 @@ struct TuplePrefixPrinter<0> { template <typename Tuple> static void TersePrintPrefixToStrings(const Tuple&, Strings*) {} }; -// We have to specialize the entire TuplePrefixPrinter<> class -// template here, even though the definition of -// TersePrintPrefixToStrings() is the same as the generic version, as -// Embarcadero (formerly CodeGear, formerly Borland) C++ doesn't -// support specializing a method template of a class template. -template <> -struct TuplePrefixPrinter<1> { - template <typename Tuple> - static void PrintPrefixTo(const Tuple& t, ::std::ostream* os) { - UniversalPrinter<typename ::std::tr1::tuple_element<0, Tuple>::type>:: - Print(::std::tr1::get<0>(t), os); - } - template <typename Tuple> - static void TersePrintPrefixToStrings(const Tuple& t, Strings* strings) { - ::std::stringstream ss; - UniversalTersePrint(::std::tr1::get<0>(t), &ss); - strings->push_back(ss.str()); - } -}; - -// Helper function for printing a tuple. T must be instantiated with -// a tuple type. -template <typename T> -void PrintTupleTo(const T& t, ::std::ostream* os) { +// Helper function for printing a tuple. +// Tuple must be either std::tr1::tuple or std::tuple type. +template <typename Tuple> +void PrintTupleTo(const Tuple& t, ::std::ostream* os) { *os << "("; - TuplePrefixPrinter< ::std::tr1::tuple_size<T>::value>:: - PrintPrefixTo(t, os); + TuplePrefixPrinter<TuplePolicy<Tuple>::tuple_size>::PrintPrefixTo(t, os); *os << ")"; } @@ -776,18 +871,18 @@ void PrintTupleTo(const T& t, ::std::ostream* os) { template <typename Tuple> Strings UniversalTersePrintTupleFieldsToStrings(const Tuple& value) { Strings result; - TuplePrefixPrinter< ::std::tr1::tuple_size<Tuple>::value>:: + TuplePrefixPrinter<TuplePolicy<Tuple>::tuple_size>:: TersePrintPrefixToStrings(value, &result); return result; } -#endif // GTEST_HAS_TR1_TUPLE +#endif // GTEST_HAS_TR1_TUPLE || GTEST_HAS_STD_TUPLE_ } // namespace internal template <typename T> ::std::string PrintToString(const T& value) { ::std::stringstream ss; - internal::UniversalTersePrint(value, &ss); + internal::UniversalTersePrinter<T>::Print(value, &ss); return ss.str(); } diff --git a/tests/gtest/include/gtest/gtest-spi.h b/tests/gtest/include/gtest/gtest-spi.h index b226e5504..f63fa9a1b 100644 --- a/tests/gtest/include/gtest/gtest-spi.h +++ b/tests/gtest/include/gtest/gtest-spi.h @@ -223,7 +223,7 @@ class GTEST_API_ SingleFailureChecker { (substr));\ {\ ::testing::ScopedFakeTestPartResultReporter gtest_reporter(\ - ::testing::ScopedFakeTestPartResultReporter::INTERCEPT_ALL_THREADS,\ + ::testing::ScopedFakeTestPartResultReporter::INTERCEPT_ALL_THREADS, \ >est_failures);\ if (::testing::internal::AlwaysTrue()) { statement; }\ }\ diff --git a/tests/gtest/include/gtest/gtest-test-part.h b/tests/gtest/include/gtest/gtest-test-part.h index 8aeea1498..77eb84483 100644 --- a/tests/gtest/include/gtest/gtest-test-part.h +++ b/tests/gtest/include/gtest/gtest-test-part.h @@ -62,7 +62,7 @@ class GTEST_API_ TestPartResult { int a_line_number, const char* a_message) : type_(a_type), - file_name_(a_file_name), + file_name_(a_file_name == NULL ? "" : a_file_name), line_number_(a_line_number), summary_(ExtractSummary(a_message)), message_(a_message) { @@ -73,7 +73,9 @@ class GTEST_API_ TestPartResult { // Gets the name of the source file where the test part took place, or // NULL if it's unknown. - const char* file_name() const { return file_name_.c_str(); } + const char* file_name() const { + return file_name_.empty() ? NULL : file_name_.c_str(); + } // Gets the line in the source file where the test part took place, // or -1 if it's unknown. @@ -96,21 +98,22 @@ class GTEST_API_ TestPartResult { // Returns true iff the test part fatally failed. bool fatally_failed() const { return type_ == kFatalFailure; } + private: Type type_; // Gets the summary of the failure message by omitting the stack // trace in it. - static internal::String ExtractSummary(const char* message); + static std::string ExtractSummary(const char* message); // The name of the source file where the test part took place, or - // NULL if the source file is unknown. - internal::String file_name_; + // "" if the source file is unknown. + std::string file_name_; // The line in the source file where the test part took place, or -1 // if the line number is unknown. int line_number_; - internal::String summary_; // The test failure summary. - internal::String message_; // The test failure message. + std::string summary_; // The test failure summary. + std::string message_; // The test failure message. }; // Prints a TestPartResult object. diff --git a/tests/gtest/include/gtest/gtest.h b/tests/gtest/include/gtest/gtest.h index 1e5154715..71d552e1a 100644 --- a/tests/gtest/include/gtest/gtest.h +++ b/tests/gtest/include/gtest/gtest.h @@ -52,6 +52,7 @@ #define GTEST_INCLUDE_GTEST_GTEST_H_ #include <limits> +#include <ostream> #include <vector> #include "gtest/internal/gtest-internal.h" @@ -69,14 +70,14 @@ // class ::string, which has the same interface as ::std::string, but // has a different implementation. // -// The user can define GTEST_HAS_GLOBAL_STRING to 1 to indicate that +// You can define GTEST_HAS_GLOBAL_STRING to 1 to indicate that // ::string is available AND is a distinct type to ::std::string, or // define it to 0 to indicate otherwise. // -// If the user's ::std::string and ::string are the same class due to -// aliasing, he should define GTEST_HAS_GLOBAL_STRING to 0. +// If ::std::string and ::string are the same class on your platform +// due to aliasing, you should define GTEST_HAS_GLOBAL_STRING to 0. // -// If the user doesn't define GTEST_HAS_GLOBAL_STRING, it is defined +// If you do not define GTEST_HAS_GLOBAL_STRING, it is defined // heuristically. namespace testing { @@ -153,25 +154,15 @@ class ExecDeathTest; class NoExecDeathTest; class FinalSuccessChecker; class GTestFlagSaver; +class StreamingListenerTest; class TestResultAccessor; class TestEventListenersAccessor; class TestEventRepeater; +class UnitTestRecordPropertyTestHelper; class WindowsDeathTest; class UnitTestImpl* GetUnitTestImpl(); void ReportFailureInUnknownLocation(TestPartResult::Type result_type, - const String& message); - -// Converts a streamable value to a String. A NULL pointer is -// converted to "(null)". When the input value is a ::string, -// ::std::string, ::wstring, or ::std::wstring object, each NUL -// character in it is replaced with "\\0". -// Declared in gtest-internal.h but defined here, so that it has access -// to the definition of the Message class, required by the ARM -// compiler. -template <typename T> -String StreamableToString(const T& streamable) { - return (Message() << streamable).GetString(); -} + const std::string& message); } // namespace internal @@ -267,8 +258,31 @@ class GTEST_API_ AssertionResult { // Copy constructor. // Used in EXPECT_TRUE/FALSE(assertion_result). AssertionResult(const AssertionResult& other); + + GTEST_DISABLE_MSC_WARNINGS_PUSH_(4800 /* forcing value to bool */) + // Used in the EXPECT_TRUE/FALSE(bool_expression). - explicit AssertionResult(bool success) : success_(success) {} + // + // T must be contextually convertible to bool. + // + // The second parameter prevents this overload from being considered if + // the argument is implicitly convertible to AssertionResult. In that case + // we want AssertionResult's copy constructor to be used. + template <typename T> + explicit AssertionResult( + const T& success, + typename internal::EnableIf< + !internal::ImplicitlyConvertible<T, AssertionResult>::value>::type* + /*enabler*/ = NULL) + : success_(success) {} + + GTEST_DISABLE_MSC_WARNINGS_POP_() + + // Assignment operator. + AssertionResult& operator=(AssertionResult other) { + swap(other); + return *this; + } // Returns true iff the assertion succeeded. operator bool() const { return success_; } // NOLINT @@ -309,6 +323,9 @@ class GTEST_API_ AssertionResult { message_->append(a_message.GetString().c_str()); } + // Swap the contents of this AssertionResult with other. + void swap(AssertionResult& other); + // Stores result of the assertion predicate. bool success_; // Stores the message describing the condition in case the expectation @@ -316,8 +333,6 @@ class GTEST_API_ AssertionResult { // Referenced via a pointer to avoid taking too much stack frame space // with test assertions. internal::scoped_ptr< ::std::string> message_; - - GTEST_DISALLOW_ASSIGN_(AssertionResult); }; // Makes a successful assertion result. @@ -344,8 +359,8 @@ GTEST_API_ AssertionResult AssertionFailure(const Message& msg); // // class FooTest : public testing::Test { // protected: -// virtual void SetUp() { ... } -// virtual void TearDown() { ... } +// void SetUp() override { ... } +// void TearDown() override { ... } // ... // }; // @@ -391,20 +406,21 @@ class GTEST_API_ Test { // non-fatal) failure. static bool HasFailure() { return HasFatalFailure() || HasNonfatalFailure(); } - // Logs a property for the current test. Only the last value for a given - // key is remembered. - // These are public static so they can be called from utility functions - // that are not members of the test fixture. - // The arguments are const char* instead strings, as Google Test is used - // on platforms where string doesn't compile. - // - // Note that a driving consideration for these RecordProperty methods - // was to produce xml output suited to the Greenspan charting utility, - // which at present will only chart values that fit in a 32-bit int. It - // is the user's responsibility to restrict their values to 32-bit ints - // if they intend them to be used with Greenspan. - static void RecordProperty(const char* key, const char* value); - static void RecordProperty(const char* key, int value); + // Logs a property for the current test, test case, or for the entire + // invocation of the test program when used outside of the context of a + // test case. Only the last value for a given key is remembered. These + // are public static so they can be called from utility functions that are + // not members of the test fixture. Calls to RecordProperty made during + // lifespan of the test (from the moment its constructor starts to the + // moment its destructor finishes) will be output in XML as attributes of + // the <testcase> element. Properties recorded from fixture's + // SetUpTestCase or TearDownTestCase are logged as attributes of the + // corresponding <testsuite> element. Calls to RecordProperty made in the + // global context (before or after invocation of RUN_ALL_TESTS and from + // SetUp/TearDown method of Environment objects registered with Google + // Test) will be output as attributes of the <testsuites> element. + static void RecordProperty(const std::string& key, const std::string& value); + static void RecordProperty(const std::string& key, int value); protected: // Creates a Test object. @@ -439,17 +455,17 @@ class GTEST_API_ Test { // Uses a GTestFlagSaver to save and restore all Google Test flags. const internal::GTestFlagSaver* const gtest_flag_saver_; - // Often a user mis-spells SetUp() as Setup() and spends a long time + // Often a user misspells SetUp() as Setup() and spends a long time // wondering why it is never called by Google Test. The declaration of // the following method is solely for catching such an error at // compile time: // // - The return type is deliberately chosen to be not void, so it - // will be a conflict if a user declares void Setup() in his test - // fixture. + // will be a conflict if void Setup() is declared in the user's + // test fixture. // // - This method is private, so it will be another compiler error - // if a user calls it from his test fixture. + // if the method is called from the user's test fixture. // // DO NOT OVERRIDE THIS FUNCTION. // @@ -473,7 +489,7 @@ class TestProperty { // C'tor. TestProperty does NOT have a default constructor. // Always use this constructor (with parameters) to create a // TestProperty object. - TestProperty(const char* a_key, const char* a_value) : + TestProperty(const std::string& a_key, const std::string& a_value) : key_(a_key), value_(a_value) { } @@ -488,15 +504,15 @@ class TestProperty { } // Sets a new value, overriding the one supplied in the constructor. - void SetValue(const char* new_value) { + void SetValue(const std::string& new_value) { value_ = new_value; } private: // The key supplied by the user. - internal::String key_; + std::string key_; // The value supplied by the user. - internal::String value_; + std::string value_; }; // The result of a single Test. This includes a list of @@ -547,6 +563,7 @@ class GTEST_API_ TestResult { private: friend class TestInfo; + friend class TestCase; friend class UnitTest; friend class internal::DefaultGlobalTestPartResultReporter; friend class internal::ExecDeathTest; @@ -571,13 +588,16 @@ class GTEST_API_ TestResult { // a non-fatal failure if invalid (e.g., if it conflicts with reserved // key names). If a property is already recorded for the same key, the // value will be updated, rather than storing multiple values for the same - // key. - void RecordProperty(const TestProperty& test_property); + // key. xml_element specifies the element for which the property is being + // recorded and is used for validation. + void RecordProperty(const std::string& xml_element, + const TestProperty& test_property); // Adds a failure if the key is a reserved attribute of Google Test // testcase tags. Returns true if the property is valid. // TODO(russr): Validate attribute names are legal and human readable. - static bool ValidateTestProperty(const TestProperty& test_property); + static bool ValidateTestProperty(const std::string& xml_element, + const TestProperty& test_property); // Adds a test part result to the list. void AddTestPartResult(const TestPartResult& test_part_result); @@ -650,9 +670,9 @@ class GTEST_API_ TestInfo { return NULL; } - // Returns true if this test should run, that is if the test is not disabled - // (or it is disabled but the also_run_disabled_tests flag has been specified) - // and its full name matches the user-specified filter. + // Returns true if this test should run, that is if the test is not + // disabled (or it is disabled but the also_run_disabled_tests flag has + // been specified) and its full name matches the user-specified filter. // // Google Test allows the user to filter the tests by their full names. // The full name of a test Bar in test case Foo is defined as @@ -668,22 +688,28 @@ class GTEST_API_ TestInfo { // contains the character 'A' or starts with "Foo.". bool should_run() const { return should_run_; } - // Returns true if the test was filtered out by --gtest_filter - bool filtered_out() const { return !matches_filter_; } + // Returns true iff this test will appear in the XML report. + bool is_reportable() const { + // For now, the XML report includes all tests matching the filter. + // In the future, we may trim tests that are excluded because of + // sharding. + return matches_filter_; + } // Returns the result of the test. const TestResult* result() const { return &result_; } private: - #if GTEST_HAS_DEATH_TEST friend class internal::DefaultDeathTestFactory; #endif // GTEST_HAS_DEATH_TEST friend class Test; friend class TestCase; friend class internal::UnitTestImpl; + friend class internal::StreamingListenerTest; friend TestInfo* internal::MakeAndRegisterTestInfo( - const char* test_case_name, const char* name, + const char* test_case_name, + const char* name, const char* type_param, const char* value_param, internal::TypeId fixture_class_id, @@ -693,9 +719,10 @@ class GTEST_API_ TestInfo { // Constructs a TestInfo object. The newly constructed instance assumes // ownership of the factory object. - TestInfo(const char* test_case_name, const char* name, - const char* a_type_param, - const char* a_value_param, + TestInfo(const std::string& test_case_name, + const std::string& name, + const char* a_type_param, // NULL if not a type-parameterized test + const char* a_value_param, // NULL if not a value-parameterized test internal::TypeId fixture_class_id, internal::TestFactoryBase* factory); @@ -775,18 +802,21 @@ class GTEST_API_ TestCase { // Returns true if any test in this test case should run. bool should_run() const { return should_run_; } - // Returns true if this test case should be skipped in the report. - bool should_skip_report() const { return should_skip_report_; } - // Gets the number of successful tests in this test case. int successful_test_count() const; // Gets the number of failed tests in this test case. int failed_test_count() const; + // Gets the number of disabled tests that will be reported in the XML report. + int reportable_disabled_test_count() const; + // Gets the number of disabled tests in this test case. int disabled_test_count() const; + // Gets the number of tests to be printed in the XML report. + int reportable_test_count() const; + // Get the number of tests in this test case that should run. int test_to_run_count() const; @@ -806,6 +836,10 @@ class GTEST_API_ TestCase { // total_test_count() - 1. If i is not in that range, returns NULL. const TestInfo* GetTestInfo(int i) const; + // Returns the TestResult that holds test properties recorded during + // execution of SetUpTestCase and TearDownTestCase. + const TestResult& ad_hoc_test_result() const { return ad_hoc_test_result_; } + private: friend class Test; friend class internal::UnitTestImpl; @@ -824,7 +858,6 @@ class GTEST_API_ TestCase { // Sets the should_run member. void set_should_run(bool should) { should_run_ = should; } - void set_should_skip_report(bool should) { should_skip_report_ = should; } // Adds a TestInfo to this test case. Will delete the TestInfo upon // destruction of the TestCase object. @@ -859,11 +892,22 @@ class GTEST_API_ TestCase { return test_info->should_run() && test_info->result()->Failed(); } + // Returns true iff the test is disabled and will be reported in the XML + // report. + static bool TestReportableDisabled(const TestInfo* test_info) { + return test_info->is_reportable() && test_info->is_disabled_; + } + // Returns true iff test is disabled. static bool TestDisabled(const TestInfo* test_info) { return test_info->is_disabled_; } + // Returns true iff this test will appear in the XML report. + static bool TestReportable(const TestInfo* test_info) { + return test_info->is_reportable(); + } + // Returns true if the given test should run. static bool ShouldRunTest(const TestInfo* test_info) { return test_info->should_run(); @@ -876,7 +920,7 @@ class GTEST_API_ TestCase { void UnshuffleTests(); // Name of the test case. - internal::String name_; + std::string name_; // Name of the parameter type, or NULL if this is not a typed or a // type-parameterized test. const internal::scoped_ptr<const ::std::string> type_param_; @@ -893,17 +937,18 @@ class GTEST_API_ TestCase { Test::TearDownTestCaseFunc tear_down_tc_; // True iff any test in this test case should run. bool should_run_; - // True if this test case should not be reported - bool should_skip_report_; // Elapsed time, in milliseconds. TimeInMillis elapsed_time_; + // Holds test properties recorded during execution of SetUpTestCase and + // TearDownTestCase. + TestResult ad_hoc_test_result_; // We disallow copying TestCases. GTEST_DISALLOW_COPY_AND_ASSIGN_(TestCase); }; // An Environment object is capable of setting up and tearing down an -// environment. The user should subclass this to define his own +// environment. You should subclass this to define your own // environment(s). // // An Environment object does the set-up and tear-down in virtual @@ -1116,11 +1161,13 @@ class GTEST_API_ UnitTest { // Returns the TestCase object for the test that's currently running, // or NULL if no test is running. - const TestCase* current_test_case() const; + const TestCase* current_test_case() const + GTEST_LOCK_EXCLUDED_(mutex_); // Returns the TestInfo object for the test that's currently running, // or NULL if no test is running. - const TestInfo* current_test_info() const; + const TestInfo* current_test_info() const + GTEST_LOCK_EXCLUDED_(mutex_); // Returns the random seed used at the start of the current test run. int random_seed() const; @@ -1130,7 +1177,8 @@ class GTEST_API_ UnitTest { // value-parameterized tests and instantiate and register them. // // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. - internal::ParameterizedTestCaseRegistry& parameterized_test_registry(); + internal::ParameterizedTestCaseRegistry& parameterized_test_registry() + GTEST_LOCK_EXCLUDED_(mutex_); #endif // GTEST_HAS_PARAM_TEST // Gets the number of successful test cases. @@ -1152,15 +1200,25 @@ class GTEST_API_ UnitTest { // Gets the number of failed tests. int failed_test_count() const; + // Gets the number of disabled tests that will be reported in the XML report. + int reportable_disabled_test_count() const; + // Gets the number of disabled tests. int disabled_test_count() const; + // Gets the number of tests to be printed in the XML report. + int reportable_test_count() const; + // Gets the number of all tests. int total_test_count() const; // Gets the number of tests that should run. int test_to_run_count() const; + // Gets the time of the test program start, in ms from the start of the + // UNIX epoch. + TimeInMillis start_timestamp() const; + // Gets the elapsed time, in milliseconds. TimeInMillis elapsed_time() const; @@ -1175,6 +1233,10 @@ class GTEST_API_ UnitTest { // total_test_case_count() - 1. If i is not in that range, returns NULL. const TestCase* GetTestCase(int i) const; + // Returns the TestResult containing information on test failures and + // properties logged outside of individual test cases. + const TestResult& ad_hoc_test_result() const; + // Returns the list of event listeners that can be used to track events // inside Google Test. TestEventListeners& listeners(); @@ -1198,12 +1260,16 @@ class GTEST_API_ UnitTest { void AddTestPartResult(TestPartResult::Type result_type, const char* file_name, int line_number, - const internal::String& message, - const internal::String& os_stack_trace); + const std::string& message, + const std::string& os_stack_trace) + GTEST_LOCK_EXCLUDED_(mutex_); - // Adds a TestProperty to the current TestResult object. If the result already - // contains a property with the same key, the value will be updated. - void RecordPropertyForCurrentTest(const char* key, const char* value); + // Adds a TestProperty to the current TestResult object when invoked from + // inside a test, to current TestCase's ad_hoc_test_result_ when invoked + // from SetUpTestCase or TearDownTestCase, or to the global property set + // when invoked elsewhere. If the result already contains a property with + // the same key, the value will be updated. + void RecordProperty(const std::string& key, const std::string& value); // Gets the i-th test case among all the test cases. i can range from 0 to // total_test_case_count() - 1. If i is not in that range, returns NULL. @@ -1218,11 +1284,13 @@ class GTEST_API_ UnitTest { friend class Test; friend class internal::AssertHelper; friend class internal::ScopedTrace; + friend class internal::StreamingListenerTest; + friend class internal::UnitTestRecordPropertyTestHelper; friend Environment* AddGlobalTestEnvironment(Environment* env); friend internal::UnitTestImpl* internal::GetUnitTestImpl(); friend void internal::ReportFailureInUnknownLocation( TestPartResult::Type result_type, - const internal::String& message); + const std::string& message); // Creates an empty UnitTest. UnitTest(); @@ -1232,10 +1300,12 @@ class GTEST_API_ UnitTest { // Pushes a trace defined by SCOPED_TRACE() on to the per-thread // Google Test trace stack. - void PushGTestTrace(const internal::TraceInfo& trace); + void PushGTestTrace(const internal::TraceInfo& trace) + GTEST_LOCK_EXCLUDED_(mutex_); // Pops a trace from the per-thread Google Test trace stack. - void PopGTestTrace(); + void PopGTestTrace() + GTEST_LOCK_EXCLUDED_(mutex_); // Protects mutable state in *impl_. This is mutable as some const // methods need to lock it too. @@ -1290,24 +1360,115 @@ GTEST_API_ void InitGoogleTest(int* argc, wchar_t** argv); namespace internal { +// FormatForComparison<ToPrint, OtherOperand>::Format(value) formats a +// value of type ToPrint that is an operand of a comparison assertion +// (e.g. ASSERT_EQ). OtherOperand is the type of the other operand in +// the comparison, and is used to help determine the best way to +// format the value. In particular, when the value is a C string +// (char pointer) and the other operand is an STL string object, we +// want to format the C string as a string, since we know it is +// compared by value with the string object. If the value is a char +// pointer but the other operand is not an STL string object, we don't +// know whether the pointer is supposed to point to a NUL-terminated +// string, and thus want to print it as a pointer to be safe. +// +// INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. + +// The default case. +template <typename ToPrint, typename OtherOperand> +class FormatForComparison { + public: + static ::std::string Format(const ToPrint& value) { + return ::testing::PrintToString(value); + } +}; + +// Array. +template <typename ToPrint, size_t N, typename OtherOperand> +class FormatForComparison<ToPrint[N], OtherOperand> { + public: + static ::std::string Format(const ToPrint* value) { + return FormatForComparison<const ToPrint*, OtherOperand>::Format(value); + } +}; + +// By default, print C string as pointers to be safe, as we don't know +// whether they actually point to a NUL-terminated string. + +#define GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(CharType) \ + template <typename OtherOperand> \ + class FormatForComparison<CharType*, OtherOperand> { \ + public: \ + static ::std::string Format(CharType* value) { \ + return ::testing::PrintToString(static_cast<const void*>(value)); \ + } \ + } + +GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(char); +GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(const char); +GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(wchar_t); +GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(const wchar_t); + +#undef GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_ + +// If a C string is compared with an STL string object, we know it's meant +// to point to a NUL-terminated string, and thus can print it as a string. + +#define GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(CharType, OtherStringType) \ + template <> \ + class FormatForComparison<CharType*, OtherStringType> { \ + public: \ + static ::std::string Format(CharType* value) { \ + return ::testing::PrintToString(value); \ + } \ + } + +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(char, ::std::string); +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(const char, ::std::string); + +#if GTEST_HAS_GLOBAL_STRING +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(char, ::string); +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(const char, ::string); +#endif + +#if GTEST_HAS_GLOBAL_WSTRING +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(wchar_t, ::wstring); +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(const wchar_t, ::wstring); +#endif + +#if GTEST_HAS_STD_WSTRING +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(wchar_t, ::std::wstring); +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(const wchar_t, ::std::wstring); +#endif + +#undef GTEST_IMPL_FORMAT_C_STRING_AS_STRING_ + // Formats a comparison assertion (e.g. ASSERT_EQ, EXPECT_LT, and etc) // operand to be used in a failure message. The type (but not value) // of the other operand may affect the format. This allows us to // print a char* as a raw pointer when it is compared against another -// char*, and print it as a C string when it is compared against an -// std::string object, for example. -// -// The default implementation ignores the type of the other operand. -// Some specialized versions are used to handle formatting wide or -// narrow C strings. +// char* or void*, and print it as a C string when it is compared +// against an std::string object, for example. // // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. template <typename T1, typename T2> -String FormatForComparisonFailureMessage(const T1& value, - const T2& /* other_operand */) { - // C++Builder compiles this incorrectly if the namespace isn't explicitly - // given. - return ::testing::PrintToString(value); +std::string FormatForComparisonFailureMessage( + const T1& value, const T2& /* other_operand */) { + return FormatForComparison<T1, T2>::Format(value); +} + +// Separate the error generating code from the code path to reduce the stack +// frame size of CmpHelperEQ. This helps reduce the overhead of some sanitizers +// when calling EXPECT_* in a tight loop. +template <typename T1, typename T2> +AssertionResult CmpHelperEQFailure(const char* expected_expression, + const char* actual_expression, + const T1& expected, const T2& actual) { + return EqFailure(expected_expression, + actual_expression, + FormatForComparisonFailureMessage(expected, actual), + FormatForComparisonFailureMessage(actual, expected), + false); } // The helper function for {ASSERT|EXPECT}_EQ. @@ -1316,25 +1477,14 @@ AssertionResult CmpHelperEQ(const char* expected_expression, const char* actual_expression, const T1& expected, const T2& actual) { -#ifdef _MSC_VER -# pragma warning(push) // Saves the current warning state. -# pragma warning(disable:4389) // Temporarily disables warning on - // signed/unsigned mismatch. -#endif - +GTEST_DISABLE_MSC_WARNINGS_PUSH_(4389 /* signed/unsigned mismatch */) if (expected == actual) { return AssertionSuccess(); } +GTEST_DISABLE_MSC_WARNINGS_POP_() -#ifdef _MSC_VER -# pragma warning(pop) // Restores the warning state. -#endif - - return EqFailure(expected_expression, - actual_expression, - FormatForComparisonFailureMessage(expected, actual), - FormatForComparisonFailureMessage(actual, expected), - false); + return CmpHelperEQFailure(expected_expression, actual_expression, expected, + actual); } // With this overloaded version, we allow anonymous enums to be used @@ -1422,6 +1572,19 @@ class EqHelper<true> { } }; +// Separate the error generating code from the code path to reduce the stack +// frame size of CmpHelperOP. This helps reduce the overhead of some sanitizers +// when calling EXPECT_OP in a tight loop. +template <typename T1, typename T2> +AssertionResult CmpHelperOpFailure(const char* expr1, const char* expr2, + const T1& val1, const T2& val2, + const char* op) { + return AssertionFailure() + << "Expected: (" << expr1 << ") " << op << " (" << expr2 + << "), actual: " << FormatForComparisonFailureMessage(val1, val2) + << " vs " << FormatForComparisonFailureMessage(val2, val1); +} + // A macro for implementing the helper functions needed to implement // ASSERT_?? and EXPECT_??. It is here just to avoid copy-and-paste // of similar code. @@ -1432,6 +1595,7 @@ class EqHelper<true> { // with gcc 4. // // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. + #define GTEST_IMPL_CMP_HELPER_(op_name, op)\ template <typename T1, typename T2>\ AssertionResult CmpHelper##op_name(const char* expr1, const char* expr2, \ @@ -1439,10 +1603,7 @@ AssertionResult CmpHelper##op_name(const char* expr1, const char* expr2, \ if (val1 op val2) {\ return AssertionSuccess();\ } else {\ - return AssertionFailure() \ - << "Expected: (" << expr1 << ") " #op " (" << expr2\ - << "), actual: " << FormatForComparisonFailureMessage(val1, val2)\ - << " vs " << FormatForComparisonFailureMessage(val2, val1);\ + return CmpHelperOpFailure(expr1, expr2, val1, val2, #op);\ }\ }\ GTEST_API_ AssertionResult CmpHelper##op_name(\ @@ -1455,11 +1616,11 @@ GTEST_IMPL_CMP_HELPER_(NE, !=); // Implements the helper function for {ASSERT|EXPECT}_LE GTEST_IMPL_CMP_HELPER_(LE, <=); // Implements the helper function for {ASSERT|EXPECT}_LT -GTEST_IMPL_CMP_HELPER_(LT, < ); +GTEST_IMPL_CMP_HELPER_(LT, <); // Implements the helper function for {ASSERT|EXPECT}_GE GTEST_IMPL_CMP_HELPER_(GE, >=); // Implements the helper function for {ASSERT|EXPECT}_GT -GTEST_IMPL_CMP_HELPER_(GT, > ); +GTEST_IMPL_CMP_HELPER_(GT, >); #undef GTEST_IMPL_CMP_HELPER_ @@ -1623,9 +1784,9 @@ class GTEST_API_ AssertHelper { : type(t), file(srcfile), line(line_num), message(msg) { } TestPartResult::Type const type; - const char* const file; - int const line; - String const message; + const char* const file; + int const line; + std::string const message; private: GTEST_DISALLOW_COPY_AND_ASSIGN_(AssertHelperData); @@ -1684,7 +1845,12 @@ class WithParamInterface { // references static data, to reduce the opportunity for incorrect uses // like writing 'WithParamInterface<bool>::GetParam()' for a test that // uses a fixture whose parameter type is int. - const ParamType& GetParam() const { return *parameter_; } + const ParamType& GetParam() const { + GTEST_CHECK_(parameter_ != NULL) + << "GetParam() can only be called inside a value-parameterized test " + << "-- did you intend to write TEST_P instead of TEST_F?"; + return *parameter_; + } private: // Sets parameter value. The caller is responsible for making sure the value @@ -1730,12 +1896,6 @@ class TestWithParam : public Test, public WithParamInterface<T> { // usually want the fail-fast behavior of FAIL and ASSERT_*, but those // writing data-driven tests often find themselves using ADD_FAILURE // and EXPECT_* more. -// -// Examples: -// -// EXPECT_TRUE(server.StatusIsOK()); -// ASSERT_FALSE(server.HasPendingRequest(port)) -// << "There are still pending requests " << "on port " << port; // Generates a nonfatal failure with a generic message. #define ADD_FAILURE() GTEST_NONFATAL_FAILURE_("Failed") @@ -1909,7 +2069,7 @@ class TestWithParam : public Test, public WithParamInterface<T> { # define ASSERT_GT(val1, val2) GTEST_ASSERT_GT(val1, val2) #endif -// C String Comparisons. All tests treat NULL and any non-NULL string +// C-string Comparisons. All tests treat NULL and any non-NULL string // as different. Two NULLs are equal. // // * {ASSERT|EXPECT}_STREQ(s1, s2): Tests that s1 == s2 @@ -2093,8 +2253,8 @@ bool StaticAssertTypeEq() { // The convention is to end the test case name with "Test". For // example, a test case for the Foo class can be named FooTest. // -// The user should put his test code between braces after using this -// macro. Example: +// Test code should appear between braces after an invocation of +// this macro. Example: // // TEST(FooTest, InitializesCorrectly) { // Foo foo; @@ -2150,15 +2310,20 @@ bool StaticAssertTypeEq() { GTEST_TEST_(test_fixture, test_name, test_fixture, \ ::testing::internal::GetTypeId<test_fixture>()) -// Use this macro in main() to run all tests. It returns 0 if all +} // namespace testing + +// Use this function in main() to run all tests. It returns 0 if all // tests are successful, or 1 otherwise. // // RUN_ALL_TESTS() should be invoked after the command line has been // parsed by InitGoogleTest(). +// +// This function was formerly a macro; thus, it is in the global +// namespace and has an all-caps name. +int RUN_ALL_TESTS() GTEST_MUST_USE_RESULT_; -#define RUN_ALL_TESTS()\ - (::testing::UnitTest::GetInstance()->Run()) - -} // namespace testing +inline int RUN_ALL_TESTS() { + return ::testing::UnitTest::GetInstance()->Run(); +} #endif // GTEST_INCLUDE_GTEST_GTEST_H_ diff --git a/tests/gtest/include/gtest/gtest_pred_impl.h b/tests/gtest/include/gtest/gtest_pred_impl.h index 3805f85bd..30ae712f5 100644 --- a/tests/gtest/include/gtest/gtest_pred_impl.h +++ b/tests/gtest/include/gtest/gtest_pred_impl.h @@ -27,7 +27,7 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// This file is AUTOMATICALLY GENERATED on 09/24/2010 by command +// This file is AUTOMATICALLY GENERATED on 10/31/2011 by command // 'gen_gtest_pred_impl.py 5'. DO NOT EDIT BY HAND! // // Implements a family of generic predicate assertion macros. @@ -98,7 +98,7 @@ AssertionResult AssertPred1Helper(const char* pred_text, // Internal macro for implementing {EXPECT|ASSERT}_PRED_FORMAT1. // Don't use this in your code. #define GTEST_PRED_FORMAT1_(pred_format, v1, on_failure)\ - GTEST_ASSERT_(pred_format(#v1, v1),\ + GTEST_ASSERT_(pred_format(#v1, v1), \ on_failure) // Internal macro for implementing {EXPECT|ASSERT}_PRED1. Don't use @@ -144,7 +144,7 @@ AssertionResult AssertPred2Helper(const char* pred_text, // Internal macro for implementing {EXPECT|ASSERT}_PRED_FORMAT2. // Don't use this in your code. #define GTEST_PRED_FORMAT2_(pred_format, v1, v2, on_failure)\ - GTEST_ASSERT_(pred_format(#v1, #v2, v1, v2),\ + GTEST_ASSERT_(pred_format(#v1, #v2, v1, v2), \ on_failure) // Internal macro for implementing {EXPECT|ASSERT}_PRED2. Don't use @@ -197,7 +197,7 @@ AssertionResult AssertPred3Helper(const char* pred_text, // Internal macro for implementing {EXPECT|ASSERT}_PRED_FORMAT3. // Don't use this in your code. #define GTEST_PRED_FORMAT3_(pred_format, v1, v2, v3, on_failure)\ - GTEST_ASSERT_(pred_format(#v1, #v2, #v3, v1, v2, v3),\ + GTEST_ASSERT_(pred_format(#v1, #v2, #v3, v1, v2, v3), \ on_failure) // Internal macro for implementing {EXPECT|ASSERT}_PRED3. Don't use @@ -257,7 +257,7 @@ AssertionResult AssertPred4Helper(const char* pred_text, // Internal macro for implementing {EXPECT|ASSERT}_PRED_FORMAT4. // Don't use this in your code. #define GTEST_PRED_FORMAT4_(pred_format, v1, v2, v3, v4, on_failure)\ - GTEST_ASSERT_(pred_format(#v1, #v2, #v3, #v4, v1, v2, v3, v4),\ + GTEST_ASSERT_(pred_format(#v1, #v2, #v3, #v4, v1, v2, v3, v4), \ on_failure) // Internal macro for implementing {EXPECT|ASSERT}_PRED4. Don't use @@ -324,7 +324,7 @@ AssertionResult AssertPred5Helper(const char* pred_text, // Internal macro for implementing {EXPECT|ASSERT}_PRED_FORMAT5. // Don't use this in your code. #define GTEST_PRED_FORMAT5_(pred_format, v1, v2, v3, v4, v5, on_failure)\ - GTEST_ASSERT_(pred_format(#v1, #v2, #v3, #v4, #v5, v1, v2, v3, v4, v5),\ + GTEST_ASSERT_(pred_format(#v1, #v2, #v3, #v4, #v5, v1, v2, v3, v4, v5), \ on_failure) // Internal macro for implementing {EXPECT|ASSERT}_PRED5. Don't use diff --git a/tests/gtest/include/gtest/internal/gtest-death-test-internal.h b/tests/gtest/include/gtest/internal/gtest-death-test-internal.h index 1d9f83b65..2b3a78f5b 100644 --- a/tests/gtest/include/gtest/internal/gtest-death-test-internal.h +++ b/tests/gtest/include/gtest/internal/gtest-death-test-internal.h @@ -127,11 +127,11 @@ class GTEST_API_ DeathTest { // the last death test. static const char* LastMessage(); - static void set_last_death_test_message(const String& message); + static void set_last_death_test_message(const std::string& message); private: // A string containing a description of the outcome of the last death test. - static String last_death_test_message_; + static std::string last_death_test_message_; GTEST_DISALLOW_COPY_AND_ASSIGN_(DeathTest); }; @@ -217,12 +217,23 @@ GTEST_API_ bool ExitedUnsuccessfully(int exit_status); // The symbol "fail" here expands to something into which a message // can be streamed. +// This macro is for implementing ASSERT/EXPECT_DEBUG_DEATH when compiled in +// NDEBUG mode. In this case we need the statements to be executed, the regex is +// ignored, and the macro must accept a streamed message even though the message +// is never printed. +# define GTEST_EXECUTE_STATEMENT_(statement, regex) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::AlwaysTrue()) { \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ + } else \ + ::testing::Message() + // A class representing the parsed contents of the // --gtest_internal_run_death_test flag, as it existed when // RUN_ALL_TESTS was called. class InternalRunDeathTestFlag { public: - InternalRunDeathTestFlag(const String& a_file, + InternalRunDeathTestFlag(const std::string& a_file, int a_line, int an_index, int a_write_fd) @@ -234,13 +245,13 @@ class InternalRunDeathTestFlag { posix::Close(write_fd_); } - String file() const { return file_; } + const std::string& file() const { return file_; } int line() const { return line_; } int index() const { return index_; } int write_fd() const { return write_fd_; } private: - String file_; + std::string file_; int line_; int index_; int write_fd_; diff --git a/tests/gtest/include/gtest/internal/gtest-filepath.h b/tests/gtest/include/gtest/internal/gtest-filepath.h index b36b3cf21..7a13b4b0d 100644 --- a/tests/gtest/include/gtest/internal/gtest-filepath.h +++ b/tests/gtest/include/gtest/internal/gtest-filepath.h @@ -61,11 +61,7 @@ class GTEST_API_ FilePath { FilePath() : pathname_("") { } FilePath(const FilePath& rhs) : pathname_(rhs.pathname_) { } - explicit FilePath(const char* pathname) : pathname_(pathname) { - Normalize(); - } - - explicit FilePath(const String& pathname) : pathname_(pathname) { + explicit FilePath(const std::string& pathname) : pathname_(pathname) { Normalize(); } @@ -78,7 +74,7 @@ class GTEST_API_ FilePath { pathname_ = rhs.pathname_; } - String ToString() const { return pathname_; } + const std::string& string() const { return pathname_; } const char* c_str() const { return pathname_.c_str(); } // Returns the current working directory, or "" if unsuccessful. @@ -111,8 +107,8 @@ class GTEST_API_ FilePath { const FilePath& base_name, const char* extension); - // Returns true iff the path is NULL or "". - bool IsEmpty() const { return c_str() == NULL || *c_str() == '\0'; } + // Returns true iff the path is "". + bool IsEmpty() const { return pathname_.empty(); } // If input name has a trailing separator character, removes it and returns // the name, otherwise return the name string unmodified. @@ -201,7 +197,7 @@ class GTEST_API_ FilePath { // separators. Returns NULL if no path separator was found. const char* FindLastPathSeparator() const; - String pathname_; + std::string pathname_; }; // class FilePath } // namespace internal diff --git a/tests/gtest/include/gtest/internal/gtest-internal.h b/tests/gtest/include/gtest/internal/gtest-internal.h index ad5369d61..ba7175cf0 100644 --- a/tests/gtest/include/gtest/internal/gtest-internal.h +++ b/tests/gtest/include/gtest/internal/gtest-internal.h @@ -44,18 +44,22 @@ # include <sys/types.h> # include <sys/wait.h> # include <unistd.h> -# include <signal.h> #endif // GTEST_OS_LINUX -#if GTEST_CAN_STREAM_RESULTS_ -# include <sys/socket.h> -#endif // GTEST_CAN_STREAM_RESULTS_ + +#if GTEST_HAS_EXCEPTIONS +# include <stdexcept> +#endif #include <ctype.h> +#include <float.h> #include <string.h> #include <iomanip> #include <limits> #include <set> +#include <string> +#include <vector> +#include "gtest/gtest-message.h" #include "gtest/internal/gtest-string.h" #include "gtest/internal/gtest-filepath.h" #include "gtest/internal/gtest-type-util.h" @@ -71,36 +75,6 @@ #define GTEST_CONCAT_TOKEN_(foo, bar) GTEST_CONCAT_TOKEN_IMPL_(foo, bar) #define GTEST_CONCAT_TOKEN_IMPL_(foo, bar) foo ## bar -// Google Test defines the testing::Message class to allow construction of -// test messages via the << operator. The idea is that anything -// streamable to std::ostream can be streamed to a testing::Message. -// This allows a user to use his own types in Google Test assertions by -// overloading the << operator. -// -// util/gtl/stl_logging-inl.h overloads << for STL containers. These -// overloads cannot be defined in the std namespace, as that will be -// undefined behavior. Therefore, they are defined in the global -// namespace instead. -// -// C++'s symbol lookup rule (i.e. Koenig lookup) says that these -// overloads are visible in either the std namespace or the global -// namespace, but not other namespaces, including the testing -// namespace which Google Test's Message class is in. -// -// To allow STL containers (and other types that has a << operator -// defined in the global namespace) to be used in Google Test assertions, -// testing::Message must access the custom << operator from the global -// namespace. Hence this helper function. -// -// Note: Jeffrey Yasskin suggested an alternative fix by "using -// ::operator<<;" in the definition of Message's operator<<. That fix -// doesn't require a helper function, but unfortunately doesn't -// compile with MSVC. -template <typename T> -inline void GTestStreamToHelper(std::ostream* os, const T& val) { - *os << val; -} - class ProtocolMessage; namespace proto2 { class Message; } @@ -126,17 +100,12 @@ class TestInfoImpl; // Opaque implementation of TestInfo class UnitTestImpl; // Opaque implementation of UnitTest // How many times InitGoogleTest() has been called. -extern int g_init_gtest_count; +GTEST_API_ extern int g_init_gtest_count; // The text used in failure messages to indicate the start of the // stack trace. GTEST_API_ extern const char kStackTraceMarker[]; -// A secret type that Google Test users don't know about. It has no -// definition on purpose. Therefore it's impossible to create a -// Secret object, which is what we want. -class Secret; - // Two overloaded helpers for checking at compile time whether an // expression is a null pointer literal (i.e. NULL or any 0-valued // compile-time integral constant). Their return values have @@ -167,8 +136,23 @@ char (&IsNullLiteralHelper(...))[2]; // NOLINT #endif // GTEST_ELLIPSIS_NEEDS_POD_ // Appends the user-supplied message to the Google-Test-generated message. -GTEST_API_ String AppendUserMessage(const String& gtest_msg, - const Message& user_msg); +GTEST_API_ std::string AppendUserMessage( + const std::string& gtest_msg, const Message& user_msg); + +#if GTEST_HAS_EXCEPTIONS + +// This exception is thrown by (and only by) a failed Google Test +// assertion when GTEST_FLAG(throw_on_failure) is true (if exceptions +// are enabled). We derive it from std::runtime_error, which is for +// errors presumably detectable only at run time. Since +// std::runtime_error inherits from std::exception, many testing +// frameworks know how to extract and print the message inside it. +class GTEST_API_ GoogleTestFailureException : public ::std::runtime_error { + public: + explicit GoogleTestFailureException(const TestPartResult& failure); +}; + +#endif // GTEST_HAS_EXCEPTIONS // A helper class for creating scoped traces in user programs. class GTEST_API_ ScopedTrace { @@ -189,76 +173,35 @@ class GTEST_API_ ScopedTrace { // c'tor and d'tor. Therefore it doesn't // need to be used otherwise. -// Converts a streamable value to a String. A NULL pointer is -// converted to "(null)". When the input value is a ::string, -// ::std::string, ::wstring, or ::std::wstring object, each NUL -// character in it is replaced with "\\0". -// Declared here but defined in gtest.h, so that it has access -// to the definition of the Message class, required by the ARM -// compiler. -template <typename T> -String StreamableToString(const T& streamable); - -// The Symbian compiler has a bug that prevents it from selecting the -// correct overload of FormatForComparisonFailureMessage (see below) -// unless we pass the first argument by reference. If we do that, -// however, Visual Age C++ 10.1 generates a compiler error. Therefore -// we only apply the work-around for Symbian. -#if defined(__SYMBIAN32__) -# define GTEST_CREF_WORKAROUND_ const& -#else -# define GTEST_CREF_WORKAROUND_ -#endif - -// When this operand is a const char* or char*, if the other operand -// is a ::std::string or ::string, we print this operand as a C string -// rather than a pointer (we do the same for wide strings); otherwise -// we print it as a pointer to be safe. - -// This internal macro is used to avoid duplicated code. -#define GTEST_FORMAT_IMPL_(operand2_type, operand1_printer)\ -inline String FormatForComparisonFailureMessage(\ - operand2_type::value_type* GTEST_CREF_WORKAROUND_ str, \ - const operand2_type& /*operand2*/) {\ - return operand1_printer(str);\ -}\ -inline String FormatForComparisonFailureMessage(\ - const operand2_type::value_type* GTEST_CREF_WORKAROUND_ str, \ - const operand2_type& /*operand2*/) {\ - return operand1_printer(str);\ -} - -GTEST_FORMAT_IMPL_(::std::string, String::ShowCStringQuoted) -#if GTEST_HAS_STD_WSTRING -GTEST_FORMAT_IMPL_(::std::wstring, String::ShowWideCStringQuoted) -#endif // GTEST_HAS_STD_WSTRING - -#if GTEST_HAS_GLOBAL_STRING -GTEST_FORMAT_IMPL_(::string, String::ShowCStringQuoted) -#endif // GTEST_HAS_GLOBAL_STRING -#if GTEST_HAS_GLOBAL_WSTRING -GTEST_FORMAT_IMPL_(::wstring, String::ShowWideCStringQuoted) -#endif // GTEST_HAS_GLOBAL_WSTRING - -#undef GTEST_FORMAT_IMPL_ - -// The next four overloads handle the case where the operand being -// printed is a char/wchar_t pointer and the other operand is not a -// string/wstring object. In such cases, we just print the operand as -// a pointer to be safe. -#define GTEST_FORMAT_CHAR_PTR_IMPL_(CharType) \ - template <typename T> \ - String FormatForComparisonFailureMessage(CharType* GTEST_CREF_WORKAROUND_ p, \ - const T&) { \ - return PrintToString(static_cast<const void*>(p)); \ - } - -GTEST_FORMAT_CHAR_PTR_IMPL_(char) -GTEST_FORMAT_CHAR_PTR_IMPL_(const char) -GTEST_FORMAT_CHAR_PTR_IMPL_(wchar_t) -GTEST_FORMAT_CHAR_PTR_IMPL_(const wchar_t) - -#undef GTEST_FORMAT_CHAR_PTR_IMPL_ +namespace edit_distance { +// Returns the optimal edits to go from 'left' to 'right'. +// All edits cost the same, with replace having lower priority than +// add/remove. +// Simple implementation of the Wagner–Fischer algorithm. +// See http://en.wikipedia.org/wiki/Wagner-Fischer_algorithm +enum EditType { kMatch, kAdd, kRemove, kReplace }; +GTEST_API_ std::vector<EditType> CalculateOptimalEdits( + const std::vector<size_t>& left, const std::vector<size_t>& right); + +// Same as above, but the input is represented as strings. +GTEST_API_ std::vector<EditType> CalculateOptimalEdits( + const std::vector<std::string>& left, + const std::vector<std::string>& right); + +// Create a diff of the input strings in Unified diff format. +GTEST_API_ std::string CreateUnifiedDiff(const std::vector<std::string>& left, + const std::vector<std::string>& right, + size_t context = 2); + +} // namespace edit_distance + +// Calculate the diff between 'left' and 'right' and return it in unified diff +// format. +// If not null, stores in 'total_line_count' the total number of lines found +// in left + right. +GTEST_API_ std::string DiffStrings(const std::string& left, + const std::string& right, + size_t* total_line_count); // Constructs and returns the message for an equality assertion // (e.g. ASSERT_EQ, EXPECT_STREQ, etc) failure. @@ -277,12 +220,12 @@ GTEST_FORMAT_CHAR_PTR_IMPL_(const wchar_t) // be inserted into the message. GTEST_API_ AssertionResult EqFailure(const char* expected_expression, const char* actual_expression, - const String& expected_value, - const String& actual_value, + const std::string& expected_value, + const std::string& actual_value, bool ignoring_case); // Constructs a failure message for Boolean assertions such as EXPECT_TRUE. -GTEST_API_ String GetBoolAssertionFailureMessage( +GTEST_API_ std::string GetBoolAssertionFailureMessage( const AssertionResult& assertion_result, const char* expression_text, const char* actual_predicate_value, @@ -357,7 +300,7 @@ class FloatingPoint { // bits. Therefore, 4 should be enough for ordinary use. // // See the following article for more details on ULP: - // http://www.cygnus-software.com/papers/comparingfloats/comparingfloats.htm. + // http://randomascii.wordpress.com/2012/02/25/comparing-floating-point-numbers-2012-edition/ static const size_t kMaxUlps = 4; // Constructs a FloatingPoint from a raw floating-point number. @@ -384,6 +327,9 @@ class FloatingPoint { return ReinterpretBits(kExponentBitMask); } + // Returns the maximum representable finite floating-point number. + static RawType Max(); + // Non-static methods // Returns the bits that represents this number. @@ -464,6 +410,13 @@ class FloatingPoint { FloatingPointUnion u_; }; +// We cannot use std::numeric_limits<T>::max() as it clashes with the max() +// macro defined by <windows.h>. +template <> +inline float FloatingPoint<float>::Max() { return FLT_MAX; } +template <> +inline double FloatingPoint<double>::Max() { return DBL_MAX; } + // Typedefs the instances of the FloatingPoint template class that we // care to use. typedef FloatingPoint<float> Float; @@ -558,7 +511,7 @@ typedef void (*TearDownTestCaseFunc)(); // test_case_name: name of the test case // name: name of the test // type_param the name of the test's type parameter, or NULL if -// this is not a typed or a type-parameterized test. +// this is not a typed or a type-parameterized test. // value_param text representation of the test's value parameter, // or NULL if this is not a type-parameterized test. // fixture_class_id: ID of the test fixture class @@ -568,7 +521,8 @@ typedef void (*TearDownTestCaseFunc)(); // The newly created TestInfo instance will assume // ownership of the factory object. GTEST_API_ TestInfo* MakeAndRegisterTestInfo( - const char* test_case_name, const char* name, + const char* test_case_name, + const char* name, const char* type_param, const char* value_param, TypeId fixture_class_id, @@ -628,9 +582,9 @@ inline const char* SkipComma(const char* str) { // Returns the prefix of 'str' before the first comma in it; returns // the entire string if it contains no comma. -inline String GetPrefixUntilComma(const char* str) { +inline std::string GetPrefixUntilComma(const char* str) { const char* comma = strchr(str, ','); - return comma == NULL ? String(str) : String(str, comma - str); + return comma == NULL ? str : std::string(str, comma); } // TypeParameterizedTest<Fixture, TestSel, Types>::Register() @@ -656,9 +610,9 @@ class TypeParameterizedTest { // First, registers the first type-parameterized test in the type // list. MakeAndRegisterTestInfo( - String::Format("%s%s%s/%d", prefix, prefix[0] == '\0' ? "" : "/", - case_name, index).c_str(), - GetPrefixUntilComma(test_names).c_str(), + (std::string(prefix) + (prefix[0] == '\0' ? "" : "/") + case_name + "/" + + StreamableToString(index)).c_str(), + StripTrailingSpaces(GetPrefixUntilComma(test_names)).c_str(), GetTypeName<Type>().c_str(), NULL, // No value parameter. GetTypeId<FixtureClass>(), @@ -715,7 +669,7 @@ class TypeParameterizedTestCase<Fixture, Templates0, Types> { #endif // GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P -// Returns the current OS stack trace as a String. +// Returns the current OS stack trace as an std::string. // // The maximum number of stack frames to be included is specified by // the gtest_stack_trace_depth flag. The skip_count parameter @@ -725,8 +679,8 @@ class TypeParameterizedTestCase<Fixture, Templates0, Types> { // For example, if Foo() calls Bar(), which in turn calls // GetCurrentOsStackTraceExceptTop(..., 1), Foo() will be included in // the trace but Bar() and GetCurrentOsStackTraceExceptTop() won't. -GTEST_API_ String GetCurrentOsStackTraceExceptTop(UnitTest* unit_test, - int skip_count); +GTEST_API_ std::string GetCurrentOsStackTraceExceptTop( + UnitTest* unit_test, int skip_count); // Helpers for suppressing warnings on unreachable code or constant // condition. @@ -801,13 +755,19 @@ struct RemoveConst<const T> { typedef T type; }; // NOLINT // MSVC 8.0, Sun C++, and IBM XL C++ have a bug which causes the above // definition to fail to remove the const in 'const int[3]' and 'const // char[3][4]'. The following specialization works around the bug. -// However, it causes trouble with GCC and thus needs to be -// conditionally compiled. -#if defined(_MSC_VER) || defined(__SUNPRO_CC) || defined(__IBMCPP__) template <typename T, size_t N> struct RemoveConst<const T[N]> { typedef typename RemoveConst<T>::type type[N]; }; + +#if defined(_MSC_VER) && _MSC_VER < 1400 +// This is the only specialization that allows VC++ 7.1 to remove const in +// 'const int[3] and 'const int[3][4]'. However, it causes trouble with GCC +// and thus needs to be conditionally compiled. +template <typename T, size_t N> +struct RemoveConst<T[N]> { + typedef typename RemoveConst<T>::type type[N]; +}; #endif // A handy wrapper around RemoveConst that works when the argument @@ -856,7 +816,7 @@ class ImplicitlyConvertible { // MakeFrom() is an expression whose type is From. We cannot simply // use From(), as the type From may not have a public default // constructor. - static From MakeFrom(); + static typename AddReference<From>::type MakeFrom(); // These two functions are overloaded. Given an expression // Helper(x), the compiler will pick the first version if x can be @@ -874,25 +834,20 @@ class ImplicitlyConvertible { // We have to put the 'public' section after the 'private' section, // or MSVC refuses to compile the code. public: - // MSVC warns about implicitly converting from double to int for - // possible loss of data, so we need to temporarily disable the - // warning. -#ifdef _MSC_VER -# pragma warning(push) // Saves the current warning state. -# pragma warning(disable:4244) // Temporarily disables warning 4244. - - static const bool value = - sizeof(Helper(ImplicitlyConvertible::MakeFrom())) == 1; -# pragma warning(pop) // Restores the warning state. -#elif defined(__BORLANDC__) +#if defined(__BORLANDC__) // C++Builder cannot use member overload resolution during template // instantiation. The simplest workaround is to use its C++0x type traits // functions (C++Builder 2009 and above only). static const bool value = __is_convertible(From, To); #else + // MSVC warns about implicitly converting from double to int for + // possible loss of data, so we need to temporarily disable the + // warning. + GTEST_DISABLE_MSC_WARNINGS_PUSH_(4244) static const bool value = sizeof(Helper(ImplicitlyConvertible::MakeFrom())) == 1; -#endif // _MSV_VER + GTEST_DISABLE_MSC_WARNINGS_POP_() +#endif // __BORLANDC__ }; template <typename From, typename To> const bool ImplicitlyConvertible<From, To>::value; @@ -1018,11 +973,10 @@ void CopyArray(const T* from, size_t size, U* to) { // The relation between an NativeArray object (see below) and the // native array it represents. -enum RelationToSource { - kReference, // The NativeArray references the native array. - kCopy // The NativeArray makes a copy of the native array and - // owns the copy. -}; +// We use 2 different structs to allow non-copyable types to be used, as long +// as RelationToSourceReference() is passed. +struct RelationToSourceReference {}; +struct RelationToSourceCopy {}; // Adapts a native array to a read-only STL-style container. Instead // of the complete STL container concept, this adaptor only implements @@ -1040,22 +994,23 @@ class NativeArray { typedef Element* iterator; typedef const Element* const_iterator; - // Constructs from a native array. - NativeArray(const Element* array, size_t count, RelationToSource relation) { - Init(array, count, relation); + // Constructs from a native array. References the source. + NativeArray(const Element* array, size_t count, RelationToSourceReference) { + InitRef(array, count); + } + + // Constructs from a native array. Copies the source. + NativeArray(const Element* array, size_t count, RelationToSourceCopy) { + InitCopy(array, count); } // Copy constructor. NativeArray(const NativeArray& rhs) { - Init(rhs.array_, rhs.size_, rhs.relation_to_source_); + (this->*rhs.clone_)(rhs.array_, rhs.size_); } ~NativeArray() { - // Ensures that the user doesn't instantiate NativeArray with a - // const or reference type. - static_cast<void>(StaticAssertTypeEqHelper<Element, - GTEST_REMOVE_REFERENCE_AND_CONST_(Element)>()); - if (relation_to_source_ == kCopy) + if (clone_ != &NativeArray::InitRef) delete[] array_; } @@ -1069,23 +1024,30 @@ class NativeArray { } private: - // Initializes this object; makes a copy of the input array if - // 'relation' is kCopy. - void Init(const Element* array, size_t a_size, RelationToSource relation) { - if (relation == kReference) { - array_ = array; - } else { - Element* const copy = new Element[a_size]; - CopyArray(array, a_size, copy); - array_ = copy; - } + enum { + kCheckTypeIsNotConstOrAReference = StaticAssertTypeEqHelper< + Element, GTEST_REMOVE_REFERENCE_AND_CONST_(Element)>::value, + }; + + // Initializes this object with a copy of the input. + void InitCopy(const Element* array, size_t a_size) { + Element* const copy = new Element[a_size]; + CopyArray(array, a_size, copy); + array_ = copy; + size_ = a_size; + clone_ = &NativeArray::InitCopy; + } + + // Initializes this object with a reference of the input. + void InitRef(const Element* array, size_t a_size) { + array_ = array; size_ = a_size; - relation_to_source_ = relation; + clone_ = &NativeArray::InitRef; } const Element* array_; size_t size_; - RelationToSource relation_to_source_; + void (NativeArray::*clone_)(const Element*, size_t); GTEST_DISALLOW_ASSIGN_(NativeArray); }; @@ -1228,3 +1190,4 @@ class GTEST_TEST_CLASS_NAME_(test_case_name, test_name) : public parent_class {\ void GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::TestBody() #endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_ + diff --git a/tests/gtest/include/gtest/internal/gtest-linked_ptr.h b/tests/gtest/include/gtest/internal/gtest-linked_ptr.h index 57147b4e8..360294221 100644 --- a/tests/gtest/include/gtest/internal/gtest-linked_ptr.h +++ b/tests/gtest/include/gtest/internal/gtest-linked_ptr.h @@ -105,25 +105,35 @@ class linked_ptr_internal { // framework. // Join an existing circle. - // L < g_linked_ptr_mutex - void join(linked_ptr_internal const* ptr) { + void join(linked_ptr_internal const* ptr) + GTEST_LOCK_EXCLUDED_(g_linked_ptr_mutex) { MutexLock lock(&g_linked_ptr_mutex); linked_ptr_internal const* p = ptr; - while (p->next_ != ptr) p = p->next_; + while (p->next_ != ptr) { + assert(p->next_ != this && + "Trying to join() a linked ring we are already in. " + "Is GMock thread safety enabled?"); + p = p->next_; + } p->next_ = this; next_ = ptr; } // Leave whatever circle we're part of. Returns true if we were the // last member of the circle. Once this is done, you can join() another. - // L < g_linked_ptr_mutex - bool depart() { + bool depart() + GTEST_LOCK_EXCLUDED_(g_linked_ptr_mutex) { MutexLock lock(&g_linked_ptr_mutex); if (next_ == this) return true; linked_ptr_internal const* p = next_; - while (p->next_ != this) p = p->next_; + while (p->next_ != this) { + assert(p->next_ != next_ && + "Trying to depart() a linked ring we are not in. " + "Is GMock thread safety enabled?"); + p = p->next_; + } p->next_ = next_; return false; } diff --git a/tests/gtest/include/gtest/internal/gtest-param-util-generated.h b/tests/gtest/include/gtest/internal/gtest-param-util-generated.h index 258267500..6dbaf4b7a 100644 --- a/tests/gtest/include/gtest/internal/gtest-param-util-generated.h +++ b/tests/gtest/include/gtest/internal/gtest-param-util-generated.h @@ -40,7 +40,7 @@ // and at most 10 arguments in Combine. Please contact // googletestframework@googlegroups.com if you need more. // Please note that the number of arguments to Combine is limited -// by the maximum arity of the implementation of tr1::tuple which is +// by the maximum arity of the implementation of tuple which is // currently set at 10. #ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_GENERATED_H_ @@ -95,7 +95,7 @@ class ValueArray2 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_)}; return ValuesIn(array); } @@ -114,7 +114,8 @@ class ValueArray3 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_)}; return ValuesIn(array); } @@ -135,7 +136,8 @@ class ValueArray4 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_)}; return ValuesIn(array); } @@ -157,7 +159,8 @@ class ValueArray5 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_)}; return ValuesIn(array); } @@ -181,7 +184,9 @@ class ValueArray6 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_)}; return ValuesIn(array); } @@ -206,7 +211,9 @@ class ValueArray7 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_)}; return ValuesIn(array); } @@ -233,7 +240,9 @@ class ValueArray8 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_)}; return ValuesIn(array); } @@ -261,7 +270,10 @@ class ValueArray9 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_)}; return ValuesIn(array); } @@ -290,7 +302,10 @@ class ValueArray10 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_)}; return ValuesIn(array); } @@ -321,7 +336,10 @@ class ValueArray11 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_)}; return ValuesIn(array); } @@ -353,8 +371,11 @@ class ValueArray12 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_)}; return ValuesIn(array); } @@ -388,8 +409,11 @@ class ValueArray13 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_)}; return ValuesIn(array); } @@ -424,8 +448,11 @@ class ValueArray14 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_)}; return ValuesIn(array); } @@ -461,8 +488,12 @@ class ValueArray15 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_)}; return ValuesIn(array); } @@ -501,8 +532,12 @@ class ValueArray16 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_)}; return ValuesIn(array); } @@ -542,8 +577,12 @@ class ValueArray17 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_)}; return ValuesIn(array); } @@ -584,8 +623,13 @@ class ValueArray18 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_)}; return ValuesIn(array); } @@ -627,8 +671,13 @@ class ValueArray19 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_), static_cast<T>(v19_)}; return ValuesIn(array); } @@ -672,8 +721,13 @@ class ValueArray20 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_), static_cast<T>(v19_), static_cast<T>(v20_)}; return ValuesIn(array); } @@ -719,8 +773,14 @@ class ValueArray21 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_), static_cast<T>(v19_), static_cast<T>(v20_), + static_cast<T>(v21_)}; return ValuesIn(array); } @@ -767,8 +827,14 @@ class ValueArray22 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_), static_cast<T>(v19_), static_cast<T>(v20_), + static_cast<T>(v21_), static_cast<T>(v22_)}; return ValuesIn(array); } @@ -817,9 +883,14 @@ class ValueArray23 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, - v23_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_), static_cast<T>(v19_), static_cast<T>(v20_), + static_cast<T>(v21_), static_cast<T>(v22_), static_cast<T>(v23_)}; return ValuesIn(array); } @@ -869,9 +940,15 @@ class ValueArray24 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, - v24_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_), static_cast<T>(v19_), static_cast<T>(v20_), + static_cast<T>(v21_), static_cast<T>(v22_), static_cast<T>(v23_), + static_cast<T>(v24_)}; return ValuesIn(array); } @@ -922,9 +999,15 @@ class ValueArray25 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, - v24_, v25_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_), static_cast<T>(v19_), static_cast<T>(v20_), + static_cast<T>(v21_), static_cast<T>(v22_), static_cast<T>(v23_), + static_cast<T>(v24_), static_cast<T>(v25_)}; return ValuesIn(array); } @@ -977,9 +1060,15 @@ class ValueArray26 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, - v24_, v25_, v26_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_), static_cast<T>(v19_), static_cast<T>(v20_), + static_cast<T>(v21_), static_cast<T>(v22_), static_cast<T>(v23_), + static_cast<T>(v24_), static_cast<T>(v25_), static_cast<T>(v26_)}; return ValuesIn(array); } @@ -1034,9 +1123,16 @@ class ValueArray27 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, - v24_, v25_, v26_, v27_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_), static_cast<T>(v19_), static_cast<T>(v20_), + static_cast<T>(v21_), static_cast<T>(v22_), static_cast<T>(v23_), + static_cast<T>(v24_), static_cast<T>(v25_), static_cast<T>(v26_), + static_cast<T>(v27_)}; return ValuesIn(array); } @@ -1092,9 +1188,16 @@ class ValueArray28 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, - v24_, v25_, v26_, v27_, v28_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_), static_cast<T>(v19_), static_cast<T>(v20_), + static_cast<T>(v21_), static_cast<T>(v22_), static_cast<T>(v23_), + static_cast<T>(v24_), static_cast<T>(v25_), static_cast<T>(v26_), + static_cast<T>(v27_), static_cast<T>(v28_)}; return ValuesIn(array); } @@ -1151,9 +1254,16 @@ class ValueArray29 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, - v24_, v25_, v26_, v27_, v28_, v29_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_), static_cast<T>(v19_), static_cast<T>(v20_), + static_cast<T>(v21_), static_cast<T>(v22_), static_cast<T>(v23_), + static_cast<T>(v24_), static_cast<T>(v25_), static_cast<T>(v26_), + static_cast<T>(v27_), static_cast<T>(v28_), static_cast<T>(v29_)}; return ValuesIn(array); } @@ -1212,9 +1322,17 @@ class ValueArray30 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, - v24_, v25_, v26_, v27_, v28_, v29_, v30_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_), static_cast<T>(v19_), static_cast<T>(v20_), + static_cast<T>(v21_), static_cast<T>(v22_), static_cast<T>(v23_), + static_cast<T>(v24_), static_cast<T>(v25_), static_cast<T>(v26_), + static_cast<T>(v27_), static_cast<T>(v28_), static_cast<T>(v29_), + static_cast<T>(v30_)}; return ValuesIn(array); } @@ -1275,9 +1393,17 @@ class ValueArray31 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, - v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_), static_cast<T>(v19_), static_cast<T>(v20_), + static_cast<T>(v21_), static_cast<T>(v22_), static_cast<T>(v23_), + static_cast<T>(v24_), static_cast<T>(v25_), static_cast<T>(v26_), + static_cast<T>(v27_), static_cast<T>(v28_), static_cast<T>(v29_), + static_cast<T>(v30_), static_cast<T>(v31_)}; return ValuesIn(array); } @@ -1339,9 +1465,17 @@ class ValueArray32 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, - v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_), static_cast<T>(v19_), static_cast<T>(v20_), + static_cast<T>(v21_), static_cast<T>(v22_), static_cast<T>(v23_), + static_cast<T>(v24_), static_cast<T>(v25_), static_cast<T>(v26_), + static_cast<T>(v27_), static_cast<T>(v28_), static_cast<T>(v29_), + static_cast<T>(v30_), static_cast<T>(v31_), static_cast<T>(v32_)}; return ValuesIn(array); } @@ -1405,9 +1539,18 @@ class ValueArray33 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, - v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_), static_cast<T>(v19_), static_cast<T>(v20_), + static_cast<T>(v21_), static_cast<T>(v22_), static_cast<T>(v23_), + static_cast<T>(v24_), static_cast<T>(v25_), static_cast<T>(v26_), + static_cast<T>(v27_), static_cast<T>(v28_), static_cast<T>(v29_), + static_cast<T>(v30_), static_cast<T>(v31_), static_cast<T>(v32_), + static_cast<T>(v33_)}; return ValuesIn(array); } @@ -1472,9 +1615,18 @@ class ValueArray34 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, - v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_), static_cast<T>(v19_), static_cast<T>(v20_), + static_cast<T>(v21_), static_cast<T>(v22_), static_cast<T>(v23_), + static_cast<T>(v24_), static_cast<T>(v25_), static_cast<T>(v26_), + static_cast<T>(v27_), static_cast<T>(v28_), static_cast<T>(v29_), + static_cast<T>(v30_), static_cast<T>(v31_), static_cast<T>(v32_), + static_cast<T>(v33_), static_cast<T>(v34_)}; return ValuesIn(array); } @@ -1540,10 +1692,18 @@ class ValueArray35 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, - v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_, - v35_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_), static_cast<T>(v19_), static_cast<T>(v20_), + static_cast<T>(v21_), static_cast<T>(v22_), static_cast<T>(v23_), + static_cast<T>(v24_), static_cast<T>(v25_), static_cast<T>(v26_), + static_cast<T>(v27_), static_cast<T>(v28_), static_cast<T>(v29_), + static_cast<T>(v30_), static_cast<T>(v31_), static_cast<T>(v32_), + static_cast<T>(v33_), static_cast<T>(v34_), static_cast<T>(v35_)}; return ValuesIn(array); } @@ -1611,10 +1771,19 @@ class ValueArray36 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, - v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_, v35_, - v36_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_), static_cast<T>(v19_), static_cast<T>(v20_), + static_cast<T>(v21_), static_cast<T>(v22_), static_cast<T>(v23_), + static_cast<T>(v24_), static_cast<T>(v25_), static_cast<T>(v26_), + static_cast<T>(v27_), static_cast<T>(v28_), static_cast<T>(v29_), + static_cast<T>(v30_), static_cast<T>(v31_), static_cast<T>(v32_), + static_cast<T>(v33_), static_cast<T>(v34_), static_cast<T>(v35_), + static_cast<T>(v36_)}; return ValuesIn(array); } @@ -1684,10 +1853,19 @@ class ValueArray37 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, - v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_, v35_, - v36_, v37_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_), static_cast<T>(v19_), static_cast<T>(v20_), + static_cast<T>(v21_), static_cast<T>(v22_), static_cast<T>(v23_), + static_cast<T>(v24_), static_cast<T>(v25_), static_cast<T>(v26_), + static_cast<T>(v27_), static_cast<T>(v28_), static_cast<T>(v29_), + static_cast<T>(v30_), static_cast<T>(v31_), static_cast<T>(v32_), + static_cast<T>(v33_), static_cast<T>(v34_), static_cast<T>(v35_), + static_cast<T>(v36_), static_cast<T>(v37_)}; return ValuesIn(array); } @@ -1758,10 +1936,19 @@ class ValueArray38 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, - v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_, v35_, - v36_, v37_, v38_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_), static_cast<T>(v19_), static_cast<T>(v20_), + static_cast<T>(v21_), static_cast<T>(v22_), static_cast<T>(v23_), + static_cast<T>(v24_), static_cast<T>(v25_), static_cast<T>(v26_), + static_cast<T>(v27_), static_cast<T>(v28_), static_cast<T>(v29_), + static_cast<T>(v30_), static_cast<T>(v31_), static_cast<T>(v32_), + static_cast<T>(v33_), static_cast<T>(v34_), static_cast<T>(v35_), + static_cast<T>(v36_), static_cast<T>(v37_), static_cast<T>(v38_)}; return ValuesIn(array); } @@ -1833,10 +2020,20 @@ class ValueArray39 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, - v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_, v35_, - v36_, v37_, v38_, v39_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_), static_cast<T>(v19_), static_cast<T>(v20_), + static_cast<T>(v21_), static_cast<T>(v22_), static_cast<T>(v23_), + static_cast<T>(v24_), static_cast<T>(v25_), static_cast<T>(v26_), + static_cast<T>(v27_), static_cast<T>(v28_), static_cast<T>(v29_), + static_cast<T>(v30_), static_cast<T>(v31_), static_cast<T>(v32_), + static_cast<T>(v33_), static_cast<T>(v34_), static_cast<T>(v35_), + static_cast<T>(v36_), static_cast<T>(v37_), static_cast<T>(v38_), + static_cast<T>(v39_)}; return ValuesIn(array); } @@ -1910,10 +2107,20 @@ class ValueArray40 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, - v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_, v35_, - v36_, v37_, v38_, v39_, v40_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_), static_cast<T>(v19_), static_cast<T>(v20_), + static_cast<T>(v21_), static_cast<T>(v22_), static_cast<T>(v23_), + static_cast<T>(v24_), static_cast<T>(v25_), static_cast<T>(v26_), + static_cast<T>(v27_), static_cast<T>(v28_), static_cast<T>(v29_), + static_cast<T>(v30_), static_cast<T>(v31_), static_cast<T>(v32_), + static_cast<T>(v33_), static_cast<T>(v34_), static_cast<T>(v35_), + static_cast<T>(v36_), static_cast<T>(v37_), static_cast<T>(v38_), + static_cast<T>(v39_), static_cast<T>(v40_)}; return ValuesIn(array); } @@ -1989,10 +2196,20 @@ class ValueArray41 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, - v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_, v35_, - v36_, v37_, v38_, v39_, v40_, v41_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_), static_cast<T>(v19_), static_cast<T>(v20_), + static_cast<T>(v21_), static_cast<T>(v22_), static_cast<T>(v23_), + static_cast<T>(v24_), static_cast<T>(v25_), static_cast<T>(v26_), + static_cast<T>(v27_), static_cast<T>(v28_), static_cast<T>(v29_), + static_cast<T>(v30_), static_cast<T>(v31_), static_cast<T>(v32_), + static_cast<T>(v33_), static_cast<T>(v34_), static_cast<T>(v35_), + static_cast<T>(v36_), static_cast<T>(v37_), static_cast<T>(v38_), + static_cast<T>(v39_), static_cast<T>(v40_), static_cast<T>(v41_)}; return ValuesIn(array); } @@ -2069,10 +2286,21 @@ class ValueArray42 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, - v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_, v35_, - v36_, v37_, v38_, v39_, v40_, v41_, v42_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_), static_cast<T>(v19_), static_cast<T>(v20_), + static_cast<T>(v21_), static_cast<T>(v22_), static_cast<T>(v23_), + static_cast<T>(v24_), static_cast<T>(v25_), static_cast<T>(v26_), + static_cast<T>(v27_), static_cast<T>(v28_), static_cast<T>(v29_), + static_cast<T>(v30_), static_cast<T>(v31_), static_cast<T>(v32_), + static_cast<T>(v33_), static_cast<T>(v34_), static_cast<T>(v35_), + static_cast<T>(v36_), static_cast<T>(v37_), static_cast<T>(v38_), + static_cast<T>(v39_), static_cast<T>(v40_), static_cast<T>(v41_), + static_cast<T>(v42_)}; return ValuesIn(array); } @@ -2150,10 +2378,21 @@ class ValueArray43 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, - v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_, v35_, - v36_, v37_, v38_, v39_, v40_, v41_, v42_, v43_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_), static_cast<T>(v19_), static_cast<T>(v20_), + static_cast<T>(v21_), static_cast<T>(v22_), static_cast<T>(v23_), + static_cast<T>(v24_), static_cast<T>(v25_), static_cast<T>(v26_), + static_cast<T>(v27_), static_cast<T>(v28_), static_cast<T>(v29_), + static_cast<T>(v30_), static_cast<T>(v31_), static_cast<T>(v32_), + static_cast<T>(v33_), static_cast<T>(v34_), static_cast<T>(v35_), + static_cast<T>(v36_), static_cast<T>(v37_), static_cast<T>(v38_), + static_cast<T>(v39_), static_cast<T>(v40_), static_cast<T>(v41_), + static_cast<T>(v42_), static_cast<T>(v43_)}; return ValuesIn(array); } @@ -2233,10 +2472,21 @@ class ValueArray44 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, - v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_, v35_, - v36_, v37_, v38_, v39_, v40_, v41_, v42_, v43_, v44_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_), static_cast<T>(v19_), static_cast<T>(v20_), + static_cast<T>(v21_), static_cast<T>(v22_), static_cast<T>(v23_), + static_cast<T>(v24_), static_cast<T>(v25_), static_cast<T>(v26_), + static_cast<T>(v27_), static_cast<T>(v28_), static_cast<T>(v29_), + static_cast<T>(v30_), static_cast<T>(v31_), static_cast<T>(v32_), + static_cast<T>(v33_), static_cast<T>(v34_), static_cast<T>(v35_), + static_cast<T>(v36_), static_cast<T>(v37_), static_cast<T>(v38_), + static_cast<T>(v39_), static_cast<T>(v40_), static_cast<T>(v41_), + static_cast<T>(v42_), static_cast<T>(v43_), static_cast<T>(v44_)}; return ValuesIn(array); } @@ -2317,10 +2567,22 @@ class ValueArray45 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, - v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_, v35_, - v36_, v37_, v38_, v39_, v40_, v41_, v42_, v43_, v44_, v45_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_), static_cast<T>(v19_), static_cast<T>(v20_), + static_cast<T>(v21_), static_cast<T>(v22_), static_cast<T>(v23_), + static_cast<T>(v24_), static_cast<T>(v25_), static_cast<T>(v26_), + static_cast<T>(v27_), static_cast<T>(v28_), static_cast<T>(v29_), + static_cast<T>(v30_), static_cast<T>(v31_), static_cast<T>(v32_), + static_cast<T>(v33_), static_cast<T>(v34_), static_cast<T>(v35_), + static_cast<T>(v36_), static_cast<T>(v37_), static_cast<T>(v38_), + static_cast<T>(v39_), static_cast<T>(v40_), static_cast<T>(v41_), + static_cast<T>(v42_), static_cast<T>(v43_), static_cast<T>(v44_), + static_cast<T>(v45_)}; return ValuesIn(array); } @@ -2403,10 +2665,22 @@ class ValueArray46 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, - v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_, v35_, - v36_, v37_, v38_, v39_, v40_, v41_, v42_, v43_, v44_, v45_, v46_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_), static_cast<T>(v19_), static_cast<T>(v20_), + static_cast<T>(v21_), static_cast<T>(v22_), static_cast<T>(v23_), + static_cast<T>(v24_), static_cast<T>(v25_), static_cast<T>(v26_), + static_cast<T>(v27_), static_cast<T>(v28_), static_cast<T>(v29_), + static_cast<T>(v30_), static_cast<T>(v31_), static_cast<T>(v32_), + static_cast<T>(v33_), static_cast<T>(v34_), static_cast<T>(v35_), + static_cast<T>(v36_), static_cast<T>(v37_), static_cast<T>(v38_), + static_cast<T>(v39_), static_cast<T>(v40_), static_cast<T>(v41_), + static_cast<T>(v42_), static_cast<T>(v43_), static_cast<T>(v44_), + static_cast<T>(v45_), static_cast<T>(v46_)}; return ValuesIn(array); } @@ -2491,11 +2765,22 @@ class ValueArray47 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, - v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_, v35_, - v36_, v37_, v38_, v39_, v40_, v41_, v42_, v43_, v44_, v45_, v46_, - v47_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_), static_cast<T>(v19_), static_cast<T>(v20_), + static_cast<T>(v21_), static_cast<T>(v22_), static_cast<T>(v23_), + static_cast<T>(v24_), static_cast<T>(v25_), static_cast<T>(v26_), + static_cast<T>(v27_), static_cast<T>(v28_), static_cast<T>(v29_), + static_cast<T>(v30_), static_cast<T>(v31_), static_cast<T>(v32_), + static_cast<T>(v33_), static_cast<T>(v34_), static_cast<T>(v35_), + static_cast<T>(v36_), static_cast<T>(v37_), static_cast<T>(v38_), + static_cast<T>(v39_), static_cast<T>(v40_), static_cast<T>(v41_), + static_cast<T>(v42_), static_cast<T>(v43_), static_cast<T>(v44_), + static_cast<T>(v45_), static_cast<T>(v46_), static_cast<T>(v47_)}; return ValuesIn(array); } @@ -2581,11 +2866,23 @@ class ValueArray48 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, - v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_, v35_, - v36_, v37_, v38_, v39_, v40_, v41_, v42_, v43_, v44_, v45_, v46_, v47_, - v48_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_), static_cast<T>(v19_), static_cast<T>(v20_), + static_cast<T>(v21_), static_cast<T>(v22_), static_cast<T>(v23_), + static_cast<T>(v24_), static_cast<T>(v25_), static_cast<T>(v26_), + static_cast<T>(v27_), static_cast<T>(v28_), static_cast<T>(v29_), + static_cast<T>(v30_), static_cast<T>(v31_), static_cast<T>(v32_), + static_cast<T>(v33_), static_cast<T>(v34_), static_cast<T>(v35_), + static_cast<T>(v36_), static_cast<T>(v37_), static_cast<T>(v38_), + static_cast<T>(v39_), static_cast<T>(v40_), static_cast<T>(v41_), + static_cast<T>(v42_), static_cast<T>(v43_), static_cast<T>(v44_), + static_cast<T>(v45_), static_cast<T>(v46_), static_cast<T>(v47_), + static_cast<T>(v48_)}; return ValuesIn(array); } @@ -2672,11 +2969,23 @@ class ValueArray49 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, - v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_, v35_, - v36_, v37_, v38_, v39_, v40_, v41_, v42_, v43_, v44_, v45_, v46_, v47_, - v48_, v49_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_), static_cast<T>(v19_), static_cast<T>(v20_), + static_cast<T>(v21_), static_cast<T>(v22_), static_cast<T>(v23_), + static_cast<T>(v24_), static_cast<T>(v25_), static_cast<T>(v26_), + static_cast<T>(v27_), static_cast<T>(v28_), static_cast<T>(v29_), + static_cast<T>(v30_), static_cast<T>(v31_), static_cast<T>(v32_), + static_cast<T>(v33_), static_cast<T>(v34_), static_cast<T>(v35_), + static_cast<T>(v36_), static_cast<T>(v37_), static_cast<T>(v38_), + static_cast<T>(v39_), static_cast<T>(v40_), static_cast<T>(v41_), + static_cast<T>(v42_), static_cast<T>(v43_), static_cast<T>(v44_), + static_cast<T>(v45_), static_cast<T>(v46_), static_cast<T>(v47_), + static_cast<T>(v48_), static_cast<T>(v49_)}; return ValuesIn(array); } @@ -2764,11 +3073,23 @@ class ValueArray50 { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {v1_, v2_, v3_, v4_, v5_, v6_, v7_, v8_, v9_, v10_, v11_, - v12_, v13_, v14_, v15_, v16_, v17_, v18_, v19_, v20_, v21_, v22_, v23_, - v24_, v25_, v26_, v27_, v28_, v29_, v30_, v31_, v32_, v33_, v34_, v35_, - v36_, v37_, v38_, v39_, v40_, v41_, v42_, v43_, v44_, v45_, v46_, v47_, - v48_, v49_, v50_}; + const T array[] = {static_cast<T>(v1_), static_cast<T>(v2_), + static_cast<T>(v3_), static_cast<T>(v4_), static_cast<T>(v5_), + static_cast<T>(v6_), static_cast<T>(v7_), static_cast<T>(v8_), + static_cast<T>(v9_), static_cast<T>(v10_), static_cast<T>(v11_), + static_cast<T>(v12_), static_cast<T>(v13_), static_cast<T>(v14_), + static_cast<T>(v15_), static_cast<T>(v16_), static_cast<T>(v17_), + static_cast<T>(v18_), static_cast<T>(v19_), static_cast<T>(v20_), + static_cast<T>(v21_), static_cast<T>(v22_), static_cast<T>(v23_), + static_cast<T>(v24_), static_cast<T>(v25_), static_cast<T>(v26_), + static_cast<T>(v27_), static_cast<T>(v28_), static_cast<T>(v29_), + static_cast<T>(v30_), static_cast<T>(v31_), static_cast<T>(v32_), + static_cast<T>(v33_), static_cast<T>(v34_), static_cast<T>(v35_), + static_cast<T>(v36_), static_cast<T>(v37_), static_cast<T>(v38_), + static_cast<T>(v39_), static_cast<T>(v40_), static_cast<T>(v41_), + static_cast<T>(v42_), static_cast<T>(v43_), static_cast<T>(v44_), + static_cast<T>(v45_), static_cast<T>(v46_), static_cast<T>(v47_), + static_cast<T>(v48_), static_cast<T>(v49_), static_cast<T>(v50_)}; return ValuesIn(array); } @@ -2836,9 +3157,9 @@ class ValueArray50 { // template <typename T1, typename T2> class CartesianProductGenerator2 - : public ParamGeneratorInterface< ::std::tr1::tuple<T1, T2> > { + : public ParamGeneratorInterface< ::testing::tuple<T1, T2> > { public: - typedef ::std::tr1::tuple<T1, T2> ParamType; + typedef ::testing::tuple<T1, T2> ParamType; CartesianProductGenerator2(const ParamGenerator<T1>& g1, const ParamGenerator<T2>& g2) @@ -2951,9 +3272,9 @@ class CartesianProductGenerator2 template <typename T1, typename T2, typename T3> class CartesianProductGenerator3 - : public ParamGeneratorInterface< ::std::tr1::tuple<T1, T2, T3> > { + : public ParamGeneratorInterface< ::testing::tuple<T1, T2, T3> > { public: - typedef ::std::tr1::tuple<T1, T2, T3> ParamType; + typedef ::testing::tuple<T1, T2, T3> ParamType; CartesianProductGenerator3(const ParamGenerator<T1>& g1, const ParamGenerator<T2>& g2, const ParamGenerator<T3>& g3) @@ -3083,9 +3404,9 @@ class CartesianProductGenerator3 template <typename T1, typename T2, typename T3, typename T4> class CartesianProductGenerator4 - : public ParamGeneratorInterface< ::std::tr1::tuple<T1, T2, T3, T4> > { + : public ParamGeneratorInterface< ::testing::tuple<T1, T2, T3, T4> > { public: - typedef ::std::tr1::tuple<T1, T2, T3, T4> ParamType; + typedef ::testing::tuple<T1, T2, T3, T4> ParamType; CartesianProductGenerator4(const ParamGenerator<T1>& g1, const ParamGenerator<T2>& g2, const ParamGenerator<T3>& g3, @@ -3234,9 +3555,9 @@ class CartesianProductGenerator4 template <typename T1, typename T2, typename T3, typename T4, typename T5> class CartesianProductGenerator5 - : public ParamGeneratorInterface< ::std::tr1::tuple<T1, T2, T3, T4, T5> > { + : public ParamGeneratorInterface< ::testing::tuple<T1, T2, T3, T4, T5> > { public: - typedef ::std::tr1::tuple<T1, T2, T3, T4, T5> ParamType; + typedef ::testing::tuple<T1, T2, T3, T4, T5> ParamType; CartesianProductGenerator5(const ParamGenerator<T1>& g1, const ParamGenerator<T2>& g2, const ParamGenerator<T3>& g3, @@ -3402,10 +3723,10 @@ class CartesianProductGenerator5 template <typename T1, typename T2, typename T3, typename T4, typename T5, typename T6> class CartesianProductGenerator6 - : public ParamGeneratorInterface< ::std::tr1::tuple<T1, T2, T3, T4, T5, + : public ParamGeneratorInterface< ::testing::tuple<T1, T2, T3, T4, T5, T6> > { public: - typedef ::std::tr1::tuple<T1, T2, T3, T4, T5, T6> ParamType; + typedef ::testing::tuple<T1, T2, T3, T4, T5, T6> ParamType; CartesianProductGenerator6(const ParamGenerator<T1>& g1, const ParamGenerator<T2>& g2, const ParamGenerator<T3>& g3, @@ -3588,10 +3909,10 @@ class CartesianProductGenerator6 template <typename T1, typename T2, typename T3, typename T4, typename T5, typename T6, typename T7> class CartesianProductGenerator7 - : public ParamGeneratorInterface< ::std::tr1::tuple<T1, T2, T3, T4, T5, T6, + : public ParamGeneratorInterface< ::testing::tuple<T1, T2, T3, T4, T5, T6, T7> > { public: - typedef ::std::tr1::tuple<T1, T2, T3, T4, T5, T6, T7> ParamType; + typedef ::testing::tuple<T1, T2, T3, T4, T5, T6, T7> ParamType; CartesianProductGenerator7(const ParamGenerator<T1>& g1, const ParamGenerator<T2>& g2, const ParamGenerator<T3>& g3, @@ -3791,10 +4112,10 @@ class CartesianProductGenerator7 template <typename T1, typename T2, typename T3, typename T4, typename T5, typename T6, typename T7, typename T8> class CartesianProductGenerator8 - : public ParamGeneratorInterface< ::std::tr1::tuple<T1, T2, T3, T4, T5, T6, + : public ParamGeneratorInterface< ::testing::tuple<T1, T2, T3, T4, T5, T6, T7, T8> > { public: - typedef ::std::tr1::tuple<T1, T2, T3, T4, T5, T6, T7, T8> ParamType; + typedef ::testing::tuple<T1, T2, T3, T4, T5, T6, T7, T8> ParamType; CartesianProductGenerator8(const ParamGenerator<T1>& g1, const ParamGenerator<T2>& g2, const ParamGenerator<T3>& g3, @@ -4013,10 +4334,10 @@ class CartesianProductGenerator8 template <typename T1, typename T2, typename T3, typename T4, typename T5, typename T6, typename T7, typename T8, typename T9> class CartesianProductGenerator9 - : public ParamGeneratorInterface< ::std::tr1::tuple<T1, T2, T3, T4, T5, T6, + : public ParamGeneratorInterface< ::testing::tuple<T1, T2, T3, T4, T5, T6, T7, T8, T9> > { public: - typedef ::std::tr1::tuple<T1, T2, T3, T4, T5, T6, T7, T8, T9> ParamType; + typedef ::testing::tuple<T1, T2, T3, T4, T5, T6, T7, T8, T9> ParamType; CartesianProductGenerator9(const ParamGenerator<T1>& g1, const ParamGenerator<T2>& g2, const ParamGenerator<T3>& g3, @@ -4252,10 +4573,10 @@ class CartesianProductGenerator9 template <typename T1, typename T2, typename T3, typename T4, typename T5, typename T6, typename T7, typename T8, typename T9, typename T10> class CartesianProductGenerator10 - : public ParamGeneratorInterface< ::std::tr1::tuple<T1, T2, T3, T4, T5, T6, + : public ParamGeneratorInterface< ::testing::tuple<T1, T2, T3, T4, T5, T6, T7, T8, T9, T10> > { public: - typedef ::std::tr1::tuple<T1, T2, T3, T4, T5, T6, T7, T8, T9, T10> ParamType; + typedef ::testing::tuple<T1, T2, T3, T4, T5, T6, T7, T8, T9, T10> ParamType; CartesianProductGenerator10(const ParamGenerator<T1>& g1, const ParamGenerator<T2>& g2, const ParamGenerator<T3>& g3, @@ -4517,8 +4838,8 @@ class CartesianProductHolder2 { CartesianProductHolder2(const Generator1& g1, const Generator2& g2) : g1_(g1), g2_(g2) {} template <typename T1, typename T2> - operator ParamGenerator< ::std::tr1::tuple<T1, T2> >() const { - return ParamGenerator< ::std::tr1::tuple<T1, T2> >( + operator ParamGenerator< ::testing::tuple<T1, T2> >() const { + return ParamGenerator< ::testing::tuple<T1, T2> >( new CartesianProductGenerator2<T1, T2>( static_cast<ParamGenerator<T1> >(g1_), static_cast<ParamGenerator<T2> >(g2_))); @@ -4539,8 +4860,8 @@ CartesianProductHolder3(const Generator1& g1, const Generator2& g2, const Generator3& g3) : g1_(g1), g2_(g2), g3_(g3) {} template <typename T1, typename T2, typename T3> - operator ParamGenerator< ::std::tr1::tuple<T1, T2, T3> >() const { - return ParamGenerator< ::std::tr1::tuple<T1, T2, T3> >( + operator ParamGenerator< ::testing::tuple<T1, T2, T3> >() const { + return ParamGenerator< ::testing::tuple<T1, T2, T3> >( new CartesianProductGenerator3<T1, T2, T3>( static_cast<ParamGenerator<T1> >(g1_), static_cast<ParamGenerator<T2> >(g2_), @@ -4564,8 +4885,8 @@ CartesianProductHolder4(const Generator1& g1, const Generator2& g2, const Generator3& g3, const Generator4& g4) : g1_(g1), g2_(g2), g3_(g3), g4_(g4) {} template <typename T1, typename T2, typename T3, typename T4> - operator ParamGenerator< ::std::tr1::tuple<T1, T2, T3, T4> >() const { - return ParamGenerator< ::std::tr1::tuple<T1, T2, T3, T4> >( + operator ParamGenerator< ::testing::tuple<T1, T2, T3, T4> >() const { + return ParamGenerator< ::testing::tuple<T1, T2, T3, T4> >( new CartesianProductGenerator4<T1, T2, T3, T4>( static_cast<ParamGenerator<T1> >(g1_), static_cast<ParamGenerator<T2> >(g2_), @@ -4591,8 +4912,8 @@ CartesianProductHolder5(const Generator1& g1, const Generator2& g2, const Generator3& g3, const Generator4& g4, const Generator5& g5) : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5) {} template <typename T1, typename T2, typename T3, typename T4, typename T5> - operator ParamGenerator< ::std::tr1::tuple<T1, T2, T3, T4, T5> >() const { - return ParamGenerator< ::std::tr1::tuple<T1, T2, T3, T4, T5> >( + operator ParamGenerator< ::testing::tuple<T1, T2, T3, T4, T5> >() const { + return ParamGenerator< ::testing::tuple<T1, T2, T3, T4, T5> >( new CartesianProductGenerator5<T1, T2, T3, T4, T5>( static_cast<ParamGenerator<T1> >(g1_), static_cast<ParamGenerator<T2> >(g2_), @@ -4622,8 +4943,8 @@ CartesianProductHolder6(const Generator1& g1, const Generator2& g2, : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6) {} template <typename T1, typename T2, typename T3, typename T4, typename T5, typename T6> - operator ParamGenerator< ::std::tr1::tuple<T1, T2, T3, T4, T5, T6> >() const { - return ParamGenerator< ::std::tr1::tuple<T1, T2, T3, T4, T5, T6> >( + operator ParamGenerator< ::testing::tuple<T1, T2, T3, T4, T5, T6> >() const { + return ParamGenerator< ::testing::tuple<T1, T2, T3, T4, T5, T6> >( new CartesianProductGenerator6<T1, T2, T3, T4, T5, T6>( static_cast<ParamGenerator<T1> >(g1_), static_cast<ParamGenerator<T2> >(g2_), @@ -4655,9 +4976,9 @@ CartesianProductHolder7(const Generator1& g1, const Generator2& g2, : g1_(g1), g2_(g2), g3_(g3), g4_(g4), g5_(g5), g6_(g6), g7_(g7) {} template <typename T1, typename T2, typename T3, typename T4, typename T5, typename T6, typename T7> - operator ParamGenerator< ::std::tr1::tuple<T1, T2, T3, T4, T5, T6, + operator ParamGenerator< ::testing::tuple<T1, T2, T3, T4, T5, T6, T7> >() const { - return ParamGenerator< ::std::tr1::tuple<T1, T2, T3, T4, T5, T6, T7> >( + return ParamGenerator< ::testing::tuple<T1, T2, T3, T4, T5, T6, T7> >( new CartesianProductGenerator7<T1, T2, T3, T4, T5, T6, T7>( static_cast<ParamGenerator<T1> >(g1_), static_cast<ParamGenerator<T2> >(g2_), @@ -4693,9 +5014,9 @@ CartesianProductHolder8(const Generator1& g1, const Generator2& g2, g8_(g8) {} template <typename T1, typename T2, typename T3, typename T4, typename T5, typename T6, typename T7, typename T8> - operator ParamGenerator< ::std::tr1::tuple<T1, T2, T3, T4, T5, T6, T7, + operator ParamGenerator< ::testing::tuple<T1, T2, T3, T4, T5, T6, T7, T8> >() const { - return ParamGenerator< ::std::tr1::tuple<T1, T2, T3, T4, T5, T6, T7, T8> >( + return ParamGenerator< ::testing::tuple<T1, T2, T3, T4, T5, T6, T7, T8> >( new CartesianProductGenerator8<T1, T2, T3, T4, T5, T6, T7, T8>( static_cast<ParamGenerator<T1> >(g1_), static_cast<ParamGenerator<T2> >(g2_), @@ -4734,9 +5055,9 @@ CartesianProductHolder9(const Generator1& g1, const Generator2& g2, g9_(g9) {} template <typename T1, typename T2, typename T3, typename T4, typename T5, typename T6, typename T7, typename T8, typename T9> - operator ParamGenerator< ::std::tr1::tuple<T1, T2, T3, T4, T5, T6, T7, T8, + operator ParamGenerator< ::testing::tuple<T1, T2, T3, T4, T5, T6, T7, T8, T9> >() const { - return ParamGenerator< ::std::tr1::tuple<T1, T2, T3, T4, T5, T6, T7, T8, + return ParamGenerator< ::testing::tuple<T1, T2, T3, T4, T5, T6, T7, T8, T9> >( new CartesianProductGenerator9<T1, T2, T3, T4, T5, T6, T7, T8, T9>( static_cast<ParamGenerator<T1> >(g1_), @@ -4778,10 +5099,10 @@ CartesianProductHolder10(const Generator1& g1, const Generator2& g2, g9_(g9), g10_(g10) {} template <typename T1, typename T2, typename T3, typename T4, typename T5, typename T6, typename T7, typename T8, typename T9, typename T10> - operator ParamGenerator< ::std::tr1::tuple<T1, T2, T3, T4, T5, T6, T7, T8, - T9, T10> >() const { - return ParamGenerator< ::std::tr1::tuple<T1, T2, T3, T4, T5, T6, T7, T8, - T9, T10> >( + operator ParamGenerator< ::testing::tuple<T1, T2, T3, T4, T5, T6, T7, T8, T9, + T10> >() const { + return ParamGenerator< ::testing::tuple<T1, T2, T3, T4, T5, T6, T7, T8, T9, + T10> >( new CartesianProductGenerator10<T1, T2, T3, T4, T5, T6, T7, T8, T9, T10>( static_cast<ParamGenerator<T1> >(g1_), diff --git a/tests/gtest/include/gtest/internal/gtest-param-util-generated.h.pump b/tests/gtest/include/gtest/internal/gtest-param-util-generated.h.pump index dbe938630..801a2fc7d 100644 --- a/tests/gtest/include/gtest/internal/gtest-param-util-generated.h.pump +++ b/tests/gtest/include/gtest/internal/gtest-param-util-generated.h.pump @@ -39,7 +39,7 @@ $var maxtuple = 10 $$ Maximum number of Combine arguments we want to support. // and at most $maxtuple arguments in Combine. Please contact // googletestframework@googlegroups.com if you need more. // Please note that the number of arguments to Combine is limited -// by the maximum arity of the implementation of tr1::tuple which is +// by the maximum arity of the implementation of tuple which is // currently set at $maxtuple. #ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_GENERATED_H_ @@ -98,7 +98,7 @@ class ValueArray$i { template <typename T> operator ParamGenerator<T>() const { - const T array[] = {$for j, [[v$(j)_]]}; + const T array[] = {$for j, [[static_cast<T>(v$(j)_)]]}; return ValuesIn(array); } @@ -128,9 +128,9 @@ $range k 2..i template <$for j, [[typename T$j]]> class CartesianProductGenerator$i - : public ParamGeneratorInterface< ::std::tr1::tuple<$for j, [[T$j]]> > { + : public ParamGeneratorInterface< ::testing::tuple<$for j, [[T$j]]> > { public: - typedef ::std::tr1::tuple<$for j, [[T$j]]> ParamType; + typedef ::testing::tuple<$for j, [[T$j]]> ParamType; CartesianProductGenerator$i($for j, [[const ParamGenerator<T$j>& g$j]]) : $for j, [[g$(j)_(g$j)]] {} @@ -269,8 +269,8 @@ class CartesianProductHolder$i { CartesianProductHolder$i($for j, [[const Generator$j& g$j]]) : $for j, [[g$(j)_(g$j)]] {} template <$for j, [[typename T$j]]> - operator ParamGenerator< ::std::tr1::tuple<$for j, [[T$j]]> >() const { - return ParamGenerator< ::std::tr1::tuple<$for j, [[T$j]]> >( + operator ParamGenerator< ::testing::tuple<$for j, [[T$j]]> >() const { + return ParamGenerator< ::testing::tuple<$for j, [[T$j]]> >( new CartesianProductGenerator$i<$for j, [[T$j]]>( $for j,[[ diff --git a/tests/gtest/include/gtest/internal/gtest-param-util.h b/tests/gtest/include/gtest/internal/gtest-param-util.h index 0ef9718cf..d5e1028b0 100644 --- a/tests/gtest/include/gtest/internal/gtest-param-util.h +++ b/tests/gtest/include/gtest/internal/gtest-param-util.h @@ -494,10 +494,10 @@ class ParameterizedTestCaseInfo : public ParameterizedTestCaseInfoBase { const string& instantiation_name = gen_it->first; ParamGenerator<ParamType> generator((*gen_it->second)()); - Message test_case_name_stream; + string test_case_name; if ( !instantiation_name.empty() ) - test_case_name_stream << instantiation_name << "/"; - test_case_name_stream << test_info->test_case_base_name; + test_case_name = instantiation_name + "/"; + test_case_name += test_info->test_case_base_name; int i = 0; for (typename ParamGenerator<ParamType>::iterator param_it = @@ -506,7 +506,7 @@ class ParameterizedTestCaseInfo : public ParameterizedTestCaseInfoBase { Message test_name_stream; test_name_stream << test_info->test_base_name << "/" << i; MakeAndRegisterTestInfo( - test_case_name_stream.GetString().c_str(), + test_case_name.c_str(), test_name_stream.GetString().c_str(), NULL, // No type parameter. PrintToString(*param_it).c_str(), diff --git a/tests/gtest/include/gtest/internal/gtest-port.h b/tests/gtest/include/gtest/internal/gtest-port.h index ce70acdd3..98498309f 100644 --- a/tests/gtest/include/gtest/internal/gtest-port.h +++ b/tests/gtest/include/gtest/internal/gtest-port.h @@ -30,15 +30,43 @@ // Authors: wan@google.com (Zhanyong Wan) // // Low-level types and utilities for porting Google Test to various -// platforms. They are subject to change without notice. DO NOT USE -// THEM IN USER CODE. +// platforms. All macros ending with _ and symbols defined in an +// internal namespace are subject to change without notice. Code +// outside Google Test MUST NOT USE THEM DIRECTLY. Macros that don't +// end with _ are part of Google Test's public API and can be used by +// code outside Google Test. +// +// This file is fundamental to Google Test. All other Google Test source +// files are expected to #include this. Therefore, it cannot #include +// any other Google Test header. #ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_ #define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_ -// The user can define the following macros in the build script to -// control Google Test's behavior. If the user doesn't define a macro -// in this list, Google Test will define it. +// Environment-describing macros +// ----------------------------- +// +// Google Test can be used in many different environments. Macros in +// this section tell Google Test what kind of environment it is being +// used in, such that Google Test can provide environment-specific +// features and implementations. +// +// Google Test tries to automatically detect the properties of its +// environment, so users usually don't need to worry about these +// macros. However, the automatic detection is not perfect. +// Sometimes it's necessary for a user to define some of the following +// macros in the build script to override Google Test's decisions. +// +// If the user doesn't define a macro in the list, Google Test will +// provide a default definition. After this header is #included, all +// macros in this list will be defined to either 1 or 0. +// +// Notes to maintainers: +// - Each macro here is a user-tweakable knob; do not grow the list +// lightly. +// - Use #if to key off these macros. Don't use #ifdef or "#if +// defined(...)", which will not work as these macros are ALWAYS +// defined. // // GTEST_HAS_CLONE - Define it to 1/0 to indicate that clone(2) // is/isn't available. @@ -72,6 +100,8 @@ // Test's own tr1 tuple implementation should be // used. Unused when the user sets // GTEST_HAS_TR1_TUPLE to 0. +// GTEST_LANG_CXX11 - Define it to 1/0 to indicate that Google Test +// is building in C++11/C++98 mode. // GTEST_LINKED_AS_SHARED_LIBRARY // - Define to 1 when compiling tests that use // Google Test as a shared library (known as @@ -80,23 +110,34 @@ // - Define to 1 when compiling Google Test itself // as a shared library. -// This header defines the following utilities: +// Platform-indicating macros +// -------------------------- +// +// Macros indicating the platform on which Google Test is being used +// (a macro is defined to 1 if compiled on the given platform; +// otherwise UNDEFINED -- it's never defined to 0.). Google Test +// defines these macros automatically. Code outside Google Test MUST +// NOT define them. // -// Macros indicating the current platform (defined to 1 if compiled on -// the given platform; otherwise undefined): // GTEST_OS_AIX - IBM AIX // GTEST_OS_CYGWIN - Cygwin +// GTEST_OS_FREEBSD - FreeBSD // GTEST_OS_HPUX - HP-UX // GTEST_OS_LINUX - Linux // GTEST_OS_LINUX_ANDROID - Google Android // GTEST_OS_MAC - Mac OS X +// GTEST_OS_IOS - iOS // GTEST_OS_NACL - Google Native Client (NaCl) +// GTEST_OS_OPENBSD - OpenBSD +// GTEST_OS_QNX - QNX // GTEST_OS_SOLARIS - Sun Solaris // GTEST_OS_SYMBIAN - Symbian // GTEST_OS_WINDOWS - Windows (Desktop, MinGW, or Mobile) // GTEST_OS_WINDOWS_DESKTOP - Windows Desktop // GTEST_OS_WINDOWS_MINGW - MinGW // GTEST_OS_WINDOWS_MOBILE - Windows Mobile +// GTEST_OS_WINDOWS_PHONE - Windows Phone +// GTEST_OS_WINDOWS_RT - Windows Store App/WinRT // GTEST_OS_ZOS - z/OS // // Among the platforms, Cygwin, Linux, Max OS X, and Windows have the @@ -106,22 +147,50 @@ // googletestframework@googlegroups.com (patches for fixing them are // even more welcome!). // -// Note that it is possible that none of the GTEST_OS_* macros are defined. +// It is possible that none of the GTEST_OS_* macros are defined. + +// Feature-indicating macros +// ------------------------- +// +// Macros indicating which Google Test features are available (a macro +// is defined to 1 if the corresponding feature is supported; +// otherwise UNDEFINED -- it's never defined to 0.). Google Test +// defines these macros automatically. Code outside Google Test MUST +// NOT define them. +// +// These macros are public so that portable tests can be written. +// Such tests typically surround code using a feature with an #if +// which controls that code. For example: +// +// #if GTEST_HAS_DEATH_TEST +// EXPECT_DEATH(DoSomethingDeadly()); +// #endif // -// Macros indicating available Google Test features (defined to 1 if -// the corresponding feature is supported; otherwise undefined): // GTEST_HAS_COMBINE - the Combine() function (for value-parameterized // tests) // GTEST_HAS_DEATH_TEST - death tests // GTEST_HAS_PARAM_TEST - value-parameterized tests // GTEST_HAS_TYPED_TEST - typed tests // GTEST_HAS_TYPED_TEST_P - type-parameterized tests +// GTEST_IS_THREADSAFE - Google Test is thread-safe. // GTEST_USES_POSIX_RE - enhanced POSIX regex is used. Do not confuse with // GTEST_HAS_POSIX_RE (see above) which users can // define themselves. // GTEST_USES_SIMPLE_RE - our own simple regex is used; // the above two are mutually exclusive. // GTEST_CAN_COMPARE_NULL - accepts untyped NULL in EXPECT_EQ(). + +// Misc public macros +// ------------------ +// +// GTEST_FLAG(flag_name) - references the variable corresponding to +// the given Google Test flag. + +// Internal utilities +// ------------------ +// +// The following macros and utilities are for Google Test's INTERNAL +// use only. Code outside Google Test MUST NOT USE THEM DIRECTLY. // // Macros for basic C++ coding: // GTEST_AMBIGUOUS_ELSE_BLOCKER_ - for disabling a gcc warning. @@ -130,13 +199,18 @@ // GTEST_DISALLOW_ASSIGN_ - disables operator=. // GTEST_DISALLOW_COPY_AND_ASSIGN_ - disables copy ctor and operator=. // GTEST_MUST_USE_RESULT_ - declares that a function's result must be used. +// GTEST_INTENTIONAL_CONST_COND_PUSH_ - start code section where MSVC C4127 is +// suppressed (constant conditional). +// GTEST_INTENTIONAL_CONST_COND_POP_ - finish code section where MSVC C4127 +// is suppressed. +// +// C++11 feature wrappers: +// +// testing::internal::move - portability wrapper for std::move. // // Synchronization: // Mutex, MutexLock, ThreadLocal, GetThreadCount() -// - synchronization primitives. -// GTEST_IS_THREADSAFE - defined to 1 to indicate that the above -// synchronization primitives have real implementations -// and Google Test is thread-safe; or 0 otherwise. +// - synchronization primitives. // // Template meta programming: // is_pointer - as in TR1; needed on Symbian and IBM XL C/C++ only. @@ -172,10 +246,9 @@ // BiggestInt - the biggest signed integer type. // // Command-line utilities: -// GTEST_FLAG() - references a flag. // GTEST_DECLARE_*() - declares a flag. // GTEST_DEFINE_*() - defines a flag. -// GetArgvs() - returns the command line as a vector of strings. +// GetInjectableArgvs() - returns the command line as a vector of strings. // // Environment variable utilities: // GetEnv() - gets the value of an environment variable. @@ -188,15 +261,21 @@ #include <stdlib.h> #include <stdio.h> #include <string.h> -#include <unistd.h> #ifndef _WIN32_WCE # include <sys/types.h> # include <sys/stat.h> #endif // !_WIN32_WCE +#if defined __APPLE__ +# include <AvailabilityMacros.h> +# include <TargetConditionals.h> +#endif + +#include <algorithm> // NOLINT #include <iostream> // NOLINT #include <sstream> // NOLINT #include <string> // NOLINT +#include <utility> #define GTEST_DEV_EMAIL_ "googletestframework@@googlegroups.com" #define GTEST_FLAG_PREFIX_ "gtest_" @@ -223,19 +302,34 @@ # define GTEST_OS_WINDOWS_MOBILE 1 # elif defined(__MINGW__) || defined(__MINGW32__) # define GTEST_OS_WINDOWS_MINGW 1 +# elif defined(WINAPI_FAMILY) +# include <winapifamily.h> +# if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) +# define GTEST_OS_WINDOWS_DESKTOP 1 +# elif WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_PHONE_APP) +# define GTEST_OS_WINDOWS_PHONE 1 +# elif WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP) +# define GTEST_OS_WINDOWS_RT 1 +# else + // WINAPI_FAMILY defined but no known partition matched. + // Default to desktop. +# define GTEST_OS_WINDOWS_DESKTOP 1 +# endif # else # define GTEST_OS_WINDOWS_DESKTOP 1 # endif // _WIN32_WCE #elif defined __APPLE__ # define GTEST_OS_MAC 1 +# if TARGET_OS_IPHONE +# define GTEST_OS_IOS 1 +# endif +#elif defined __FreeBSD__ +# define GTEST_OS_FREEBSD 1 #elif defined __linux__ # define GTEST_OS_LINUX 1 -# ifdef ANDROID +# if defined __ANDROID__ # define GTEST_OS_LINUX_ANDROID 1 -# endif // ANDROID -#elif defined __FreeBSD__ -# define GTEST_OS_LINUX 1 -# define GTEST_HAS_CLONE 0 +# endif #elif defined __MVS__ # define GTEST_OS_ZOS 1 #elif defined(__sun) && defined(__SVR4) @@ -246,30 +340,130 @@ # define GTEST_OS_HPUX 1 #elif defined __native_client__ # define GTEST_OS_NACL 1 +#elif defined __OpenBSD__ +# define GTEST_OS_OPENBSD 1 +#elif defined __QNX__ +# define GTEST_OS_QNX 1 #endif // __CYGWIN__ +// Macros for disabling Microsoft Visual C++ warnings. +// +// GTEST_DISABLE_MSC_WARNINGS_PUSH_(4800 4385) +// /* code that triggers warnings C4800 and C4385 */ +// GTEST_DISABLE_MSC_WARNINGS_POP_() +#if _MSC_VER >= 1500 +# define GTEST_DISABLE_MSC_WARNINGS_PUSH_(warnings) \ + __pragma(warning(push)) \ + __pragma(warning(disable: warnings)) +# define GTEST_DISABLE_MSC_WARNINGS_POP_() \ + __pragma(warning(pop)) +#else +// Older versions of MSVC don't have __pragma. +# define GTEST_DISABLE_MSC_WARNINGS_PUSH_(warnings) +# define GTEST_DISABLE_MSC_WARNINGS_POP_() +#endif + +#ifndef GTEST_LANG_CXX11 +// gcc and clang define __GXX_EXPERIMENTAL_CXX0X__ when +// -std={c,gnu}++{0x,11} is passed. The C++11 standard specifies a +// value for __cplusplus, and recent versions of clang, gcc, and +// probably other compilers set that too in C++11 mode. +# if __GXX_EXPERIMENTAL_CXX0X__ || __cplusplus >= 201103L +// Compiling in at least C++11 mode. +# define GTEST_LANG_CXX11 1 +# else +# define GTEST_LANG_CXX11 0 +# endif +#endif + +// Distinct from C++11 language support, some environments don't provide +// proper C++11 library support. Notably, it's possible to build in +// C++11 mode when targeting Mac OS X 10.6, which has an old libstdc++ +// with no C++11 support. +// +// libstdc++ has sufficient C++11 support as of GCC 4.6.0, __GLIBCXX__ +// 20110325, but maintenance releases in the 4.4 and 4.5 series followed +// this date, so check for those versions by their date stamps. +// https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html#abi.versioning +#if GTEST_LANG_CXX11 && \ + (!defined(__GLIBCXX__) || ( \ + __GLIBCXX__ >= 20110325ul && /* GCC >= 4.6.0 */ \ + /* Blacklist of patch releases of older branches: */ \ + __GLIBCXX__ != 20110416ul && /* GCC 4.4.6 */ \ + __GLIBCXX__ != 20120313ul && /* GCC 4.4.7 */ \ + __GLIBCXX__ != 20110428ul && /* GCC 4.5.3 */ \ + __GLIBCXX__ != 20120702ul)) /* GCC 4.5.4 */ +# define GTEST_STDLIB_CXX11 1 +#endif + +// Only use C++11 library features if the library provides them. +#if GTEST_STDLIB_CXX11 +# define GTEST_HAS_STD_BEGIN_AND_END_ 1 +# define GTEST_HAS_STD_FORWARD_LIST_ 1 +# define GTEST_HAS_STD_FUNCTION_ 1 +# define GTEST_HAS_STD_INITIALIZER_LIST_ 1 +# define GTEST_HAS_STD_MOVE_ 1 +# define GTEST_HAS_STD_UNIQUE_PTR_ 1 +#endif + +// C++11 specifies that <tuple> provides std::tuple. +// Some platforms still might not have it, however. +#if GTEST_LANG_CXX11 +# define GTEST_HAS_STD_TUPLE_ 1 +# if defined(__clang__) +// Inspired by http://clang.llvm.org/docs/LanguageExtensions.html#__has_include +# if defined(__has_include) && !__has_include(<tuple>) +# undef GTEST_HAS_STD_TUPLE_ +# endif +# elif defined(_MSC_VER) +// Inspired by boost/config/stdlib/dinkumware.hpp +# if defined(_CPPLIB_VER) && _CPPLIB_VER < 520 +# undef GTEST_HAS_STD_TUPLE_ +# endif +# elif defined(__GLIBCXX__) +// Inspired by boost/config/stdlib/libstdcpp3.hpp, +// http://gcc.gnu.org/gcc-4.2/changes.html and +// http://gcc.gnu.org/onlinedocs/libstdc++/manual/bk01pt01ch01.html#manual.intro.status.standard.200x +# if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 2) +# undef GTEST_HAS_STD_TUPLE_ +# endif +# endif +#endif + // Brings in definitions for functions used in the testing::internal::posix // namespace (read, write, close, chdir, isatty, stat). We do not currently // use them on Windows Mobile. -#if !GTEST_OS_WINDOWS +#if GTEST_OS_WINDOWS +# if !GTEST_OS_WINDOWS_MOBILE +# include <direct.h> +# include <io.h> +# endif +// In order to avoid having to include <windows.h>, use forward declaration +// assuming CRITICAL_SECTION is a typedef of _RTL_CRITICAL_SECTION. +// This assumption is verified by +// WindowsTypesTest.CRITICAL_SECTIONIs_RTL_CRITICAL_SECTION. +struct _RTL_CRITICAL_SECTION; +#else // This assumes that non-Windows OSes provide unistd.h. For OSes where this // is not the case, we need to include headers that provide the functions // mentioned above. # include <unistd.h> -# if !GTEST_OS_NACL -// TODO(vladl@google.com): Remove this condition when Native Client SDK adds -// strings.h (tracked in -// http://code.google.com/p/nativeclient/issues/detail?id=1175). -# include <strings.h> // Native Client doesn't provide strings.h. -# endif -#elif !GTEST_OS_WINDOWS_MOBILE -# include <direct.h> -# include <io.h> +# include <strings.h> +#endif // GTEST_OS_WINDOWS + +#if GTEST_OS_LINUX_ANDROID +// Used to define __ANDROID_API__ matching the target NDK API level. +# include <android/api-level.h> // NOLINT #endif // Defines this to true iff Google Test can use POSIX regular expressions. #ifndef GTEST_HAS_POSIX_RE -# define GTEST_HAS_POSIX_RE (!GTEST_OS_WINDOWS) +# if GTEST_OS_LINUX_ANDROID +// On Android, <regex.h> is only available starting with Gingerbread. +# define GTEST_HAS_POSIX_RE (__ANDROID_API__ >= 9) +# else +# define GTEST_HAS_POSIX_RE (!GTEST_OS_WINDOWS) +# endif #endif #if GTEST_HAS_POSIX_RE @@ -307,6 +501,15 @@ # define _HAS_EXCEPTIONS 1 # endif // _HAS_EXCEPTIONS # define GTEST_HAS_EXCEPTIONS _HAS_EXCEPTIONS +# elif defined(__clang__) +// clang defines __EXCEPTIONS iff exceptions are enabled before clang 220714, +// but iff cleanups are enabled after that. In Obj-C++ files, there can be +// cleanups for ObjC exceptions which also need cleanups, even if C++ exceptions +// are disabled. clang has __has_feature(cxx_exceptions) which checks for C++ +// exceptions starting at clang r206352, but which checked for cleanups prior to +// that. To reliably check for C++ exception availability with clang, check for +// __EXCEPTIONS && __has_feature(cxx_exceptions). +# define GTEST_HAS_EXCEPTIONS (__EXCEPTIONS && __has_feature(cxx_exceptions)) # elif defined(__GNUC__) && __EXCEPTIONS // gcc defines __EXCEPTIONS to 1 iff exceptions are enabled. # define GTEST_HAS_EXCEPTIONS 1 @@ -384,11 +587,27 @@ # elif defined(__GNUC__) && (GTEST_GCC_VER_ >= 40302) # ifdef __GXX_RTTI -# define GTEST_HAS_RTTI 1 +// When building against STLport with the Android NDK and with +// -frtti -fno-exceptions, the build fails at link time with undefined +// references to __cxa_bad_typeid. Note sure if STL or toolchain bug, +// so disable RTTI when detected. +# if GTEST_OS_LINUX_ANDROID && defined(_STLPORT_MAJOR) && \ + !defined(__EXCEPTIONS) +# define GTEST_HAS_RTTI 0 +# else +# define GTEST_HAS_RTTI 1 +# endif // GTEST_OS_LINUX_ANDROID && __STLPORT_MAJOR && !__EXCEPTIONS # else # define GTEST_HAS_RTTI 0 # endif // __GXX_RTTI +// Clang defines __GXX_RTTI starting with version 3.0, but its manual recommends +// using has_feature instead. has_feature(cxx_rtti) is supported since 2.7, the +// first version with C++ support. +# elif defined(__clang__) + +# define GTEST_HAS_RTTI __has_feature(cxx_rtti) + // Starting with version 9.0 IBM Visual Age defines __RTTI_ALL__ to 1 if // both the typeid and dynamic_cast features are present. # elif defined(__IBMCPP__) && (__IBMCPP__ >= 900) @@ -416,12 +635,13 @@ // Determines whether Google Test can use the pthreads library. #ifndef GTEST_HAS_PTHREAD -// The user didn't tell us explicitly, so we assume pthreads support is -// available on Linux and Mac. +// The user didn't tell us explicitly, so we make reasonable assumptions about +// which platforms have pthreads support. // // To disable threading support in Google Test, add -DGTEST_HAS_PTHREAD=0 // to your compiler flags. -# define GTEST_HAS_PTHREAD (GTEST_OS_LINUX || GTEST_OS_MAC || GTEST_OS_HPUX) +# define GTEST_HAS_PTHREAD (GTEST_OS_LINUX || GTEST_OS_MAC || GTEST_OS_HPUX \ + || GTEST_OS_QNX || GTEST_OS_FREEBSD || GTEST_OS_NACL) #endif // GTEST_HAS_PTHREAD #if GTEST_HAS_PTHREAD @@ -437,8 +657,13 @@ // this macro to 0 to prevent Google Test from using tuple (any // feature depending on tuple with be disabled in this mode). #ifndef GTEST_HAS_TR1_TUPLE +# if GTEST_OS_LINUX_ANDROID && defined(_STLPORT_MAJOR) +// STLport, provided with the Android NDK, has neither <tr1/tuple> or <tuple>. +# define GTEST_HAS_TR1_TUPLE 0 +# else // The user didn't tell us not to do it, so we assume it's OK. -# define GTEST_HAS_TR1_TUPLE 1 +# define GTEST_HAS_TR1_TUPLE 1 +# endif #endif // GTEST_HAS_TR1_TUPLE // Determines whether Google Test's own tr1 tuple implementation @@ -447,14 +672,28 @@ // The user didn't tell us, so we need to figure it out. // We use our own TR1 tuple if we aren't sure the user has an -// implementation of it already. At this time, GCC 4.0.0+ and MSVC -// 2010 are the only mainstream compilers that come with a TR1 tuple -// implementation. NVIDIA's CUDA NVCC compiler pretends to be GCC by -// defining __GNUC__ and friends, but cannot compile GCC's tuple -// implementation. MSVC 2008 (9.0) provides TR1 tuple in a 323 MB -// Feature Pack download, which we cannot assume the user has. -# if (defined(__GNUC__) && !defined(__CUDACC__) && !defined(_LIBCPP_VERSION) && (GTEST_GCC_VER_ >= 40000)) \ - || _MSC_VER >= 1600 +// implementation of it already. At this time, libstdc++ 4.0.0+ and +// MSVC 2010 are the only mainstream standard libraries that come +// with a TR1 tuple implementation. NVIDIA's CUDA NVCC compiler +// pretends to be GCC by defining __GNUC__ and friends, but cannot +// compile GCC's tuple implementation. MSVC 2008 (9.0) provides TR1 +// tuple in a 323 MB Feature Pack download, which we cannot assume the +// user has. QNX's QCC compiler is a modified GCC but it doesn't +// support TR1 tuple. libc++ only provides std::tuple, in C++11 mode, +// and it can be used with some compilers that define __GNUC__. +# if (defined(__GNUC__) && !defined(__CUDACC__) && (GTEST_GCC_VER_ >= 40000) \ + && !GTEST_OS_QNX && !defined(_LIBCPP_VERSION)) || _MSC_VER >= 1600 +# define GTEST_ENV_HAS_TR1_TUPLE_ 1 +# endif + +// C++11 specifies that <tuple> provides std::tuple. Use that if gtest is used +// in C++11 mode and libstdc++ isn't very old (binaries targeting OS X 10.6 +// can build with clang but need to use gcc4.2's libstdc++). +# if GTEST_LANG_CXX11 && (!defined(__GLIBCXX__) || __GLIBCXX__ > 20110325) +# define GTEST_ENV_HAS_STD_TUPLE_ 1 +# endif + +# if GTEST_ENV_HAS_TR1_TUPLE_ || GTEST_ENV_HAS_STD_TUPLE_ # define GTEST_USE_OWN_TR1_TUPLE 0 # else # define GTEST_USE_OWN_TR1_TUPLE 1 @@ -464,11 +703,37 @@ // To avoid conditional compilation everywhere, we make it // gtest-port.h's responsibility to #include the header implementing -// tr1/tuple. +// tuple. +#if GTEST_HAS_STD_TUPLE_ +# include <tuple> // IWYU pragma: export +# define GTEST_TUPLE_NAMESPACE_ ::std +#endif // GTEST_HAS_STD_TUPLE_ + +// We include tr1::tuple even if std::tuple is available to define printers for +// them. #if GTEST_HAS_TR1_TUPLE +# ifndef GTEST_TUPLE_NAMESPACE_ +# define GTEST_TUPLE_NAMESPACE_ ::std::tr1 +# endif // GTEST_TUPLE_NAMESPACE_ # if GTEST_USE_OWN_TR1_TUPLE -# include "gtest/internal/gtest-tuple.h" +# include "gtest/internal/gtest-tuple.h" // IWYU pragma: export // NOLINT +# elif GTEST_ENV_HAS_STD_TUPLE_ +# include <tuple> +// C++11 puts its tuple into the ::std namespace rather than +// ::std::tr1. gtest expects tuple to live in ::std::tr1, so put it there. +// This causes undefined behavior, but supported compilers react in +// the way we intend. +namespace std { +namespace tr1 { +using ::std::get; +using ::std::make_tuple; +using ::std::tuple; +using ::std::tuple_element; +using ::std::tuple_size; +} +} + # elif GTEST_OS_SYMBIAN // On Symbian, BOOST_HAS_TR1_TUPLE causes Boost's TR1 tuple library to @@ -483,7 +748,7 @@ // This prevents <boost/tr1/detail/config.hpp>, which defines // BOOST_HAS_TR1_TUPLE, from being #included by Boost's <tuple>. # define BOOST_TR1_DETAIL_CONFIG_HPP_INCLUDED -# include <tuple> +# include <tuple> // IWYU pragma: export // NOLINT # elif defined(__GNUC__) && (GTEST_GCC_VER_ >= 40000) // GCC 4.0+ implements tr1/tuple in the <tr1/tuple> header. This does @@ -506,7 +771,7 @@ # else // If the compiler is not GCC 4.0+, we assume the user is using a // spec-conforming TR1 implementation. -# include <tuple> // NOLINT +# include <tuple> // IWYU pragma: export // NOLINT # endif // GTEST_USE_OWN_TR1_TUPLE #endif // GTEST_HAS_TR1_TUPLE @@ -519,7 +784,16 @@ // The user didn't tell us, so we need to figure it out. # if GTEST_OS_LINUX && !defined(__ia64__) -# define GTEST_HAS_CLONE 1 +# if GTEST_OS_LINUX_ANDROID +// On Android, clone() is only available on ARM starting with Gingerbread. +# if defined(__arm__) && __ANDROID_API__ >= 9 +# define GTEST_HAS_CLONE 1 +# else +# define GTEST_HAS_CLONE 0 +# endif +# else +# define GTEST_HAS_CLONE 1 +# endif # else # define GTEST_HAS_CLONE 0 # endif // GTEST_OS_LINUX && !defined(__ia64__) @@ -531,7 +805,8 @@ #ifndef GTEST_HAS_STREAM_REDIRECTION // By default, we assume that stream redirection is supported on all // platforms except known mobile ones. -# if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_SYMBIAN +# if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_SYMBIAN || \ + GTEST_OS_WINDOWS_PHONE || GTEST_OS_WINDOWS_RT # define GTEST_HAS_STREAM_REDIRECTION 0 # else # define GTEST_HAS_STREAM_REDIRECTION 1 @@ -542,9 +817,11 @@ // Google Test does not support death tests for VC 7.1 and earlier as // abort() in a VC 7.1 application compiled as GUI in debug config // pops up a dialog window that cannot be suppressed programmatically. -#if (GTEST_OS_LINUX || GTEST_OS_MAC || GTEST_OS_CYGWIN || GTEST_OS_SOLARIS || \ +#if (GTEST_OS_LINUX || GTEST_OS_CYGWIN || GTEST_OS_SOLARIS || \ + (GTEST_OS_MAC && !GTEST_OS_IOS) || \ (GTEST_OS_WINDOWS_DESKTOP && _MSC_VER >= 1400) || \ - GTEST_OS_WINDOWS_MINGW || GTEST_OS_AIX || GTEST_OS_HPUX) + GTEST_OS_WINDOWS_MINGW || GTEST_OS_AIX || GTEST_OS_HPUX || \ + GTEST_OS_OPENBSD || GTEST_OS_QNX || GTEST_OS_FREEBSD) # define GTEST_HAS_DEATH_TEST 1 # include <vector> // NOLINT #endif @@ -610,7 +887,12 @@ // compiler the variable/parameter does not have to be used. #if defined(__GNUC__) && !defined(COMPILER_ICC) # define GTEST_ATTRIBUTE_UNUSED_ __attribute__ ((unused)) -#else +#elif defined(__clang__) +# if __has_attribute(unused) +# define GTEST_ATTRIBUTE_UNUSED_ __attribute__ ((unused)) +# endif +#endif +#ifndef GTEST_ATTRIBUTE_UNUSED_ # define GTEST_ATTRIBUTE_UNUSED_ #endif @@ -636,6 +918,19 @@ # define GTEST_MUST_USE_RESULT_ #endif // __GNUC__ && (GTEST_GCC_VER_ >= 30400) && !COMPILER_ICC +// MS C++ compiler emits warning when a conditional expression is compile time +// constant. In some contexts this warning is false positive and needs to be +// suppressed. Use the following two macros in such cases: +// +// GTEST_INTENTIONAL_CONST_COND_PUSH_() +// while (true) { +// GTEST_INTENTIONAL_CONST_COND_POP_() +// } +# define GTEST_INTENTIONAL_CONST_COND_PUSH_() \ + GTEST_DISABLE_MSC_WARNINGS_PUSH_(4127) +# define GTEST_INTENTIONAL_CONST_COND_POP_() \ + GTEST_DISABLE_MSC_WARNINGS_POP_() + // Determine whether the compiler supports Microsoft's Structured Exception // Handling. This is supported by several Windows compilers but generally // does not exist on any other system. @@ -650,6 +945,11 @@ # define GTEST_HAS_SEH 0 # endif +#define GTEST_IS_THREADSAFE \ + (0 \ + || (GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT) \ + || GTEST_HAS_PTHREAD) + #endif // GTEST_HAS_SEH #ifdef _MSC_VER @@ -673,20 +973,78 @@ # define GTEST_NO_INLINE_ #endif +// _LIBCPP_VERSION is defined by the libc++ library from the LLVM project. +#if defined(__GLIBCXX__) || defined(_LIBCPP_VERSION) +# define GTEST_HAS_CXXABI_H_ 1 +#else +# define GTEST_HAS_CXXABI_H_ 0 +#endif + +// A function level attribute to disable checking for use of uninitialized +// memory when built with MemorySanitizer. +#if defined(__clang__) +# if __has_feature(memory_sanitizer) +# define GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ \ + __attribute__((no_sanitize_memory)) +# else +# define GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ +# endif // __has_feature(memory_sanitizer) +#else +# define GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ +#endif // __clang__ + +// A function level attribute to disable AddressSanitizer instrumentation. +#if defined(__clang__) +# if __has_feature(address_sanitizer) +# define GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ \ + __attribute__((no_sanitize_address)) +# else +# define GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ +# endif // __has_feature(address_sanitizer) +#else +# define GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ +#endif // __clang__ + +// A function level attribute to disable ThreadSanitizer instrumentation. +#if defined(__clang__) +# if __has_feature(thread_sanitizer) +# define GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ \ + __attribute__((no_sanitize_thread)) +# else +# define GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ +# endif // __has_feature(thread_sanitizer) +#else +# define GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ +#endif // __clang__ + namespace testing { class Message; +#if defined(GTEST_TUPLE_NAMESPACE_) +// Import tuple and friends into the ::testing namespace. +// It is part of our interface, having them in ::testing allows us to change +// their types as needed. +using GTEST_TUPLE_NAMESPACE_::get; +using GTEST_TUPLE_NAMESPACE_::make_tuple; +using GTEST_TUPLE_NAMESPACE_::tuple; +using GTEST_TUPLE_NAMESPACE_::tuple_size; +using GTEST_TUPLE_NAMESPACE_::tuple_element; +#endif // defined(GTEST_TUPLE_NAMESPACE_) + namespace internal { -class String; +// A secret type that Google Test users don't know about. It has no +// definition on purpose. Therefore it's impossible to create a +// Secret object, which is what we want. +class Secret; // The GTEST_COMPILE_ASSERT_ macro can be used to verify that a compile time // expression is true. For example, you could use it to verify the // size of a static array: // -// GTEST_COMPILE_ASSERT_(ARRAYSIZE(content_type_names) == CONTENT_NUM_TYPES, -// content_type_names_incorrect_size); +// GTEST_COMPILE_ASSERT_(GTEST_ARRAY_SIZE_(names) == NUM_NAMES, +// names_incorrect_size); // // or to make sure a struct is smaller than a certain size: // @@ -696,16 +1054,22 @@ class String; // the expression is false, most compilers will issue a warning/error // containing the name of the variable. +#if GTEST_LANG_CXX11 +# define GTEST_COMPILE_ASSERT_(expr, msg) static_assert(expr, #msg) +#else // !GTEST_LANG_CXX11 template <bool> -struct CompileAssert { + struct CompileAssert { }; -#define GTEST_COMPILE_ASSERT_(expr, msg) \ - typedef ::testing::internal::CompileAssert<(bool(expr))> \ - msg[bool(expr) ? 1 : -1] +# define GTEST_COMPILE_ASSERT_(expr, msg) \ + typedef ::testing::internal::CompileAssert<(static_cast<bool>(expr))> \ + msg[static_cast<bool>(expr) ? 1 : -1] GTEST_ATTRIBUTE_UNUSED_ +#endif // !GTEST_LANG_CXX11 // Implementation details of GTEST_COMPILE_ASSERT_: // +// (In C++11, we simply use static_assert instead of the following) +// // - GTEST_COMPILE_ASSERT_ works by defining an array type that has -1 // elements (and thus is invalid) when the expression is false. // @@ -752,7 +1116,12 @@ template <typename T1, typename T2> struct StaticAssertTypeEqHelper; template <typename T> -struct StaticAssertTypeEqHelper<T, T> {}; +struct StaticAssertTypeEqHelper<T, T> { + enum { value = true }; +}; + +// Evaluates to the number of elements in 'array'. +#define GTEST_ARRAY_SIZE_(array) (sizeof(array) / sizeof(array[0])) #if GTEST_HAS_GLOBAL_STRING typedef ::string string; @@ -800,6 +1169,12 @@ class scoped_ptr { ptr_ = p; } } + + friend void swap(scoped_ptr& a, scoped_ptr& b) { + using std::swap; + swap(a.ptr_, b.ptr_); + } + private: T* ptr_; @@ -862,10 +1237,9 @@ class GTEST_API_ RE { private: void Init(const char* regex); - // We use a const char* instead of a string, as Google Test may be used - // where string is not available. We also do not use Google Test's own - // String type here, in order to simplify dependencies between the - // files. + // We use a const char* instead of an std::string, as Google Test used to be + // used where std::string is not available. TODO(wan@google.com): change to + // std::string. const char* pattern_; bool is_valid_; @@ -962,6 +1336,15 @@ inline void FlushInfoLog() { fflush(NULL); } GTEST_LOG_(FATAL) << #posix_call << "failed with error " \ << gtest_error +#if GTEST_HAS_STD_MOVE_ +using std::move; +#else // GTEST_HAS_STD_MOVE_ +template <typename T> +const T& move(const T& t) { + return t; +} +#endif // GTEST_HAS_STD_MOVE_ + // INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. // // Use ImplicitCast_ as a safe version of static_cast for upcasting in @@ -983,7 +1366,7 @@ inline void FlushInfoLog() { fflush(NULL); } // similar functions users may have (e.g., implicit_cast). The internal // namespace alone is not enough because the function can be found by ADL. template<typename To> -inline To ImplicitCast_(To x) { return x; } +inline To ImplicitCast_(To x) { return ::testing::internal::move(x); } // When you upcast (that is, cast a pointer from type Foo to type // SuperclassOfFoo), it's fine to use ImplicitCast_<>, since upcasts @@ -1012,7 +1395,9 @@ inline To DownCast_(From* f) { // so we only accept pointers // for compile-time type checking, and has no overhead in an // optimized build at run-time, as it will be optimized away // completely. + GTEST_INTENTIONAL_CONST_COND_PUSH_() if (false) { + GTEST_INTENTIONAL_CONST_COND_POP_() const To to = NULL; ::testing::internal::ImplicitCast_<From*>(to); } @@ -1048,30 +1433,30 @@ Derived* CheckedDowncastToActualType(Base* base) { // GetCapturedStderr - stops capturing stderr and returns the captured string. // GTEST_API_ void CaptureStdout(); -GTEST_API_ String GetCapturedStdout(); +GTEST_API_ std::string GetCapturedStdout(); GTEST_API_ void CaptureStderr(); -GTEST_API_ String GetCapturedStderr(); +GTEST_API_ std::string GetCapturedStderr(); #endif // GTEST_HAS_STREAM_REDIRECTION #if GTEST_HAS_DEATH_TEST -// A copy of all command line arguments. Set by InitGoogleTest(). -extern ::std::vector<String> g_argvs; +const ::std::vector<testing::internal::string>& GetInjectableArgvs(); +void SetInjectableArgvs(const ::std::vector<testing::internal::string>* + new_argvs); -// GTEST_HAS_DEATH_TEST implies we have ::std::string. -const ::std::vector<String>& GetArgvs(); +// A copy of all command line arguments. Set by InitGoogleTest(). +extern ::std::vector<testing::internal::string> g_argvs; #endif // GTEST_HAS_DEATH_TEST // Defines synchronization primitives. - -#if GTEST_HAS_PTHREAD - -// Sleeps for (roughly) n milli-seconds. This function is only for -// testing Google Test's own constructs. Don't use it in user tests, -// either directly or indirectly. +#if GTEST_IS_THREADSAFE +# if GTEST_HAS_PTHREAD +// Sleeps for (roughly) n milliseconds. This function is only for testing +// Google Test's own constructs. Don't use it in user tests, either +// directly or indirectly. inline void SleepMilliseconds(int n) { const timespec time = { 0, // 0 seconds. @@ -1079,7 +1464,10 @@ inline void SleepMilliseconds(int n) { }; nanosleep(&time, NULL); } +# endif // GTEST_HAS_PTHREAD +# if 0 // OS detection +# elif GTEST_HAS_PTHREAD // Allows a controller thread to pause execution of newly created // threads until notified. Instances of this class must be created // and destroyed in the controller thread. @@ -1088,26 +1476,97 @@ inline void SleepMilliseconds(int n) { // use it in user tests, either directly or indirectly. class Notification { public: - Notification() : notified_(false) {} + Notification() : notified_(false) { + GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_init(&mutex_, NULL)); + } + ~Notification() { + pthread_mutex_destroy(&mutex_); + } // Notifies all threads created with this notification to start. Must // be called from the controller thread. - void Notify() { notified_ = true; } + void Notify() { + pthread_mutex_lock(&mutex_); + notified_ = true; + pthread_mutex_unlock(&mutex_); + } // Blocks until the controller thread notifies. Must be called from a test // thread. void WaitForNotification() { - while(!notified_) { + for (;;) { + pthread_mutex_lock(&mutex_); + const bool notified = notified_; + pthread_mutex_unlock(&mutex_); + if (notified) + break; SleepMilliseconds(10); } } private: - volatile bool notified_; + pthread_mutex_t mutex_; + bool notified_; GTEST_DISALLOW_COPY_AND_ASSIGN_(Notification); }; +# elif GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT + +GTEST_API_ void SleepMilliseconds(int n); + +// Provides leak-safe Windows kernel handle ownership. +// Used in death tests and in threading support. +class GTEST_API_ AutoHandle { + public: + // Assume that Win32 HANDLE type is equivalent to void*. Doing so allows us to + // avoid including <windows.h> in this header file. Including <windows.h> is + // undesirable because it defines a lot of symbols and macros that tend to + // conflict with client code. This assumption is verified by + // WindowsTypesTest.HANDLEIsVoidStar. + typedef void* Handle; + AutoHandle(); + explicit AutoHandle(Handle handle); + + ~AutoHandle(); + + Handle Get() const; + void Reset(); + void Reset(Handle handle); + + private: + // Returns true iff the handle is a valid handle object that can be closed. + bool IsCloseable() const; + + Handle handle_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(AutoHandle); +}; + +// Allows a controller thread to pause execution of newly created +// threads until notified. Instances of this class must be created +// and destroyed in the controller thread. +// +// This class is only for testing Google Test's own constructs. Do not +// use it in user tests, either directly or indirectly. +class GTEST_API_ Notification { + public: + Notification(); + void Notify(); + void WaitForNotification(); + + private: + AutoHandle event_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(Notification); +}; +# endif // OS detection + +// On MinGW, we can have both GTEST_OS_WINDOWS and GTEST_HAS_PTHREAD +// defined, but we don't want to use MinGW's pthreads implementation, which +// has conformance problems with some versions of the POSIX standard. +# if GTEST_HAS_PTHREAD && !GTEST_OS_WINDOWS_MINGW + // As a C-function, ThreadFuncWithCLinkage cannot be templated itself. // Consequently, it cannot select a correct instantiation of ThreadWithParam // in order to call its Run(). Introducing ThreadWithParamBase as a @@ -1145,10 +1604,9 @@ extern "C" inline void* ThreadFuncWithCLinkage(void* thread) { template <typename T> class ThreadWithParam : public ThreadWithParamBase { public: - typedef void (*UserThreadFunc)(T); + typedef void UserThreadFunc(T); - ThreadWithParam( - UserThreadFunc func, T param, Notification* thread_can_start) + ThreadWithParam(UserThreadFunc* func, T param, Notification* thread_can_start) : func_(func), param_(param), thread_can_start_(thread_can_start), @@ -1175,7 +1633,7 @@ class ThreadWithParam : public ThreadWithParamBase { } private: - const UserThreadFunc func_; // User-supplied thread function. + UserThreadFunc* const func_; // User-supplied thread function. const T param_; // User-supplied parameter to the thread function. // When non-NULL, used to block execution until the controller thread // notifies. @@ -1185,47 +1643,278 @@ class ThreadWithParam : public ThreadWithParamBase { GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadWithParam); }; +# endif // GTEST_HAS_PTHREAD && !GTEST_OS_WINDOWS_MINGW + +# if 0 // OS detection +# elif GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT -// MutexBase and Mutex implement mutex on pthreads-based platforms. They -// are used in conjunction with class MutexLock: +// Mutex implements mutex on Windows platforms. It is used in conjunction +// with class MutexLock: // // Mutex mutex; // ... -// MutexLock lock(&mutex); // Acquires the mutex and releases it at the end -// // of the current scope. -// -// MutexBase implements behavior for both statically and dynamically -// allocated mutexes. Do not use MutexBase directly. Instead, write -// the following to define a static mutex: +// MutexLock lock(&mutex); // Acquires the mutex and releases it at the +// // end of the current scope. // +// A static Mutex *must* be defined or declared using one of the following +// macros: // GTEST_DEFINE_STATIC_MUTEX_(g_some_mutex); +// GTEST_DECLARE_STATIC_MUTEX_(g_some_mutex); // -// You can forward declare a static mutex like this: +// (A non-static Mutex is defined/declared in the usual way). +class GTEST_API_ Mutex { + public: + enum MutexType { kStatic = 0, kDynamic = 1 }; + // We rely on kStaticMutex being 0 as it is to what the linker initializes + // type_ in static mutexes. critical_section_ will be initialized lazily + // in ThreadSafeLazyInit(). + enum StaticConstructorSelector { kStaticMutex = 0 }; + + // This constructor intentionally does nothing. It relies on type_ being + // statically initialized to 0 (effectively setting it to kStatic) and on + // ThreadSafeLazyInit() to lazily initialize the rest of the members. + explicit Mutex(StaticConstructorSelector /*dummy*/) {} + + Mutex(); + ~Mutex(); + + void Lock(); + + void Unlock(); + + // Does nothing if the current thread holds the mutex. Otherwise, crashes + // with high probability. + void AssertHeld(); + + private: + // Initializes owner_thread_id_ and critical_section_ in static mutexes. + void ThreadSafeLazyInit(); + + // Per http://blogs.msdn.com/b/oldnewthing/archive/2004/02/23/78395.aspx, + // we assume that 0 is an invalid value for thread IDs. + unsigned int owner_thread_id_; + + // For static mutexes, we rely on these members being initialized to zeros + // by the linker. + MutexType type_; + long critical_section_init_phase_; // NOLINT + _RTL_CRITICAL_SECTION* critical_section_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(Mutex); +}; + +# define GTEST_DECLARE_STATIC_MUTEX_(mutex) \ + extern ::testing::internal::Mutex mutex + +# define GTEST_DEFINE_STATIC_MUTEX_(mutex) \ + ::testing::internal::Mutex mutex(::testing::internal::Mutex::kStaticMutex) + +// We cannot name this class MutexLock because the ctor declaration would +// conflict with a macro named MutexLock, which is defined on some +// platforms. That macro is used as a defensive measure to prevent against +// inadvertent misuses of MutexLock like "MutexLock(&mu)" rather than +// "MutexLock l(&mu)". Hence the typedef trick below. +class GTestMutexLock { + public: + explicit GTestMutexLock(Mutex* mutex) + : mutex_(mutex) { mutex_->Lock(); } + + ~GTestMutexLock() { mutex_->Unlock(); } + + private: + Mutex* const mutex_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(GTestMutexLock); +}; + +typedef GTestMutexLock MutexLock; + +// Base class for ValueHolder<T>. Allows a caller to hold and delete a value +// without knowing its type. +class ThreadLocalValueHolderBase { + public: + virtual ~ThreadLocalValueHolderBase() {} +}; + +// Provides a way for a thread to send notifications to a ThreadLocal +// regardless of its parameter type. +class ThreadLocalBase { + public: + // Creates a new ValueHolder<T> object holding a default value passed to + // this ThreadLocal<T>'s constructor and returns it. It is the caller's + // responsibility not to call this when the ThreadLocal<T> instance already + // has a value on the current thread. + virtual ThreadLocalValueHolderBase* NewValueForCurrentThread() const = 0; + + protected: + ThreadLocalBase() {} + virtual ~ThreadLocalBase() {} + + private: + GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadLocalBase); +}; + +// Maps a thread to a set of ThreadLocals that have values instantiated on that +// thread and notifies them when the thread exits. A ThreadLocal instance is +// expected to persist until all threads it has values on have terminated. +class GTEST_API_ ThreadLocalRegistry { + public: + // Registers thread_local_instance as having value on the current thread. + // Returns a value that can be used to identify the thread from other threads. + static ThreadLocalValueHolderBase* GetValueOnCurrentThread( + const ThreadLocalBase* thread_local_instance); + + // Invoked when a ThreadLocal instance is destroyed. + static void OnThreadLocalDestroyed( + const ThreadLocalBase* thread_local_instance); +}; + +class GTEST_API_ ThreadWithParamBase { + public: + void Join(); + + protected: + class Runnable { + public: + virtual ~Runnable() {} + virtual void Run() = 0; + }; + + ThreadWithParamBase(Runnable *runnable, Notification* thread_can_start); + virtual ~ThreadWithParamBase(); + + private: + AutoHandle thread_; +}; + +// Helper class for testing Google Test's multi-threading constructs. +template <typename T> +class ThreadWithParam : public ThreadWithParamBase { + public: + typedef void UserThreadFunc(T); + + ThreadWithParam(UserThreadFunc* func, T param, Notification* thread_can_start) + : ThreadWithParamBase(new RunnableImpl(func, param), thread_can_start) { + } + virtual ~ThreadWithParam() {} + + private: + class RunnableImpl : public Runnable { + public: + RunnableImpl(UserThreadFunc* func, T param) + : func_(func), + param_(param) { + } + virtual ~RunnableImpl() {} + virtual void Run() { + func_(param_); + } + + private: + UserThreadFunc* const func_; + const T param_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(RunnableImpl); + }; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadWithParam); +}; + +// Implements thread-local storage on Windows systems. // -// GTEST_DECLARE_STATIC_MUTEX_(g_some_mutex); +// // Thread 1 +// ThreadLocal<int> tl(100); // 100 is the default value for each thread. // -// To create a dynamic mutex, just define an object of type Mutex. +// // Thread 2 +// tl.set(150); // Changes the value for thread 2 only. +// EXPECT_EQ(150, tl.get()); +// +// // Thread 1 +// EXPECT_EQ(100, tl.get()); // In thread 1, tl has the original value. +// tl.set(200); +// EXPECT_EQ(200, tl.get()); +// +// The template type argument T must have a public copy constructor. +// In addition, the default ThreadLocal constructor requires T to have +// a public default constructor. +// +// The users of a TheadLocal instance have to make sure that all but one +// threads (including the main one) using that instance have exited before +// destroying it. Otherwise, the per-thread objects managed for them by the +// ThreadLocal instance are not guaranteed to be destroyed on all platforms. +// +// Google Test only uses global ThreadLocal objects. That means they +// will die after main() has returned. Therefore, no per-thread +// object managed by Google Test will be leaked as long as all threads +// using Google Test have exited when main() returns. +template <typename T> +class ThreadLocal : public ThreadLocalBase { + public: + ThreadLocal() : default_() {} + explicit ThreadLocal(const T& value) : default_(value) {} + + ~ThreadLocal() { ThreadLocalRegistry::OnThreadLocalDestroyed(this); } + + T* pointer() { return GetOrCreateValue(); } + const T* pointer() const { return GetOrCreateValue(); } + const T& get() const { return *pointer(); } + void set(const T& value) { *pointer() = value; } + + private: + // Holds a value of T. Can be deleted via its base class without the caller + // knowing the type of T. + class ValueHolder : public ThreadLocalValueHolderBase { + public: + explicit ValueHolder(const T& value) : value_(value) {} + + T* pointer() { return &value_; } + + private: + T value_; + GTEST_DISALLOW_COPY_AND_ASSIGN_(ValueHolder); + }; + + + T* GetOrCreateValue() const { + return static_cast<ValueHolder*>( + ThreadLocalRegistry::GetValueOnCurrentThread(this))->pointer(); + } + + virtual ThreadLocalValueHolderBase* NewValueForCurrentThread() const { + return new ValueHolder(default_); + } + + const T default_; // The default value for each thread. + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadLocal); +}; + +# elif GTEST_HAS_PTHREAD + +// MutexBase and Mutex implement mutex on pthreads-based platforms. class MutexBase { public: // Acquires this mutex. void Lock() { GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_lock(&mutex_)); owner_ = pthread_self(); + has_owner_ = true; } // Releases this mutex. void Unlock() { - // We don't protect writing to owner_ here, as it's the caller's - // responsibility to ensure that the current thread holds the + // Since the lock is being released the owner_ field should no longer be + // considered valid. We don't protect writing to has_owner_ here, as it's + // the caller's responsibility to ensure that the current thread holds the // mutex when this is called. - owner_ = 0; + has_owner_ = false; GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_unlock(&mutex_)); } // Does nothing if the current thread holds the mutex. Otherwise, crashes // with high probability. void AssertHeld() const { - GTEST_CHECK_(owner_ == pthread_self()) + GTEST_CHECK_(has_owner_ && pthread_equal(owner_, pthread_self())) << "The current thread is not holding the mutex @" << this; } @@ -1236,16 +1925,28 @@ class MutexBase { // have to be public. public: pthread_mutex_t mutex_; // The underlying pthread mutex. - pthread_t owner_; // The thread holding the mutex; 0 means no one holds it. + // has_owner_ indicates whether the owner_ field below contains a valid thread + // ID and is therefore safe to inspect (e.g., to use in pthread_equal()). All + // accesses to the owner_ field should be protected by a check of this field. + // An alternative might be to memset() owner_ to all zeros, but there's no + // guarantee that a zero'd pthread_t is necessarily invalid or even different + // from pthread_self(). + bool has_owner_; + pthread_t owner_; // The thread holding the mutex. }; // Forward-declares a static mutex. -# define GTEST_DECLARE_STATIC_MUTEX_(mutex) \ - extern ::testing::internal::MutexBase mutex +# define GTEST_DECLARE_STATIC_MUTEX_(mutex) \ + extern ::testing::internal::MutexBase mutex // Defines and statically (i.e. at link time) initializes a static mutex. -# define GTEST_DEFINE_STATIC_MUTEX_(mutex) \ - ::testing::internal::MutexBase mutex = { PTHREAD_MUTEX_INITIALIZER, 0 } +// The initialization list here does not explicitly initialize each field, +// instead relying on default initialization for the unspecified fields. In +// particular, the owner_ field (a pthread_t) is not explicitly initialized. +// This allows initialization to work whether pthread_t is a scalar or struct. +// The flag -Wmissing-field-initializers must not be specified for this to work. +# define GTEST_DEFINE_STATIC_MUTEX_(mutex) \ + ::testing::internal::MutexBase mutex = { PTHREAD_MUTEX_INITIALIZER, false } // The Mutex class can only be used for mutexes created at runtime. It // shares its API with MutexBase otherwise. @@ -1253,7 +1954,7 @@ class Mutex : public MutexBase { public: Mutex() { GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_init(&mutex_, NULL)); - owner_ = 0; + has_owner_ = false; } ~Mutex() { GTEST_CHECK_POSIX_SUCCESS_(pthread_mutex_destroy(&mutex_)); @@ -1263,9 +1964,11 @@ class Mutex : public MutexBase { GTEST_DISALLOW_COPY_AND_ASSIGN_(Mutex); }; -// We cannot name this class MutexLock as the ctor declaration would +// We cannot name this class MutexLock because the ctor declaration would // conflict with a macro named MutexLock, which is defined on some -// platforms. Hence the typedef trick below. +// platforms. That macro is used as a defensive measure to prevent against +// inadvertent misuses of MutexLock like "MutexLock(&mu)" rather than +// "MutexLock l(&mu)". Hence the typedef trick below. class GTestMutexLock { public: explicit GTestMutexLock(MutexBase* mutex) @@ -1299,34 +2002,6 @@ extern "C" inline void DeleteThreadLocalValue(void* value_holder) { } // Implements thread-local storage on pthreads-based systems. -// -// // Thread 1 -// ThreadLocal<int> tl(100); // 100 is the default value for each thread. -// -// // Thread 2 -// tl.set(150); // Changes the value for thread 2 only. -// EXPECT_EQ(150, tl.get()); -// -// // Thread 1 -// EXPECT_EQ(100, tl.get()); // In thread 1, tl has the original value. -// tl.set(200); -// EXPECT_EQ(200, tl.get()); -// -// The template type argument T must have a public copy constructor. -// In addition, the default ThreadLocal constructor requires T to have -// a public default constructor. -// -// An object managed for a thread by a ThreadLocal instance is deleted -// when the thread exits. Or, if the ThreadLocal instance dies in -// that thread, when the ThreadLocal dies. It's the user's -// responsibility to ensure that all other threads using a ThreadLocal -// have exited when it dies, or the per-thread objects for those -// threads will not be deleted. -// -// Google Test only uses global ThreadLocal objects. That means they -// will die after main() has returned. Therefore, no per-thread -// object managed by Google Test will be leaked as long as all threads -// using Google Test have exited when main() returns. template <typename T> class ThreadLocal { public: @@ -1391,9 +2066,9 @@ class ThreadLocal { GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadLocal); }; -# define GTEST_IS_THREADSAFE 1 +# endif // OS detection -#else // GTEST_HAS_PTHREAD +#else // GTEST_IS_THREADSAFE // A dummy implementation of synchronization primitives (mutex, lock, // and thread-local variable). Necessary for compiling Google Test where @@ -1403,6 +2078,8 @@ class ThreadLocal { class Mutex { public: Mutex() {} + void Lock() {} + void Unlock() {} void AssertHeld() const {} }; @@ -1411,6 +2088,11 @@ class Mutex { # define GTEST_DEFINE_STATIC_MUTEX_(mutex) ::testing::internal::Mutex mutex +// We cannot name this class MutexLock because the ctor declaration would +// conflict with a macro named MutexLock, which is defined on some +// platforms. That macro is used as a defensive measure to prevent against +// inadvertent misuses of MutexLock like "MutexLock(&mu)" rather than +// "MutexLock l(&mu)". Hence the typedef trick below. class GTestMutexLock { public: explicit GTestMutexLock(Mutex*) {} // NOLINT @@ -1431,11 +2113,7 @@ class ThreadLocal { T value_; }; -// The above synchronization primitives have dummy implementations. -// Therefore Google Test is not thread-safe. -# define GTEST_IS_THREADSAFE 0 - -#endif // GTEST_HAS_PTHREAD +#endif // GTEST_IS_THREADSAFE // Returns the number of threads running in the process, or 0 to indicate that // we cannot detect it. @@ -1533,6 +2211,10 @@ inline bool IsUpper(char ch) { inline bool IsXDigit(char ch) { return isxdigit(static_cast<unsigned char>(ch)) != 0; } +inline bool IsXDigit(wchar_t ch) { + const unsigned char low_byte = static_cast<unsigned char>(ch); + return ch == low_byte && isxdigit(low_byte) != 0; +} inline char ToLower(char ch) { return static_cast<char>(tolower(static_cast<unsigned char>(ch))); @@ -1541,6 +2223,13 @@ inline char ToUpper(char ch) { return static_cast<char>(toupper(static_cast<unsigned char>(ch))); } +inline std::string StripTrailingSpaces(std::string str) { + std::string::iterator it = str.end(); + while (it != str.begin() && IsSpace(*--it)) + it = str.erase(it); + return str; +} + // The testing::internal::posix namespace holds wrappers for common // POSIX functions. These wrappers hide the differences between // Windows/MSVC and POSIX systems. Since some compilers define these @@ -1604,11 +2293,7 @@ inline bool IsDir(const StatStruct& st) { return S_ISDIR(st.st_mode); } // Functions deprecated by MSVC 8.0. -#ifdef _MSC_VER -// Temporarily disable warning 4996 (deprecated function). -# pragma warning(push) -# pragma warning(disable:4996) -#endif +GTEST_DISABLE_MSC_WARNINGS_PUSH_(4996 /* deprecated function */) inline const char* StrNCpy(char* dest, const char* src, size_t n) { return strncpy(dest, src, n); @@ -1618,7 +2303,7 @@ inline const char* StrNCpy(char* dest, const char* src, size_t n) { // StrError() aren't needed on Windows CE at this time and thus not // defined there. -#if !GTEST_OS_WINDOWS_MOBILE +#if !GTEST_OS_WINDOWS_MOBILE && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT inline int ChDir(const char* dir) { return chdir(dir); } #endif inline FILE* FOpen(const char* path, const char* mode) { @@ -1642,8 +2327,9 @@ inline int Close(int fd) { return close(fd); } inline const char* StrError(int errnum) { return strerror(errnum); } #endif inline const char* GetEnv(const char* name) { -#if GTEST_OS_WINDOWS_MOBILE +#if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_WINDOWS_PHONE | GTEST_OS_WINDOWS_RT // We are on Windows CE, which has no environment variables. + static_cast<void>(name); // To prevent 'unused argument' warning. return NULL; #elif defined(__BORLANDC__) || defined(__SunOS_5_8) || defined(__SunOS_5_9) // Environment variables which we programmatically clear will be set to the @@ -1655,9 +2341,7 @@ inline const char* GetEnv(const char* name) { #endif } -#ifdef _MSC_VER -# pragma warning(pop) // Restores the warning state. -#endif +GTEST_DISABLE_MSC_WARNINGS_POP_() #if GTEST_OS_WINDOWS_MOBILE // Windows CE has no C library. The abort() function is used in @@ -1670,6 +2354,23 @@ inline void Abort() { abort(); } } // namespace posix +// MSVC "deprecates" snprintf and issues warnings wherever it is used. In +// order to avoid these warnings, we need to use _snprintf or _snprintf_s on +// MSVC-based platforms. We map the GTEST_SNPRINTF_ macro to the appropriate +// function in order to achieve that. We use macro definition here because +// snprintf is a variadic function. +#if _MSC_VER >= 1400 && !GTEST_OS_WINDOWS_MOBILE +// MSVC 2005 and above support variadic macros. +# define GTEST_SNPRINTF_(buffer, size, format, ...) \ + _snprintf_s(buffer, size, size, format, __VA_ARGS__) +#elif defined(_MSC_VER) +// Windows CE does not define _snprintf_s and MSVC prior to 2005 doesn't +// complain about _snprintf. +# define GTEST_SNPRINTF_ _snprintf +#else +# define GTEST_SNPRINTF_ snprintf +#endif + // The maximum number a BiggestInt can represent. This definition // works no matter BiggestInt is represented in one's complement or // two's complement. @@ -1722,7 +2423,6 @@ class TypeWithSize<4> { template <> class TypeWithSize<8> { public: - #if GTEST_OS_WINDOWS typedef __int64 Int; typedef unsigned __int64 UInt; @@ -1749,7 +2449,7 @@ typedef TypeWithSize<8>::Int TimeInMillis; // Represents time in milliseconds. #define GTEST_DECLARE_int32_(name) \ GTEST_API_ extern ::testing::internal::Int32 GTEST_FLAG(name) #define GTEST_DECLARE_string_(name) \ - GTEST_API_ extern ::testing::internal::String GTEST_FLAG(name) + GTEST_API_ extern ::std::string GTEST_FLAG(name) // Macros for defining flags. #define GTEST_DEFINE_bool_(name, default_val, doc) \ @@ -1757,7 +2457,11 @@ typedef TypeWithSize<8>::Int TimeInMillis; // Represents time in milliseconds. #define GTEST_DEFINE_int32_(name, default_val, doc) \ GTEST_API_ ::testing::internal::Int32 GTEST_FLAG(name) = (default_val) #define GTEST_DEFINE_string_(name, default_val, doc) \ - GTEST_API_ ::testing::internal::String GTEST_FLAG(name) = (default_val) + GTEST_API_ ::std::string GTEST_FLAG(name) = (default_val) + +// Thread annotations +#define GTEST_EXCLUSIVE_LOCK_REQUIRED_(locks) +#define GTEST_LOCK_EXCLUDED_(locks) // Parses 'str' for a 32-bit signed integer. If successful, writes the result // to *value and returns true; otherwise leaves *value unchanged and returns @@ -1777,3 +2481,4 @@ const char* StringFromGTestEnv(const char* flag, const char* default_val); } // namespace testing #endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_ + diff --git a/tests/gtest/include/gtest/internal/gtest-string.h b/tests/gtest/include/gtest/internal/gtest-string.h index dc3a07be8..97f1a7fdd 100644 --- a/tests/gtest/include/gtest/internal/gtest-string.h +++ b/tests/gtest/include/gtest/internal/gtest-string.h @@ -47,50 +47,18 @@ #endif #include <string.h> -#include "gtest/internal/gtest-port.h" - #include <string> +#include "gtest/internal/gtest-port.h" + namespace testing { namespace internal { -// String - a UTF-8 string class. -// -// For historic reasons, we don't use std::string. -// -// TODO(wan@google.com): replace this class with std::string or -// implement it in terms of the latter. -// -// Note that String can represent both NULL and the empty string, -// while std::string cannot represent NULL. -// -// NULL and the empty string are considered different. NULL is less -// than anything (including the empty string) except itself. -// -// This class only provides minimum functionality necessary for -// implementing Google Test. We do not intend to implement a full-fledged -// string class here. -// -// Since the purpose of this class is to provide a substitute for -// std::string on platforms where it cannot be used, we define a copy -// constructor and assignment operators such that we don't need -// conditional compilation in a lot of places. -// -// In order to make the representation efficient, the d'tor of String -// is not virtual. Therefore DO NOT INHERIT FROM String. +// String - an abstract class holding static string utilities. class GTEST_API_ String { public: // Static utility methods - // Returns the input enclosed in double quotes if it's not NULL; - // otherwise returns "(null)". For example, "\"Hello\"" is returned - // for input "Hello". - // - // This is useful for printing a C string in the syntax of a literal. - // - // Known issue: escape sequences are not handled yet. - static String ShowCStringQuoted(const char* c_str); - // Clones a 0-terminated C string, allocating memory using new. The // caller is responsible for deleting the return value using // delete[]. Returns the cloned string, or NULL if the input is @@ -137,11 +105,7 @@ class GTEST_API_ String { // NULL will be converted to "(null)". If an error occurred during // the conversion, "(failed to convert from wide string)" is // returned. - static String ShowWideCString(const wchar_t* wide_c_str); - - // Similar to ShowWideCString(), except that this function encloses - // the converted string in double quotes. - static String ShowWideCStringQuoted(const wchar_t* wide_c_str); + static std::string ShowWideCString(const wchar_t* wide_c_str); // Compares two wide C strings. Returns true iff they have the same // content. @@ -175,174 +139,27 @@ class GTEST_API_ String { static bool CaseInsensitiveWideCStringEquals(const wchar_t* lhs, const wchar_t* rhs); - // Formats a list of arguments to a String, using the same format - // spec string as for printf. - // - // We do not use the StringPrintf class as it is not universally - // available. - // - // The result is limited to 4096 characters (including the tailing - // 0). If 4096 characters are not enough to format the input, - // "<buffer exceeded>" is returned. - static String Format(const char* format, ...); - - // C'tors - - // The default c'tor constructs a NULL string. - String() : c_str_(NULL), length_(0) {} - - // Constructs a String by cloning a 0-terminated C string. - String(const char* a_c_str) { // NOLINT - if (a_c_str == NULL) { - c_str_ = NULL; - length_ = 0; - } else { - ConstructNonNull(a_c_str, strlen(a_c_str)); - } - } + // Returns true iff the given string ends with the given suffix, ignoring + // case. Any string is considered to end with an empty suffix. + static bool EndsWithCaseInsensitive( + const std::string& str, const std::string& suffix); - // Constructs a String by copying a given number of chars from a - // buffer. E.g. String("hello", 3) creates the string "hel", - // String("a\0bcd", 4) creates "a\0bc", String(NULL, 0) creates "", - // and String(NULL, 1) results in access violation. - String(const char* buffer, size_t a_length) { - ConstructNonNull(buffer, a_length); - } + // Formats an int value as "%02d". + static std::string FormatIntWidth2(int value); // "%02d" for width == 2 - // The copy c'tor creates a new copy of the string. The two - // String objects do not share content. - String(const String& str) : c_str_(NULL), length_(0) { *this = str; } + // Formats an int value as "%X". + static std::string FormatHexInt(int value); - // D'tor. String is intended to be a final class, so the d'tor - // doesn't need to be virtual. - ~String() { delete[] c_str_; } - - // Allows a String to be implicitly converted to an ::std::string or - // ::string, and vice versa. Converting a String containing a NULL - // pointer to ::std::string or ::string is undefined behavior. - // Converting a ::std::string or ::string containing an embedded NUL - // character to a String will result in the prefix up to the first - // NUL character. - String(const ::std::string& str) { - ConstructNonNull(str.c_str(), str.length()); - } - - operator ::std::string() const { return ::std::string(c_str(), length()); } - -#if GTEST_HAS_GLOBAL_STRING - String(const ::string& str) { - ConstructNonNull(str.c_str(), str.length()); - } - - operator ::string() const { return ::string(c_str(), length()); } -#endif // GTEST_HAS_GLOBAL_STRING - - // Returns true iff this is an empty string (i.e. ""). - bool empty() const { return (c_str() != NULL) && (length() == 0); } - - // Compares this with another String. - // Returns < 0 if this is less than rhs, 0 if this is equal to rhs, or > 0 - // if this is greater than rhs. - int Compare(const String& rhs) const; - - // Returns true iff this String equals the given C string. A NULL - // string and a non-NULL string are considered not equal. - bool operator==(const char* a_c_str) const { return Compare(a_c_str) == 0; } - - // Returns true iff this String is less than the given String. A - // NULL string is considered less than "". - bool operator<(const String& rhs) const { return Compare(rhs) < 0; } - - // Returns true iff this String doesn't equal the given C string. A NULL - // string and a non-NULL string are considered not equal. - bool operator!=(const char* a_c_str) const { return !(*this == a_c_str); } - - // Returns true iff this String ends with the given suffix. *Any* - // String is considered to end with a NULL or empty suffix. - bool EndsWith(const char* suffix) const; - - // Returns true iff this String ends with the given suffix, not considering - // case. Any String is considered to end with a NULL or empty suffix. - bool EndsWithCaseInsensitive(const char* suffix) const; - - // Returns the length of the encapsulated string, or 0 if the - // string is NULL. - size_t length() const { return length_; } - - // Gets the 0-terminated C string this String object represents. - // The String object still owns the string. Therefore the caller - // should NOT delete the return value. - const char* c_str() const { return c_str_; } - - // Assigns a C string to this object. Self-assignment works. - const String& operator=(const char* a_c_str) { - return *this = String(a_c_str); - } - - // Assigns a String object to this object. Self-assignment works. - const String& operator=(const String& rhs) { - if (this != &rhs) { - delete[] c_str_; - if (rhs.c_str() == NULL) { - c_str_ = NULL; - length_ = 0; - } else { - ConstructNonNull(rhs.c_str(), rhs.length()); - } - } - - return *this; - } + // Formats a byte as "%02X". + static std::string FormatByte(unsigned char value); private: - // Constructs a non-NULL String from the given content. This - // function can only be called when c_str_ has not been allocated. - // ConstructNonNull(NULL, 0) results in an empty string (""). - // ConstructNonNull(NULL, non_zero) is undefined behavior. - void ConstructNonNull(const char* buffer, size_t a_length) { - char* const str = new char[a_length + 1]; - memcpy(str, buffer, a_length); - str[a_length] = '\0'; - c_str_ = str; - length_ = a_length; - } - - const char* c_str_; - size_t length_; + String(); // Not meant to be instantiated. }; // class String -// Streams a String to an ostream. Each '\0' character in the String -// is replaced with "\\0". -inline ::std::ostream& operator<<(::std::ostream& os, const String& str) { - if (str.c_str() == NULL) { - os << "(null)"; - } else { - const char* const c_str = str.c_str(); - for (size_t i = 0; i != str.length(); i++) { - if (c_str[i] == '\0') { - os << "\\0"; - } else { - os << c_str[i]; - } - } - } - return os; -} - -// Gets the content of the stringstream's buffer as a String. Each '\0' +// Gets the content of the stringstream's buffer as an std::string. Each '\0' // character in the buffer is replaced with "\\0". -GTEST_API_ String StringStreamToString(::std::stringstream* stream); - -// Converts a streamable value to a String. A NULL pointer is -// converted to "(null)". When the input value is a ::string, -// ::std::string, ::wstring, or ::std::wstring object, each NUL -// character in it is replaced with "\\0". - -// Declared here but defined in gtest.h, so that it has access -// to the definition of the Message class, required by the ARM -// compiler. -template <typename T> -String StreamableToString(const T& streamable); +GTEST_API_ std::string StringStreamToString(::std::stringstream* stream); } // namespace internal } // namespace testing diff --git a/tests/gtest/include/gtest/internal/gtest-tuple.h b/tests/gtest/include/gtest/internal/gtest-tuple.h index d1af50e18..e9b405340 100644 --- a/tests/gtest/include/gtest/internal/gtest-tuple.h +++ b/tests/gtest/include/gtest/internal/gtest-tuple.h @@ -1,4 +1,6 @@ -// This file was GENERATED by a script. DO NOT EDIT BY HAND!!! +// This file was GENERATED by command: +// pump.py gtest-tuple.h.pump +// DO NOT EDIT BY HAND!!! // Copyright 2009 Google Inc. // All Rights Reserved. @@ -51,6 +53,14 @@ private: #endif +// Visual Studio 2010, 2012, and 2013 define symbols in std::tr1 that conflict +// with our own definitions. Therefore using our own tuple does not work on +// those compilers. +#if defined(_MSC_VER) && _MSC_VER >= 1600 /* 1600 is Visual Studio 2010 */ +# error "gtest's tuple doesn't compile on Visual Studio 2010 or later. \ +GTEST_USE_OWN_TR1_TUPLE must be set to 0 on those compilers." +#endif + // GTEST_n_TUPLE_(T) is the type of an n-tuple. #define GTEST_0_TUPLE_(T) tuple<> #define GTEST_1_TUPLE_(T) tuple<T##0, void, void, void, void, void, void, \ @@ -140,34 +150,54 @@ template <bool kIndexValid, int kIndex, class Tuple> struct TupleElement; template <GTEST_10_TYPENAMES_(T)> -struct TupleElement<true, 0, GTEST_10_TUPLE_(T)> { typedef T0 type; }; +struct TupleElement<true, 0, GTEST_10_TUPLE_(T) > { + typedef T0 type; +}; template <GTEST_10_TYPENAMES_(T)> -struct TupleElement<true, 1, GTEST_10_TUPLE_(T)> { typedef T1 type; }; +struct TupleElement<true, 1, GTEST_10_TUPLE_(T) > { + typedef T1 type; +}; template <GTEST_10_TYPENAMES_(T)> -struct TupleElement<true, 2, GTEST_10_TUPLE_(T)> { typedef T2 type; }; +struct TupleElement<true, 2, GTEST_10_TUPLE_(T) > { + typedef T2 type; +}; template <GTEST_10_TYPENAMES_(T)> -struct TupleElement<true, 3, GTEST_10_TUPLE_(T)> { typedef T3 type; }; +struct TupleElement<true, 3, GTEST_10_TUPLE_(T) > { + typedef T3 type; +}; template <GTEST_10_TYPENAMES_(T)> -struct TupleElement<true, 4, GTEST_10_TUPLE_(T)> { typedef T4 type; }; +struct TupleElement<true, 4, GTEST_10_TUPLE_(T) > { + typedef T4 type; +}; template <GTEST_10_TYPENAMES_(T)> -struct TupleElement<true, 5, GTEST_10_TUPLE_(T)> { typedef T5 type; }; +struct TupleElement<true, 5, GTEST_10_TUPLE_(T) > { + typedef T5 type; +}; template <GTEST_10_TYPENAMES_(T)> -struct TupleElement<true, 6, GTEST_10_TUPLE_(T)> { typedef T6 type; }; +struct TupleElement<true, 6, GTEST_10_TUPLE_(T) > { + typedef T6 type; +}; template <GTEST_10_TYPENAMES_(T)> -struct TupleElement<true, 7, GTEST_10_TUPLE_(T)> { typedef T7 type; }; +struct TupleElement<true, 7, GTEST_10_TUPLE_(T) > { + typedef T7 type; +}; template <GTEST_10_TYPENAMES_(T)> -struct TupleElement<true, 8, GTEST_10_TUPLE_(T)> { typedef T8 type; }; +struct TupleElement<true, 8, GTEST_10_TUPLE_(T) > { + typedef T8 type; +}; template <GTEST_10_TYPENAMES_(T)> -struct TupleElement<true, 9, GTEST_10_TUPLE_(T)> { typedef T9 type; }; +struct TupleElement<true, 9, GTEST_10_TUPLE_(T) > { + typedef T9 type; +}; } // namespace gtest_internal @@ -708,37 +738,59 @@ inline GTEST_10_TUPLE_(T) make_tuple(const T0& f0, const T1& f1, const T2& f2, template <typename Tuple> struct tuple_size; template <GTEST_0_TYPENAMES_(T)> -struct tuple_size<GTEST_0_TUPLE_(T)> { static const int value = 0; }; +struct tuple_size<GTEST_0_TUPLE_(T) > { + static const int value = 0; +}; template <GTEST_1_TYPENAMES_(T)> -struct tuple_size<GTEST_1_TUPLE_(T)> { static const int value = 1; }; +struct tuple_size<GTEST_1_TUPLE_(T) > { + static const int value = 1; +}; template <GTEST_2_TYPENAMES_(T)> -struct tuple_size<GTEST_2_TUPLE_(T)> { static const int value = 2; }; +struct tuple_size<GTEST_2_TUPLE_(T) > { + static const int value = 2; +}; template <GTEST_3_TYPENAMES_(T)> -struct tuple_size<GTEST_3_TUPLE_(T)> { static const int value = 3; }; +struct tuple_size<GTEST_3_TUPLE_(T) > { + static const int value = 3; +}; template <GTEST_4_TYPENAMES_(T)> -struct tuple_size<GTEST_4_TUPLE_(T)> { static const int value = 4; }; +struct tuple_size<GTEST_4_TUPLE_(T) > { + static const int value = 4; +}; template <GTEST_5_TYPENAMES_(T)> -struct tuple_size<GTEST_5_TUPLE_(T)> { static const int value = 5; }; +struct tuple_size<GTEST_5_TUPLE_(T) > { + static const int value = 5; +}; template <GTEST_6_TYPENAMES_(T)> -struct tuple_size<GTEST_6_TUPLE_(T)> { static const int value = 6; }; +struct tuple_size<GTEST_6_TUPLE_(T) > { + static const int value = 6; +}; template <GTEST_7_TYPENAMES_(T)> -struct tuple_size<GTEST_7_TUPLE_(T)> { static const int value = 7; }; +struct tuple_size<GTEST_7_TUPLE_(T) > { + static const int value = 7; +}; template <GTEST_8_TYPENAMES_(T)> -struct tuple_size<GTEST_8_TUPLE_(T)> { static const int value = 8; }; +struct tuple_size<GTEST_8_TUPLE_(T) > { + static const int value = 8; +}; template <GTEST_9_TYPENAMES_(T)> -struct tuple_size<GTEST_9_TUPLE_(T)> { static const int value = 9; }; +struct tuple_size<GTEST_9_TUPLE_(T) > { + static const int value = 9; +}; template <GTEST_10_TYPENAMES_(T)> -struct tuple_size<GTEST_10_TUPLE_(T)> { static const int value = 10; }; +struct tuple_size<GTEST_10_TUPLE_(T) > { + static const int value = 10; +}; template <int k, class Tuple> struct tuple_element { @@ -922,8 +974,8 @@ template <GTEST_10_TYPENAMES_(T), GTEST_10_TYPENAMES_(U)> inline bool operator==(const GTEST_10_TUPLE_(T)& t, const GTEST_10_TUPLE_(U)& u) { return gtest_internal::SameSizeTuplePrefixComparator< - tuple_size<GTEST_10_TUPLE_(T)>::value, - tuple_size<GTEST_10_TUPLE_(U)>::value>::Eq(t, u); + tuple_size<GTEST_10_TUPLE_(T) >::value, + tuple_size<GTEST_10_TUPLE_(U) >::value>::Eq(t, u); } template <GTEST_10_TYPENAMES_(T), GTEST_10_TYPENAMES_(U)> diff --git a/tests/gtest/include/gtest/internal/gtest-tuple.h.pump b/tests/gtest/include/gtest/internal/gtest-tuple.h.pump index ef519094a..429ddfeec 100644 --- a/tests/gtest/include/gtest/internal/gtest-tuple.h.pump +++ b/tests/gtest/include/gtest/internal/gtest-tuple.h.pump @@ -52,6 +52,14 @@ $$ This meta comment fixes auto-indentation in Emacs. }} private: #endif +// Visual Studio 2010, 2012, and 2013 define symbols in std::tr1 that conflict +// with our own definitions. Therefore using our own tuple does not work on +// those compilers. +#if defined(_MSC_VER) && _MSC_VER >= 1600 /* 1600 is Visual Studio 2010 */ +# error "gtest's tuple doesn't compile on Visual Studio 2010 or later. \ +GTEST_USE_OWN_TR1_TUPLE must be set to 0 on those compilers." +#endif + $range i 0..n-1 $range j 0..n @@ -118,8 +126,9 @@ struct TupleElement; $for i [[ template <GTEST_$(n)_TYPENAMES_(T)> -struct TupleElement<true, $i, GTEST_$(n)_TUPLE_(T)> [[]] -{ typedef T$i type; }; +struct TupleElement<true, $i, GTEST_$(n)_TUPLE_(T) > { + typedef T$i type; +}; ]] @@ -220,7 +229,9 @@ template <typename Tuple> struct tuple_size; $for j [[ template <GTEST_$(j)_TYPENAMES_(T)> -struct tuple_size<GTEST_$(j)_TUPLE_(T)> { static const int value = $j; }; +struct tuple_size<GTEST_$(j)_TUPLE_(T) > { + static const int value = $j; +}; ]] @@ -302,8 +313,8 @@ template <GTEST_$(n)_TYPENAMES_(T), GTEST_$(n)_TYPENAMES_(U)> inline bool operator==(const GTEST_$(n)_TUPLE_(T)& t, const GTEST_$(n)_TUPLE_(U)& u) { return gtest_internal::SameSizeTuplePrefixComparator< - tuple_size<GTEST_$(n)_TUPLE_(T)>::value, - tuple_size<GTEST_$(n)_TUPLE_(U)>::value>::Eq(t, u); + tuple_size<GTEST_$(n)_TUPLE_(T) >::value, + tuple_size<GTEST_$(n)_TUPLE_(U) >::value>::Eq(t, u); } template <GTEST_$(n)_TYPENAMES_(T), GTEST_$(n)_TYPENAMES_(U)> diff --git a/tests/gtest/include/gtest/internal/gtest-type-util.h b/tests/gtest/include/gtest/internal/gtest-type-util.h index b7b01b094..e46f7cfcb 100644 --- a/tests/gtest/include/gtest/internal/gtest-type-util.h +++ b/tests/gtest/include/gtest/internal/gtest-type-util.h @@ -45,15 +45,14 @@ #define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ #include "gtest/internal/gtest-port.h" -#include "gtest/internal/gtest-string.h" // #ifdef __GNUC__ is too general here. It is possible to use gcc without using // libstdc++ (which is where cxxabi.h comes from). -# ifdef __GLIBCXX__ +# if GTEST_HAS_CXXABI_H_ # include <cxxabi.h> # elif defined(__HP_aCC) # include <acxx_demangle.h> -# endif // __GLIBCXX__ +# endif // GTEST_HASH_CXXABI_H_ namespace testing { namespace internal { @@ -62,24 +61,24 @@ namespace internal { // NB: This function is also used in Google Mock, so don't move it inside of // the typed-test-only section below. template <typename T> -String GetTypeName() { +std::string GetTypeName() { # if GTEST_HAS_RTTI const char* const name = typeid(T).name(); -# if defined(__GLIBCXX__) || defined(__HP_aCC) +# if GTEST_HAS_CXXABI_H_ || defined(__HP_aCC) int status = 0; // gcc's implementation of typeid(T).name() mangles the type name, // so we have to demangle it. -# ifdef __GLIBCXX__ +# if GTEST_HAS_CXXABI_H_ using abi::__cxa_demangle; -# endif // __GLIBCXX__ +# endif // GTEST_HAS_CXXABI_H_ char* const readable_name = __cxa_demangle(name, 0, 0, &status); - const String name_str(status == 0 ? readable_name : name); + const std::string name_str(status == 0 ? readable_name : name); free(readable_name); return name_str; # else return name; -# endif // __GLIBCXX__ || __HP_aCC +# endif // GTEST_HAS_CXXABI_H_ || __HP_aCC # else @@ -3300,7 +3299,9 @@ struct Templates<T1, T2, T3, T4, T5, T6, T7, T8, T9, T10, T11, T12, T13, T14, // INSTANTIATE_TYPED_TEST_CASE_P(). template <typename T> -struct TypeList { typedef Types1<T> type; }; +struct TypeList { + typedef Types1<T> type; +}; template <typename T1, typename T2, typename T3, typename T4, typename T5, typename T6, typename T7, typename T8, typename T9, typename T10, diff --git a/tests/gtest/include/gtest/internal/gtest-type-util.h.pump b/tests/gtest/include/gtest/internal/gtest-type-util.h.pump index 27f331dea..251fdf025 100644 --- a/tests/gtest/include/gtest/internal/gtest-type-util.h.pump +++ b/tests/gtest/include/gtest/internal/gtest-type-util.h.pump @@ -43,15 +43,14 @@ $var n = 50 $$ Maximum length of type lists we want to support. #define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ #include "gtest/internal/gtest-port.h" -#include "gtest/internal/gtest-string.h" // #ifdef __GNUC__ is too general here. It is possible to use gcc without using // libstdc++ (which is where cxxabi.h comes from). -# ifdef __GLIBCXX__ +# if GTEST_HAS_CXXABI_H_ # include <cxxabi.h> # elif defined(__HP_aCC) # include <acxx_demangle.h> -# endif // __GLIBCXX__ +# endif // GTEST_HASH_CXXABI_H_ namespace testing { namespace internal { @@ -60,24 +59,24 @@ namespace internal { // NB: This function is also used in Google Mock, so don't move it inside of // the typed-test-only section below. template <typename T> -String GetTypeName() { +std::string GetTypeName() { # if GTEST_HAS_RTTI const char* const name = typeid(T).name(); -# if defined(__GLIBCXX__) || defined(__HP_aCC) +# if GTEST_HAS_CXXABI_H_ || defined(__HP_aCC) int status = 0; // gcc's implementation of typeid(T).name() mangles the type name, // so we have to demangle it. -# ifdef __GLIBCXX__ +# if GTEST_HAS_CXXABI_H_ using abi::__cxa_demangle; -# endif // __GLIBCXX__ +# endif // GTEST_HAS_CXXABI_H_ char* const readable_name = __cxa_demangle(name, 0, 0, &status); - const String name_str(status == 0 ? readable_name : name); + const std::string name_str(status == 0 ? readable_name : name); free(readable_name); return name_str; # else return name; -# endif // __GLIBCXX__ || __HP_aCC +# endif // GTEST_HAS_CXXABI_H_ || __HP_aCC # else @@ -279,7 +278,9 @@ struct Templates<$for j, [[T$j]]$for k[[, NoneT]]> { // INSTANTIATE_TYPED_TEST_CASE_P(). template <typename T> -struct TypeList { typedef Types1<T> type; }; +struct TypeList { + typedef Types1<T> type; +}; $range i 1..n diff --git a/tests/gtest/src/gtest-death-test.cc b/tests/gtest/src/gtest-death-test.cc index 8b2e4131c..a0a8c7baf 100644 --- a/tests/gtest/src/gtest-death-test.cc +++ b/tests/gtest/src/gtest-death-test.cc @@ -43,6 +43,11 @@ # include <errno.h> # include <fcntl.h> # include <limits.h> + +# if GTEST_OS_LINUX +# include <signal.h> +# endif // GTEST_OS_LINUX + # include <stdarg.h> # if GTEST_OS_WINDOWS @@ -52,6 +57,10 @@ # include <sys/wait.h> # endif // GTEST_OS_WINDOWS +# if GTEST_OS_QNX +# include <spawn.h> +# endif // GTEST_OS_QNX + #endif // GTEST_HAS_DEATH_TEST #include "gtest/gtest-message.h" @@ -59,9 +68,9 @@ // Indicates that this translation unit is part of Google Test's // implementation. It must come before gtest-internal-inl.h is -// included, or there will be a compiler error. This trick is to -// prevent a user from accidentally including gtest-internal-inl.h in -// his code. +// included, or there will be a compiler error. This trick exists to +// prevent the accidental inclusion of gtest-internal-inl.h in the +// user's code. #define GTEST_IMPLEMENTATION_ 1 #include "src/gtest-internal-inl.h" #undef GTEST_IMPLEMENTATION_ @@ -100,13 +109,42 @@ GTEST_DEFINE_string_( "Indicates the file, line number, temporal index of " "the single death test to run, and a file descriptor to " "which a success code may be sent, all separated by " - "colons. This flag is specified if and only if the current " + "the '|' characters. This flag is specified if and only if the current " "process is a sub-process launched for running a thread-safe " "death test. FOR INTERNAL USE ONLY."); } // namespace internal #if GTEST_HAS_DEATH_TEST +namespace internal { + +// Valid only for fast death tests. Indicates the code is running in the +// child process of a fast style death test. +static bool g_in_fast_death_test_child = false; + +// Returns a Boolean value indicating whether the caller is currently +// executing in the context of the death test child process. Tools such as +// Valgrind heap checkers may need this to modify their behavior in death +// tests. IMPORTANT: This is an internal utility. Using it may break the +// implementation of death tests. User code MUST NOT use it. +bool InDeathTestChild() { +# if GTEST_OS_WINDOWS + + // On Windows, death tests are thread-safe regardless of the value of the + // death_test_style flag. + return !GTEST_FLAG(internal_run_death_test).empty(); + +# else + + if (GTEST_FLAG(death_test_style) == "threadsafe") + return !GTEST_FLAG(internal_run_death_test).empty(); + else + return g_in_fast_death_test_child; +#endif +} + +} // namespace internal + // ExitedWithCode constructor. ExitedWithCode::ExitedWithCode(int exit_code) : exit_code_(exit_code) { } @@ -141,7 +179,7 @@ namespace internal { // Generates a textual description of a given exit code, in the format // specified by wait(2). -static String ExitSummary(int exit_code) { +static std::string ExitSummary(int exit_code) { Message m; # if GTEST_OS_WINDOWS @@ -176,7 +214,7 @@ bool ExitedUnsuccessfully(int exit_status) { // one thread running, or cannot determine the number of threads, prior // to executing the given statement. It is the responsibility of the // caller not to pass a thread_count of 1. -static String DeathTestThreadWarning(size_t thread_count) { +static std::string DeathTestThreadWarning(size_t thread_count) { Message msg; msg << "Death tests use fork(), which is unsafe particularly" << " in a threaded context. For this test, " << GTEST_NAME_ << " "; @@ -210,7 +248,7 @@ enum DeathTestOutcome { IN_PROGRESS, DIED, LIVED, RETURNED, THREW }; // message is propagated back to the parent process. Otherwise, the // message is simply printed to stderr. In either case, the program // then exits with status 1. -void DeathTestAbort(const String& message) { +void DeathTestAbort(const std::string& message) { // On a POSIX system, this function may be called from a threadsafe-style // death test child process, which operates on a very small stack. Use // the heap for any additional non-minuscule memory requirements. @@ -234,9 +272,10 @@ void DeathTestAbort(const String& message) { # define GTEST_DEATH_TEST_CHECK_(expression) \ do { \ if (!::testing::internal::IsTrue(expression)) { \ - DeathTestAbort(::testing::internal::String::Format( \ - "CHECK failed: File %s, line %d: %s", \ - __FILE__, __LINE__, #expression)); \ + DeathTestAbort( \ + ::std::string("CHECK failed: File ") + __FILE__ + ", line " \ + + ::testing::internal::StreamableToString(__LINE__) + ": " \ + + #expression); \ } \ } while (::testing::internal::AlwaysFalse()) @@ -254,15 +293,16 @@ void DeathTestAbort(const String& message) { gtest_retval = (expression); \ } while (gtest_retval == -1 && errno == EINTR); \ if (gtest_retval == -1) { \ - DeathTestAbort(::testing::internal::String::Format( \ - "CHECK failed: File %s, line %d: %s != -1", \ - __FILE__, __LINE__, #expression)); \ + DeathTestAbort( \ + ::std::string("CHECK failed: File ") + __FILE__ + ", line " \ + + ::testing::internal::StreamableToString(__LINE__) + ": " \ + + #expression + " != -1"); \ } \ } while (::testing::internal::AlwaysFalse()) // Returns the message describing the last system error in errno. -String GetLastErrnoDescription() { - return String(errno == 0 ? "" : posix::StrError(errno)); +std::string GetLastErrnoDescription() { + return errno == 0 ? "" : posix::StrError(errno); } // This is called from a death test parent process to read a failure @@ -312,11 +352,11 @@ const char* DeathTest::LastMessage() { return last_death_test_message_.c_str(); } -void DeathTest::set_last_death_test_message(const String& message) { +void DeathTest::set_last_death_test_message(const std::string& message) { last_death_test_message_ = message; } -String DeathTest::last_death_test_message_; +std::string DeathTest::last_death_test_message_; // Provides cross platform implementation for some death functionality. class DeathTestImpl : public DeathTest { @@ -491,7 +531,7 @@ bool DeathTestImpl::Passed(bool status_ok) { if (!spawned()) return false; - const String error_message = GetCapturedStderr(); + const std::string error_message = GetCapturedStderr(); bool success = false; Message buffer; @@ -673,22 +713,19 @@ DeathTest::TestRole WindowsDeathTest::AssumeRole() { FALSE, // The initial state is non-signalled. NULL)); // The even is unnamed. GTEST_DEATH_TEST_CHECK_(event_handle_.Get() != NULL); - const String filter_flag = String::Format("--%s%s=%s.%s", - GTEST_FLAG_PREFIX_, kFilterFlag, - info->test_case_name(), - info->name()); - const String internal_flag = String::Format( - "--%s%s=%s|%d|%d|%u|%Iu|%Iu", - GTEST_FLAG_PREFIX_, - kInternalRunDeathTestFlag, - file_, line_, - death_test_index, - static_cast<unsigned int>(::GetCurrentProcessId()), - // size_t has the same with as pointers on both 32-bit and 64-bit + const std::string filter_flag = + std::string("--") + GTEST_FLAG_PREFIX_ + kFilterFlag + "=" + + info->test_case_name() + "." + info->name(); + const std::string internal_flag = + std::string("--") + GTEST_FLAG_PREFIX_ + kInternalRunDeathTestFlag + + "=" + file_ + "|" + StreamableToString(line_) + "|" + + StreamableToString(death_test_index) + "|" + + StreamableToString(static_cast<unsigned int>(::GetCurrentProcessId())) + + // size_t has the same width as pointers on both 32-bit and 64-bit // Windows platforms. // See http://msdn.microsoft.com/en-us/library/tcxf1dw6.aspx. - reinterpret_cast<size_t>(write_handle), - reinterpret_cast<size_t>(event_handle_.Get())); + "|" + StreamableToString(reinterpret_cast<size_t>(write_handle)) + + "|" + StreamableToString(reinterpret_cast<size_t>(event_handle_.Get())); char executable_path[_MAX_PATH + 1]; // NOLINT GTEST_DEATH_TEST_CHECK_( @@ -696,10 +733,9 @@ DeathTest::TestRole WindowsDeathTest::AssumeRole() { executable_path, _MAX_PATH)); - String command_line = String::Format("%s %s \"%s\"", - ::GetCommandLineA(), - filter_flag.c_str(), - internal_flag.c_str()); + std::string command_line = + std::string(::GetCommandLineA()) + " " + filter_flag + " \"" + + internal_flag + "\""; DeathTest::set_last_death_test_message(""); @@ -816,6 +852,7 @@ DeathTest::TestRole NoExecDeathTest::AssumeRole() { // Event forwarding to the listeners of event listener API mush be shut // down in death test subprocesses. GetUnitTestImpl()->listeners()->SuppressEventForwarding(); + g_in_fast_death_test_child = true; return EXECUTE_TEST; } else { GTEST_DEATH_TEST_CHECK_SYSCALL_(close(pipe_fd[1])); @@ -835,6 +872,11 @@ class ExecDeathTest : public ForkingDeathTest { ForkingDeathTest(a_statement, a_regex), file_(file), line_(line) { } virtual TestRole AssumeRole(); private: + static ::std::vector<testing::internal::string> + GetArgvsForDeathTestChildProcess() { + ::std::vector<testing::internal::string> args = GetInjectableArgvs(); + return args; + } // The name of the file in which the death test is located. const char* const file_; // The line number on which the death test is located. @@ -869,6 +911,7 @@ class Arguments { char* const* Argv() { return &args_[0]; } + private: std::vector<char*> args_; }; @@ -894,6 +937,7 @@ extern "C" char** environ; inline char** GetEnviron() { return environ; } # endif // GTEST_OS_MAC +# if !GTEST_OS_QNX // The main function for a threadsafe-style death test child process. // This function is called in a clone()-ed process and thus must avoid // any potentially unsafe operations like malloc or libc functions. @@ -908,9 +952,8 @@ static int ExecDeathTestChildMain(void* child_arg) { UnitTest::GetInstance()->original_working_dir(); // We can safely call chdir() as it's a direct system call. if (chdir(original_dir) != 0) { - DeathTestAbort(String::Format("chdir(\"%s\") failed: %s", - original_dir, - GetLastErrnoDescription().c_str())); + DeathTestAbort(std::string("chdir(\"") + original_dir + "\") failed: " + + GetLastErrnoDescription()); return EXIT_FAILURE; } @@ -920,12 +963,12 @@ static int ExecDeathTestChildMain(void* child_arg) { // invoke the test program via a valid path that contains at least // one path separator. execve(args->argv[0], args->argv, GetEnviron()); - DeathTestAbort(String::Format("execve(%s, ...) in %s failed: %s", - args->argv[0], - original_dir, - GetLastErrnoDescription().c_str())); + DeathTestAbort(std::string("execve(") + args->argv[0] + ", ...) in " + + original_dir + " failed: " + + GetLastErrnoDescription()); return EXIT_FAILURE; } +# endif // !GTEST_OS_QNX // Two utility routines that together determine the direction the stack // grows. @@ -936,25 +979,77 @@ static int ExecDeathTestChildMain(void* child_arg) { // GTEST_NO_INLINE_ is required to prevent GCC 4.6 from inlining // StackLowerThanAddress into StackGrowsDown, which then doesn't give // correct answer. -bool StackLowerThanAddress(const void* ptr) GTEST_NO_INLINE_; -bool StackLowerThanAddress(const void* ptr) { +void StackLowerThanAddress(const void* ptr, bool* result) GTEST_NO_INLINE_; +void StackLowerThanAddress(const void* ptr, bool* result) { int dummy; - return &dummy < ptr; + *result = (&dummy < ptr); } +// Make sure AddressSanitizer does not tamper with the stack here. +GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ bool StackGrowsDown() { int dummy; - return StackLowerThanAddress(&dummy); + bool result; + StackLowerThanAddress(&dummy, &result); + return result; } -// A threadsafe implementation of fork(2) for threadsafe-style death tests -// that uses clone(2). It dies with an error message if anything goes -// wrong. -static pid_t ExecDeathTestFork(char* const* argv, int close_fd) { +// Spawns a child process with the same executable as the current process in +// a thread-safe manner and instructs it to run the death test. The +// implementation uses fork(2) + exec. On systems where clone(2) is +// available, it is used instead, being slightly more thread-safe. On QNX, +// fork supports only single-threaded environments, so this function uses +// spawn(2) there instead. The function dies with an error message if +// anything goes wrong. +static pid_t ExecDeathTestSpawnChild(char* const* argv, int close_fd) { ExecDeathTestArgs args = { argv, close_fd }; pid_t child_pid = -1; -# if GTEST_HAS_CLONE +# if GTEST_OS_QNX + // Obtains the current directory and sets it to be closed in the child + // process. + const int cwd_fd = open(".", O_RDONLY); + GTEST_DEATH_TEST_CHECK_(cwd_fd != -1); + GTEST_DEATH_TEST_CHECK_SYSCALL_(fcntl(cwd_fd, F_SETFD, FD_CLOEXEC)); + // We need to execute the test program in the same environment where + // it was originally invoked. Therefore we change to the original + // working directory first. + const char* const original_dir = + UnitTest::GetInstance()->original_working_dir(); + // We can safely call chdir() as it's a direct system call. + if (chdir(original_dir) != 0) { + DeathTestAbort(std::string("chdir(\"") + original_dir + "\") failed: " + + GetLastErrnoDescription()); + return EXIT_FAILURE; + } + + int fd_flags; + // Set close_fd to be closed after spawn. + GTEST_DEATH_TEST_CHECK_SYSCALL_(fd_flags = fcntl(close_fd, F_GETFD)); + GTEST_DEATH_TEST_CHECK_SYSCALL_(fcntl(close_fd, F_SETFD, + fd_flags | FD_CLOEXEC)); + struct inheritance inherit = {0}; + // spawn is a system call. + child_pid = spawn(args.argv[0], 0, NULL, &inherit, args.argv, GetEnviron()); + // Restores the current working directory. + GTEST_DEATH_TEST_CHECK_(fchdir(cwd_fd) != -1); + GTEST_DEATH_TEST_CHECK_SYSCALL_(close(cwd_fd)); + +# else // GTEST_OS_QNX +# if GTEST_OS_LINUX + // When a SIGPROF signal is received while fork() or clone() are executing, + // the process may hang. To avoid this, we ignore SIGPROF here and re-enable + // it after the call to fork()/clone() is complete. + struct sigaction saved_sigprof_action; + struct sigaction ignore_sigprof_action; + memset(&ignore_sigprof_action, 0, sizeof(ignore_sigprof_action)); + sigemptyset(&ignore_sigprof_action.sa_mask); + ignore_sigprof_action.sa_handler = SIG_IGN; + GTEST_DEATH_TEST_CHECK_SYSCALL_(sigaction( + SIGPROF, &ignore_sigprof_action, &saved_sigprof_action)); +# endif // GTEST_OS_LINUX + +# if GTEST_HAS_CLONE const bool use_fork = GTEST_FLAG(death_test_use_fork); if (!use_fork) { @@ -964,21 +1059,37 @@ static pid_t ExecDeathTestFork(char* const* argv, int close_fd) { void* const stack = mmap(NULL, stack_size, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0); GTEST_DEATH_TEST_CHECK_(stack != MAP_FAILED); + + // Maximum stack alignment in bytes: For a downward-growing stack, this + // amount is subtracted from size of the stack space to get an address + // that is within the stack space and is aligned on all systems we care + // about. As far as I know there is no ABI with stack alignment greater + // than 64. We assume stack and stack_size already have alignment of + // kMaxStackAlignment. + const size_t kMaxStackAlignment = 64; void* const stack_top = - static_cast<char*>(stack) + (stack_grows_down ? stack_size : 0); + static_cast<char*>(stack) + + (stack_grows_down ? stack_size - kMaxStackAlignment : 0); + GTEST_DEATH_TEST_CHECK_(stack_size > kMaxStackAlignment && + reinterpret_cast<intptr_t>(stack_top) % kMaxStackAlignment == 0); child_pid = clone(&ExecDeathTestChildMain, stack_top, SIGCHLD, &args); GTEST_DEATH_TEST_CHECK_(munmap(stack, stack_size) != -1); } -# else +# else const bool use_fork = true; -# endif // GTEST_HAS_CLONE +# endif // GTEST_HAS_CLONE if (use_fork && (child_pid = fork()) == 0) { ExecDeathTestChildMain(&args); _exit(0); } +# endif // GTEST_OS_QNX +# if GTEST_OS_LINUX + GTEST_DEATH_TEST_CHECK_SYSCALL_( + sigaction(SIGPROF, &saved_sigprof_action, NULL)); +# endif // GTEST_OS_LINUX GTEST_DEATH_TEST_CHECK_(child_pid != -1); return child_pid; @@ -1006,16 +1117,16 @@ DeathTest::TestRole ExecDeathTest::AssumeRole() { // it be closed when the child process does an exec: GTEST_DEATH_TEST_CHECK_(fcntl(pipe_fd[1], F_SETFD, 0) != -1); - const String filter_flag = - String::Format("--%s%s=%s.%s", - GTEST_FLAG_PREFIX_, kFilterFlag, - info->test_case_name(), info->name()); - const String internal_flag = - String::Format("--%s%s=%s|%d|%d|%d", - GTEST_FLAG_PREFIX_, kInternalRunDeathTestFlag, - file_, line_, death_test_index, pipe_fd[1]); + const std::string filter_flag = + std::string("--") + GTEST_FLAG_PREFIX_ + kFilterFlag + "=" + + info->test_case_name() + "." + info->name(); + const std::string internal_flag = + std::string("--") + GTEST_FLAG_PREFIX_ + kInternalRunDeathTestFlag + "=" + + file_ + "|" + StreamableToString(line_) + "|" + + StreamableToString(death_test_index) + "|" + + StreamableToString(pipe_fd[1]); Arguments args; - args.AddArguments(GetArgvs()); + args.AddArguments(GetArgvsForDeathTestChildProcess()); args.AddArgument(filter_flag.c_str()); args.AddArgument(internal_flag.c_str()); @@ -1026,7 +1137,7 @@ DeathTest::TestRole ExecDeathTest::AssumeRole() { // is necessary. FlushInfoLog(); - const pid_t child_pid = ExecDeathTestFork(args.Argv(), pipe_fd[0]); + const pid_t child_pid = ExecDeathTestSpawnChild(args.Argv(), pipe_fd[0]); GTEST_DEATH_TEST_CHECK_SYSCALL_(close(pipe_fd[1])); set_child_pid(child_pid); set_read_fd(pipe_fd[0]); @@ -1052,9 +1163,10 @@ bool DefaultDeathTestFactory::Create(const char* statement, const RE* regex, if (flag != NULL) { if (death_test_index > flag->index()) { - DeathTest::set_last_death_test_message(String::Format( - "Death test count (%d) somehow exceeded expected maximum (%d)", - death_test_index, flag->index())); + DeathTest::set_last_death_test_message( + "Death test count (" + StreamableToString(death_test_index) + + ") somehow exceeded expected maximum (" + + StreamableToString(flag->index()) + ")"); return false; } @@ -1083,9 +1195,9 @@ bool DefaultDeathTestFactory::Create(const char* statement, const RE* regex, # endif // GTEST_OS_WINDOWS else { // NOLINT - this is more readable than unbalanced brackets inside #if. - DeathTest::set_last_death_test_message(String::Format( - "Unknown death test style \"%s\" encountered", - GTEST_FLAG(death_test_style).c_str())); + DeathTest::set_last_death_test_message( + "Unknown death test style \"" + GTEST_FLAG(death_test_style) + + "\" encountered"); return false; } @@ -1123,8 +1235,8 @@ int GetStatusFileDescriptor(unsigned int parent_process_id, FALSE, // Non-inheritable. parent_process_id)); if (parent_process_handle.Get() == INVALID_HANDLE_VALUE) { - DeathTestAbort(String::Format("Unable to open parent process %u", - parent_process_id)); + DeathTestAbort("Unable to open parent process " + + StreamableToString(parent_process_id)); } // TODO(vladl@google.com): Replace the following check with a @@ -1144,9 +1256,10 @@ int GetStatusFileDescriptor(unsigned int parent_process_id, // DUPLICATE_SAME_ACCESS is used. FALSE, // Request non-inheritable handler. DUPLICATE_SAME_ACCESS)) { - DeathTestAbort(String::Format( - "Unable to duplicate the pipe handle %Iu from the parent process %u", - write_handle_as_size_t, parent_process_id)); + DeathTestAbort("Unable to duplicate the pipe handle " + + StreamableToString(write_handle_as_size_t) + + " from the parent process " + + StreamableToString(parent_process_id)); } const HANDLE event_handle = reinterpret_cast<HANDLE>(event_handle_as_size_t); @@ -1157,17 +1270,18 @@ int GetStatusFileDescriptor(unsigned int parent_process_id, 0x0, FALSE, DUPLICATE_SAME_ACCESS)) { - DeathTestAbort(String::Format( - "Unable to duplicate the event handle %Iu from the parent process %u", - event_handle_as_size_t, parent_process_id)); + DeathTestAbort("Unable to duplicate the event handle " + + StreamableToString(event_handle_as_size_t) + + " from the parent process " + + StreamableToString(parent_process_id)); } const int write_fd = ::_open_osfhandle(reinterpret_cast<intptr_t>(dup_write_handle), O_APPEND); if (write_fd == -1) { - DeathTestAbort(String::Format( - "Unable to convert pipe handle %Iu to a file descriptor", - write_handle_as_size_t)); + DeathTestAbort("Unable to convert pipe handle " + + StreamableToString(write_handle_as_size_t) + + " to a file descriptor"); } // Signals the parent that the write end of the pipe has been acquired @@ -1204,9 +1318,8 @@ InternalRunDeathTestFlag* ParseInternalRunDeathTestFlag() { || !ParseNaturalNumber(fields[3], &parent_process_id) || !ParseNaturalNumber(fields[4], &write_handle_as_size_t) || !ParseNaturalNumber(fields[5], &event_handle_as_size_t)) { - DeathTestAbort(String::Format( - "Bad --gtest_internal_run_death_test flag: %s", - GTEST_FLAG(internal_run_death_test).c_str())); + DeathTestAbort("Bad --gtest_internal_run_death_test flag: " + + GTEST_FLAG(internal_run_death_test)); } write_fd = GetStatusFileDescriptor(parent_process_id, write_handle_as_size_t, @@ -1217,9 +1330,8 @@ InternalRunDeathTestFlag* ParseInternalRunDeathTestFlag() { || !ParseNaturalNumber(fields[1], &line) || !ParseNaturalNumber(fields[2], &index) || !ParseNaturalNumber(fields[3], &write_fd)) { - DeathTestAbort(String::Format( - "Bad --gtest_internal_run_death_test flag: %s", - GTEST_FLAG(internal_run_death_test).c_str())); + DeathTestAbort("Bad --gtest_internal_run_death_test flag: " + + GTEST_FLAG(internal_run_death_test)); } # endif // GTEST_OS_WINDOWS diff --git a/tests/gtest/src/gtest-filepath.cc b/tests/gtest/src/gtest-filepath.cc index 91b257138..0292dc119 100644 --- a/tests/gtest/src/gtest-filepath.cc +++ b/tests/gtest/src/gtest-filepath.cc @@ -29,6 +29,7 @@ // // Authors: keith.ray@gmail.com (Keith Ray) +#include "gtest/gtest-message.h" #include "gtest/internal/gtest-filepath.h" #include "gtest/internal/gtest-port.h" @@ -39,8 +40,8 @@ #elif GTEST_OS_WINDOWS # include <direct.h> # include <io.h> -#elif GTEST_OS_SYMBIAN || GTEST_OS_NACL -// Symbian OpenC and NaCl have PATH_MAX in sys/syslimits.h +#elif GTEST_OS_SYMBIAN +// Symbian OpenC has PATH_MAX in sys/syslimits.h # include <sys/syslimits.h> #else # include <limits.h> @@ -69,7 +70,6 @@ namespace internal { // of them. const char kPathSeparator = '\\'; const char kAlternatePathSeparator = '/'; -const char kPathSeparatorString[] = "\\"; const char kAlternatePathSeparatorString[] = "/"; # if GTEST_OS_WINDOWS_MOBILE // Windows CE doesn't have a current directory. You should not use @@ -83,7 +83,6 @@ const char kCurrentDirectoryString[] = ".\\"; # endif // GTEST_OS_WINDOWS_MOBILE #else const char kPathSeparator = '/'; -const char kPathSeparatorString[] = "/"; const char kCurrentDirectoryString[] = "./"; #endif // GTEST_OS_WINDOWS @@ -98,7 +97,7 @@ static bool IsPathSeparator(char c) { // Returns the current working directory, or "" if unsuccessful. FilePath FilePath::GetCurrentDir() { -#if GTEST_OS_WINDOWS_MOBILE +#if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_WINDOWS_PHONE || GTEST_OS_WINDOWS_RT // Windows CE doesn't have a current directory, so we just return // something reasonable. return FilePath(kCurrentDirectoryString); @@ -107,7 +106,14 @@ FilePath FilePath::GetCurrentDir() { return FilePath(_getcwd(cwd, sizeof(cwd)) == NULL ? "" : cwd); #else char cwd[GTEST_PATH_MAX_ + 1] = { '\0' }; - return FilePath(getcwd(cwd, sizeof(cwd)) == NULL ? "" : cwd); + char* result = getcwd(cwd, sizeof(cwd)); +# if GTEST_OS_NACL + // getcwd will likely fail in NaCl due to the sandbox, so return something + // reasonable. The user may have provided a shim implementation for getcwd, + // however, so fallback only when failure is detected. + return FilePath(result == NULL ? kCurrentDirectoryString : cwd); +# endif // GTEST_OS_NACL + return FilePath(result == NULL ? "" : cwd); #endif // GTEST_OS_WINDOWS_MOBILE } @@ -116,9 +122,10 @@ FilePath FilePath::GetCurrentDir() { // FilePath("dir/file"). If a case-insensitive extension is not // found, returns a copy of the original FilePath. FilePath FilePath::RemoveExtension(const char* extension) const { - String dot_extension(String::Format(".%s", extension)); - if (pathname_.EndsWithCaseInsensitive(dot_extension.c_str())) { - return FilePath(String(pathname_.c_str(), pathname_.length() - 4)); + const std::string dot_extension = std::string(".") + extension; + if (String::EndsWithCaseInsensitive(pathname_, dot_extension)) { + return FilePath(pathname_.substr( + 0, pathname_.length() - dot_extension.length())); } return *this; } @@ -147,7 +154,7 @@ const char* FilePath::FindLastPathSeparator() const { // On Windows platform, '\' is the path separator, otherwise it is '/'. FilePath FilePath::RemoveDirectoryName() const { const char* const last_sep = FindLastPathSeparator(); - return last_sep ? FilePath(String(last_sep + 1)) : *this; + return last_sep ? FilePath(last_sep + 1) : *this; } // RemoveFileName returns the directory path with the filename removed. @@ -158,9 +165,9 @@ FilePath FilePath::RemoveDirectoryName() const { // On Windows platform, '\' is the path separator, otherwise it is '/'. FilePath FilePath::RemoveFileName() const { const char* const last_sep = FindLastPathSeparator(); - String dir; + std::string dir; if (last_sep) { - dir = String(c_str(), last_sep + 1 - c_str()); + dir = std::string(c_str(), last_sep + 1 - c_str()); } else { dir = kCurrentDirectoryString; } @@ -177,11 +184,12 @@ FilePath FilePath::MakeFileName(const FilePath& directory, const FilePath& base_name, int number, const char* extension) { - String file; + std::string file; if (number == 0) { - file = String::Format("%s.%s", base_name.c_str(), extension); + file = base_name.string() + "." + extension; } else { - file = String::Format("%s_%d.%s", base_name.c_str(), number, extension); + file = base_name.string() + "_" + StreamableToString(number) + + "." + extension; } return ConcatPaths(directory, FilePath(file)); } @@ -193,8 +201,7 @@ FilePath FilePath::ConcatPaths(const FilePath& directory, if (directory.IsEmpty()) return relative_path; const FilePath dir(directory.RemoveTrailingPathSeparator()); - return FilePath(String::Format("%s%c%s", dir.c_str(), kPathSeparator, - relative_path.c_str())); + return FilePath(dir.string() + kPathSeparator + relative_path.string()); } // Returns true if pathname describes something findable in the file-system, @@ -338,7 +345,7 @@ bool FilePath::CreateFolder() const { // On Windows platform, uses \ as the separator, other platforms use /. FilePath FilePath::RemoveTrailingPathSeparator() const { return IsDirectory() - ? FilePath(String(pathname_.c_str(), pathname_.length() - 1)) + ? FilePath(pathname_.substr(0, pathname_.length() - 1)) : *this; } diff --git a/tests/gtest/src/gtest-internal-inl.h b/tests/gtest/src/gtest-internal-inl.h index 65a2101a4..0ac7a109b 100644 --- a/tests/gtest/src/gtest-internal-inl.h +++ b/tests/gtest/src/gtest-internal-inl.h @@ -40,7 +40,7 @@ // GTEST_IMPLEMENTATION_ is defined to 1 iff the current translation unit is // part of Google Test's implementation; otherwise it's undefined. #if !GTEST_IMPLEMENTATION_ -// A user is trying to include this from his code - just say no. +// If this file is included from the user's code, just say no. # error "gtest-internal-inl.h is part of Google Test's internal implementation." # error "It must not be included except by Google Test itself." #endif // GTEST_IMPLEMENTATION_ @@ -58,6 +58,11 @@ #include "gtest/internal/gtest-port.h" +#if GTEST_CAN_STREAM_RESULTS_ +# include <arpa/inet.h> // NOLINT +# include <netdb.h> // NOLINT +#endif + #if GTEST_OS_WINDOWS # include <windows.h> // NOLINT #endif // GTEST_OS_WINDOWS @@ -112,6 +117,12 @@ GTEST_API_ bool ShouldUseColor(bool stdout_is_tty); // Formats the given time in milliseconds as seconds. GTEST_API_ std::string FormatTimeInMillisAsSeconds(TimeInMillis ms); +// Converts the given time in milliseconds to a date string in the ISO 8601 +// format, without the timezone information. N.B.: due to the use the +// non-reentrant localtime() function, this function is not thread safe. Do +// not use it in any code that can be called from multiple threads. +GTEST_API_ std::string FormatEpochTimeInMillisAsIso8601(TimeInMillis ms); + // Parses a string for an Int32 flag, in the form of "--flag=value". // // On success, stores the value of the flag in *value, and returns @@ -190,37 +201,35 @@ class GTestFlagSaver { GTEST_FLAG(stream_result_to) = stream_result_to_; GTEST_FLAG(throw_on_failure) = throw_on_failure_; } + private: // Fields for saving the original values of flags. bool also_run_disabled_tests_; bool break_on_failure_; bool catch_exceptions_; - String color_; - String death_test_style_; + std::string color_; + std::string death_test_style_; bool death_test_use_fork_; - String filter_; - String internal_run_death_test_; + std::string filter_; + std::string internal_run_death_test_; bool list_tests_; - String output_; + std::string output_; bool print_time_; - bool pretty_; internal::Int32 random_seed_; internal::Int32 repeat_; bool shuffle_; internal::Int32 stack_trace_depth_; - String stream_result_to_; + std::string stream_result_to_; bool throw_on_failure_; } GTEST_ATTRIBUTE_UNUSED_; // Converts a Unicode code point to a narrow string in UTF-8 encoding. // code_point parameter is of type UInt32 because wchar_t may not be // wide enough to contain a code point. -// The output buffer str must containt at least 32 characters. -// The function returns the address of the output buffer. // If the code_point is not a valid Unicode code point -// (i.e. outside of Unicode range U+0 to U+10FFFF) it will be output -// as '(Invalid Unicode 0xXXXXXXXX)'. -GTEST_API_ char* CodePointToUtf8(UInt32 code_point, char* str); +// (i.e. outside of Unicode range U+0 to U+10FFFF) it will be converted +// to "(Invalid Unicode 0xXXXXXXXX)". +GTEST_API_ std::string CodePointToUtf8(UInt32 code_point); // Converts a wide string to a narrow string in UTF-8 encoding. // The wide string is assumed to have the following encoding: @@ -235,7 +244,7 @@ GTEST_API_ char* CodePointToUtf8(UInt32 code_point, char* str); // as '(Invalid Unicode 0xXXXXXXXX)'. If the string is in UTF16 encoding // and contains invalid UTF-16 surrogate pairs, values in those pairs // will be encoded as individual Unicode characters from Basic Normal Plane. -GTEST_API_ String WideStringToUtf8(const wchar_t* str, int num_chars); +GTEST_API_ std::string WideStringToUtf8(const wchar_t* str, int num_chars); // Reads the GTEST_SHARD_STATUS_FILE environment variable, and creates the file // if the variable is present. If a file already exists at this location, this @@ -339,16 +348,15 @@ class TestPropertyKeyIs { // Constructor. // // TestPropertyKeyIs has NO default constructor. - explicit TestPropertyKeyIs(const char* key) - : key_(key) {} + explicit TestPropertyKeyIs(const std::string& key) : key_(key) {} // Returns true iff the test name of test property matches on key_. bool operator()(const TestProperty& test_property) const { - return String(test_property.key()).Compare(key_) == 0; + return test_property.key() == key_; } private: - String key_; + std::string key_; }; // Class UnitTestOptions. @@ -366,12 +374,12 @@ class GTEST_API_ UnitTestOptions { // Functions for processing the gtest_output flag. // Returns the output format, or "" for normal printed output. - static String GetOutputFormat(); + static std::string GetOutputFormat(); // Returns the absolute path of the requested output file, or the // default (test_detail.xml in the original working directory) if // none was explicitly specified. - static String GetAbsolutePathToOutputFile(); + static std::string GetAbsolutePathToOutputFile(); // Functions for processing the gtest_filter flag. @@ -384,8 +392,8 @@ class GTEST_API_ UnitTestOptions { // Returns true iff the user-specified filter matches the test case // name and the test name. - static bool FilterMatchesTest(const String &test_case_name, - const String &test_name); + static bool FilterMatchesTest(const std::string &test_case_name, + const std::string &test_name); #if GTEST_OS_WINDOWS // Function for supporting the gtest_catch_exception flag. @@ -398,7 +406,7 @@ class GTEST_API_ UnitTestOptions { // Returns true if "name" matches the ':' separated list of glob-style // filters in "filter". - static bool MatchesFilter(const String& name, const char* filter); + static bool MatchesFilter(const std::string& name, const char* filter); }; // Returns the current application's name, removing directory path if that @@ -411,13 +419,13 @@ class OsStackTraceGetterInterface { OsStackTraceGetterInterface() {} virtual ~OsStackTraceGetterInterface() {} - // Returns the current OS stack trace as a String. Parameters: + // Returns the current OS stack trace as an std::string. Parameters: // // max_depth - the maximum number of stack frames to be included // in the trace. // skip_count - the number of top frames to be skipped; doesn't count // against max_depth. - virtual String CurrentStackTrace(int max_depth, int skip_count) = 0; + virtual string CurrentStackTrace(int max_depth, int skip_count) = 0; // UponLeavingGTest() should be called immediately before Google Test calls // user code. It saves some information about the current stack that @@ -432,8 +440,11 @@ class OsStackTraceGetterInterface { class OsStackTraceGetter : public OsStackTraceGetterInterface { public: OsStackTraceGetter() : caller_frame_(NULL) {} - virtual String CurrentStackTrace(int max_depth, int skip_count); - virtual void UponLeavingGTest(); + + virtual string CurrentStackTrace(int max_depth, int skip_count) + GTEST_LOCK_EXCLUDED_(mutex_); + + virtual void UponLeavingGTest() GTEST_LOCK_EXCLUDED_(mutex_); // This string is inserted in place of stack frames that are part of // Google Test's implementation. @@ -455,7 +466,7 @@ class OsStackTraceGetter : public OsStackTraceGetterInterface { struct TraceInfo { const char* file; int line; - String message; + std::string message; }; // This is the default global test part result reporter used in UnitTestImpl. @@ -539,15 +550,25 @@ class GTEST_API_ UnitTestImpl { // Gets the number of failed tests. int failed_test_count() const; + // Gets the number of disabled tests that will be reported in the XML report. + int reportable_disabled_test_count() const; + // Gets the number of disabled tests. int disabled_test_count() const; + // Gets the number of tests to be printed in the XML report. + int reportable_test_count() const; + // Gets the number of all tests. int total_test_count() const; // Gets the number of tests that should run. int test_to_run_count() const; + // Gets the time of the test program start, in ms from the start of the + // UNIX epoch. + TimeInMillis start_timestamp() const { return start_timestamp_; } + // Gets the elapsed time, in milliseconds. TimeInMillis elapsed_time() const { return elapsed_time_; } @@ -596,7 +617,7 @@ class GTEST_API_ UnitTestImpl { // getter, and returns it. OsStackTraceGetterInterface* os_stack_trace_getter(); - // Returns the current OS stack trace as a String. + // Returns the current OS stack trace as an std::string. // // The maximum number of stack frames to be included is specified by // the gtest_stack_trace_depth flag. The skip_count parameter @@ -606,7 +627,7 @@ class GTEST_API_ UnitTestImpl { // For example, if Foo() calls Bar(), which in turn calls // CurrentOsStackTraceExceptTop(1), Foo() will be included in the // trace but Bar() and CurrentOsStackTraceExceptTop() won't. - String CurrentOsStackTraceExceptTop(int skip_count); + std::string CurrentOsStackTraceExceptTop(int skip_count) GTEST_NO_INLINE_; // Finds and returns a TestCase with the given name. If one doesn't // exist, creates one and returns it. @@ -696,6 +717,12 @@ class GTEST_API_ UnitTestImpl { ad_hoc_test_result_.Clear(); } + // Adds a TestProperty to the current TestResult object when invoked in a + // context of a test or a test case, or to the global property set. If the + // result already contains a property with the same key, the value will be + // updated. + void RecordProperty(const TestProperty& test_property); + enum ReactionToSharding { HONOR_SHARDING_PROTOCOL, IGNORE_SHARDING_PROTOCOL @@ -880,6 +907,10 @@ class GTEST_API_ UnitTestImpl { // Our random number generator. internal::Random random_; + // The time of the test program start, in ms from the start of the + // UNIX epoch. + TimeInMillis start_timestamp_; + // How long the test took to run, in milliseconds. TimeInMillis elapsed_time_; @@ -935,33 +966,7 @@ GTEST_API_ void ParseGoogleTestFlagsOnly(int* argc, wchar_t** argv); // Returns the message describing the last system error, regardless of the // platform. -GTEST_API_ String GetLastErrnoDescription(); - -# if GTEST_OS_WINDOWS -// Provides leak-safe Windows kernel handle ownership. -class AutoHandle { - public: - AutoHandle() : handle_(INVALID_HANDLE_VALUE) {} - explicit AutoHandle(HANDLE handle) : handle_(handle) {} - - ~AutoHandle() { Reset(); } - - HANDLE Get() const { return handle_; } - void Reset() { Reset(INVALID_HANDLE_VALUE); } - void Reset(HANDLE handle) { - if (handle != handle_) { - if (handle_ != INVALID_HANDLE_VALUE) - ::CloseHandle(handle_); - handle_ = handle; - } - } - - private: - HANDLE handle_; - - GTEST_DISALLOW_COPY_AND_ASSIGN_(AutoHandle); -}; -# endif // GTEST_OS_WINDOWS +GTEST_API_ std::string GetLastErrnoDescription(); // Attempts to parse a string into a positive integer pointed to by the // number parameter. Returns true if that is possible. @@ -1018,8 +1023,9 @@ bool ParseNaturalNumber(const ::std::string& str, Integer* number) { class TestResultAccessor { public: static void RecordProperty(TestResult* test_result, + const std::string& xml_element, const TestProperty& property) { - test_result->RecordProperty(property); + test_result->RecordProperty(xml_element, property); } static void ClearTestPartResults(TestResult* test_result) { @@ -1032,6 +1038,154 @@ class TestResultAccessor { } }; +#if GTEST_CAN_STREAM_RESULTS_ + +// Streams test results to the given port on the given host machine. +class StreamingListener : public EmptyTestEventListener { + public: + // Abstract base class for writing strings to a socket. + class AbstractSocketWriter { + public: + virtual ~AbstractSocketWriter() {} + + // Sends a string to the socket. + virtual void Send(const string& message) = 0; + + // Closes the socket. + virtual void CloseConnection() {} + + // Sends a string and a newline to the socket. + void SendLn(const string& message) { + Send(message + "\n"); + } + }; + + // Concrete class for actually writing strings to a socket. + class SocketWriter : public AbstractSocketWriter { + public: + SocketWriter(const string& host, const string& port) + : sockfd_(-1), host_name_(host), port_num_(port) { + MakeConnection(); + } + + virtual ~SocketWriter() { + if (sockfd_ != -1) + CloseConnection(); + } + + // Sends a string to the socket. + virtual void Send(const string& message) { + GTEST_CHECK_(sockfd_ != -1) + << "Send() can be called only when there is a connection."; + + const int len = static_cast<int>(message.length()); + if (write(sockfd_, message.c_str(), len) != len) { + GTEST_LOG_(WARNING) + << "stream_result_to: failed to stream to " + << host_name_ << ":" << port_num_; + } + } + + private: + // Creates a client socket and connects to the server. + void MakeConnection(); + + // Closes the socket. + void CloseConnection() { + GTEST_CHECK_(sockfd_ != -1) + << "CloseConnection() can be called only when there is a connection."; + + close(sockfd_); + sockfd_ = -1; + } + + int sockfd_; // socket file descriptor + const string host_name_; + const string port_num_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(SocketWriter); + }; // class SocketWriter + + // Escapes '=', '&', '%', and '\n' characters in str as "%xx". + static string UrlEncode(const char* str); + + StreamingListener(const string& host, const string& port) + : socket_writer_(new SocketWriter(host, port)) { Start(); } + + explicit StreamingListener(AbstractSocketWriter* socket_writer) + : socket_writer_(socket_writer) { Start(); } + + void OnTestProgramStart(const UnitTest& /* unit_test */) { + SendLn("event=TestProgramStart"); + } + + void OnTestProgramEnd(const UnitTest& unit_test) { + // Note that Google Test current only report elapsed time for each + // test iteration, not for the entire test program. + SendLn("event=TestProgramEnd&passed=" + FormatBool(unit_test.Passed())); + + // Notify the streaming server to stop. + socket_writer_->CloseConnection(); + } + + void OnTestIterationStart(const UnitTest& /* unit_test */, int iteration) { + SendLn("event=TestIterationStart&iteration=" + + StreamableToString(iteration)); + } + + void OnTestIterationEnd(const UnitTest& unit_test, int /* iteration */) { + SendLn("event=TestIterationEnd&passed=" + + FormatBool(unit_test.Passed()) + "&elapsed_time=" + + StreamableToString(unit_test.elapsed_time()) + "ms"); + } + + void OnTestCaseStart(const TestCase& test_case) { + SendLn(std::string("event=TestCaseStart&name=") + test_case.name()); + } + + void OnTestCaseEnd(const TestCase& test_case) { + SendLn("event=TestCaseEnd&passed=" + FormatBool(test_case.Passed()) + + "&elapsed_time=" + StreamableToString(test_case.elapsed_time()) + + "ms"); + } + + void OnTestStart(const TestInfo& test_info) { + SendLn(std::string("event=TestStart&name=") + test_info.name()); + } + + void OnTestEnd(const TestInfo& test_info) { + SendLn("event=TestEnd&passed=" + + FormatBool((test_info.result())->Passed()) + + "&elapsed_time=" + + StreamableToString((test_info.result())->elapsed_time()) + "ms"); + } + + void OnTestPartResult(const TestPartResult& test_part_result) { + const char* file_name = test_part_result.file_name(); + if (file_name == NULL) + file_name = ""; + SendLn("event=TestPartResult&file=" + UrlEncode(file_name) + + "&line=" + StreamableToString(test_part_result.line_number()) + + "&message=" + UrlEncode(test_part_result.message())); + } + + private: + // Sends the given message and a newline to the socket. + void SendLn(const string& message) { socket_writer_->SendLn(message); } + + // Called at the start of streaming to notify the receiver what + // protocol we are using. + void Start() { SendLn("gtest_streaming_protocol_version=1.0"); } + + string FormatBool(bool value) { return value ? "1" : "0"; } + + const scoped_ptr<AbstractSocketWriter> socket_writer_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(StreamingListener); +}; // class StreamingListener + +#endif // GTEST_CAN_STREAM_RESULTS_ + } // namespace internal } // namespace testing diff --git a/tests/gtest/src/gtest-port.cc b/tests/gtest/src/gtest-port.cc index b860d4812..b032745b4 100644 --- a/tests/gtest/src/gtest-port.cc +++ b/tests/gtest/src/gtest-port.cc @@ -36,14 +36,14 @@ #include <stdio.h> #include <string.h> -#if GTEST_OS_WINDOWS_MOBILE -# include <windows.h> // For TerminateProcess() -#elif GTEST_OS_WINDOWS +#if GTEST_OS_WINDOWS +# include <windows.h> # include <io.h> # include <sys/stat.h> +# include <map> // Used in ThreadLocal. #else # include <unistd.h> -#endif // GTEST_OS_WINDOWS_MOBILE +#endif // GTEST_OS_WINDOWS #if GTEST_OS_MAC # include <mach/mach_init.h> @@ -51,6 +51,12 @@ # include <mach/vm_map.h> #endif // GTEST_OS_MAC +#if GTEST_OS_QNX +# include <devctl.h> +# include <fcntl.h> +# include <sys/procfs.h> +#endif // GTEST_OS_QNX + #include "gtest/gtest-spi.h" #include "gtest/gtest-message.h" #include "gtest/internal/gtest-internal.h" @@ -58,9 +64,9 @@ // Indicates that this translation unit is part of Google Test's // implementation. It must come before gtest-internal-inl.h is -// included, or there will be a compiler error. This trick is to -// prevent a user from accidentally including gtest-internal-inl.h in -// his code. +// included, or there will be a compiler error. This trick exists to +// prevent the accidental inclusion of gtest-internal-inl.h in the +// user's code. #define GTEST_IMPLEMENTATION_ 1 #include "src/gtest-internal-inl.h" #undef GTEST_IMPLEMENTATION_ @@ -98,6 +104,26 @@ size_t GetThreadCount() { } } +#elif GTEST_OS_QNX + +// Returns the number of threads running in the process, or 0 to indicate that +// we cannot detect it. +size_t GetThreadCount() { + const int fd = open("/proc/self/as", O_RDONLY); + if (fd < 0) { + return 0; + } + procfs_info process_info; + const int status = + devctl(fd, DCMD_PROC_INFO, &process_info, sizeof(process_info), NULL); + close(fd); + if (status == EOK) { + return static_cast<size_t>(process_info.num_threads); + } else { + return 0; + } +} + #else size_t GetThreadCount() { @@ -108,6 +134,389 @@ size_t GetThreadCount() { #endif // GTEST_OS_MAC +#if GTEST_IS_THREADSAFE && GTEST_OS_WINDOWS + +void SleepMilliseconds(int n) { + ::Sleep(n); +} + +AutoHandle::AutoHandle() + : handle_(INVALID_HANDLE_VALUE) {} + +AutoHandle::AutoHandle(Handle handle) + : handle_(handle) {} + +AutoHandle::~AutoHandle() { + Reset(); +} + +AutoHandle::Handle AutoHandle::Get() const { + return handle_; +} + +void AutoHandle::Reset() { + Reset(INVALID_HANDLE_VALUE); +} + +void AutoHandle::Reset(HANDLE handle) { + // Resetting with the same handle we already own is invalid. + if (handle_ != handle) { + if (IsCloseable()) { + ::CloseHandle(handle_); + } + handle_ = handle; + } else { + GTEST_CHECK_(!IsCloseable()) + << "Resetting a valid handle to itself is likely a programmer error " + "and thus not allowed."; + } +} + +bool AutoHandle::IsCloseable() const { + // Different Windows APIs may use either of these values to represent an + // invalid handle. + return handle_ != NULL && handle_ != INVALID_HANDLE_VALUE; +} + +Notification::Notification() + : event_(::CreateEvent(NULL, // Default security attributes. + TRUE, // Do not reset automatically. + FALSE, // Initially unset. + NULL)) { // Anonymous event. + GTEST_CHECK_(event_.Get() != NULL); +} + +void Notification::Notify() { + GTEST_CHECK_(::SetEvent(event_.Get()) != FALSE); +} + +void Notification::WaitForNotification() { + GTEST_CHECK_( + ::WaitForSingleObject(event_.Get(), INFINITE) == WAIT_OBJECT_0); +} + +Mutex::Mutex() + : type_(kDynamic), + owner_thread_id_(0), + critical_section_init_phase_(0), + critical_section_(new CRITICAL_SECTION) { + ::InitializeCriticalSection(critical_section_); +} + +Mutex::~Mutex() { + // Static mutexes are leaked intentionally. It is not thread-safe to try + // to clean them up. + // TODO(yukawa): Switch to Slim Reader/Writer (SRW) Locks, which requires + // nothing to clean it up but is available only on Vista and later. + // http://msdn.microsoft.com/en-us/library/windows/desktop/aa904937.aspx + if (type_ == kDynamic) { + ::DeleteCriticalSection(critical_section_); + delete critical_section_; + critical_section_ = NULL; + } +} + +void Mutex::Lock() { + ThreadSafeLazyInit(); + ::EnterCriticalSection(critical_section_); + owner_thread_id_ = ::GetCurrentThreadId(); +} + +void Mutex::Unlock() { + ThreadSafeLazyInit(); + // We don't protect writing to owner_thread_id_ here, as it's the + // caller's responsibility to ensure that the current thread holds the + // mutex when this is called. + owner_thread_id_ = 0; + ::LeaveCriticalSection(critical_section_); +} + +// Does nothing if the current thread holds the mutex. Otherwise, crashes +// with high probability. +void Mutex::AssertHeld() { + ThreadSafeLazyInit(); + GTEST_CHECK_(owner_thread_id_ == ::GetCurrentThreadId()) + << "The current thread is not holding the mutex @" << this; +} + +// Initializes owner_thread_id_ and critical_section_ in static mutexes. +void Mutex::ThreadSafeLazyInit() { + // Dynamic mutexes are initialized in the constructor. + if (type_ == kStatic) { + switch ( + ::InterlockedCompareExchange(&critical_section_init_phase_, 1L, 0L)) { + case 0: + // If critical_section_init_phase_ was 0 before the exchange, we + // are the first to test it and need to perform the initialization. + owner_thread_id_ = 0; + critical_section_ = new CRITICAL_SECTION; + ::InitializeCriticalSection(critical_section_); + // Updates the critical_section_init_phase_ to 2 to signal + // initialization complete. + GTEST_CHECK_(::InterlockedCompareExchange( + &critical_section_init_phase_, 2L, 1L) == + 1L); + break; + case 1: + // Somebody else is already initializing the mutex; spin until they + // are done. + while (::InterlockedCompareExchange(&critical_section_init_phase_, + 2L, + 2L) != 2L) { + // Possibly yields the rest of the thread's time slice to other + // threads. + ::Sleep(0); + } + break; + + case 2: + break; // The mutex is already initialized and ready for use. + + default: + GTEST_CHECK_(false) + << "Unexpected value of critical_section_init_phase_ " + << "while initializing a static mutex."; + } + } +} + +namespace { + +class ThreadWithParamSupport : public ThreadWithParamBase { + public: + static HANDLE CreateThread(Runnable* runnable, + Notification* thread_can_start) { + ThreadMainParam* param = new ThreadMainParam(runnable, thread_can_start); + DWORD thread_id; + // TODO(yukawa): Consider to use _beginthreadex instead. + HANDLE thread_handle = ::CreateThread( + NULL, // Default security. + 0, // Default stack size. + &ThreadWithParamSupport::ThreadMain, + param, // Parameter to ThreadMainStatic + 0x0, // Default creation flags. + &thread_id); // Need a valid pointer for the call to work under Win98. + GTEST_CHECK_(thread_handle != NULL) << "CreateThread failed with error " + << ::GetLastError() << "."; + if (thread_handle == NULL) { + delete param; + } + return thread_handle; + } + + private: + struct ThreadMainParam { + ThreadMainParam(Runnable* runnable, Notification* thread_can_start) + : runnable_(runnable), + thread_can_start_(thread_can_start) { + } + scoped_ptr<Runnable> runnable_; + // Does not own. + Notification* thread_can_start_; + }; + + static DWORD WINAPI ThreadMain(void* ptr) { + // Transfers ownership. + scoped_ptr<ThreadMainParam> param(static_cast<ThreadMainParam*>(ptr)); + if (param->thread_can_start_ != NULL) + param->thread_can_start_->WaitForNotification(); + param->runnable_->Run(); + return 0; + } + + // Prohibit instantiation. + ThreadWithParamSupport(); + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ThreadWithParamSupport); +}; + +} // namespace + +ThreadWithParamBase::ThreadWithParamBase(Runnable *runnable, + Notification* thread_can_start) + : thread_(ThreadWithParamSupport::CreateThread(runnable, + thread_can_start)) { +} + +ThreadWithParamBase::~ThreadWithParamBase() { + Join(); +} + +void ThreadWithParamBase::Join() { + GTEST_CHECK_(::WaitForSingleObject(thread_.Get(), INFINITE) == WAIT_OBJECT_0) + << "Failed to join the thread with error " << ::GetLastError() << "."; +} + +// Maps a thread to a set of ThreadIdToThreadLocals that have values +// instantiated on that thread and notifies them when the thread exits. A +// ThreadLocal instance is expected to persist until all threads it has +// values on have terminated. +class ThreadLocalRegistryImpl { + public: + // Registers thread_local_instance as having value on the current thread. + // Returns a value that can be used to identify the thread from other threads. + static ThreadLocalValueHolderBase* GetValueOnCurrentThread( + const ThreadLocalBase* thread_local_instance) { + DWORD current_thread = ::GetCurrentThreadId(); + MutexLock lock(&mutex_); + ThreadIdToThreadLocals* const thread_to_thread_locals = + GetThreadLocalsMapLocked(); + ThreadIdToThreadLocals::iterator thread_local_pos = + thread_to_thread_locals->find(current_thread); + if (thread_local_pos == thread_to_thread_locals->end()) { + thread_local_pos = thread_to_thread_locals->insert( + std::make_pair(current_thread, ThreadLocalValues())).first; + StartWatcherThreadFor(current_thread); + } + ThreadLocalValues& thread_local_values = thread_local_pos->second; + ThreadLocalValues::iterator value_pos = + thread_local_values.find(thread_local_instance); + if (value_pos == thread_local_values.end()) { + value_pos = + thread_local_values + .insert(std::make_pair( + thread_local_instance, + linked_ptr<ThreadLocalValueHolderBase>( + thread_local_instance->NewValueForCurrentThread()))) + .first; + } + return value_pos->second.get(); + } + + static void OnThreadLocalDestroyed( + const ThreadLocalBase* thread_local_instance) { + std::vector<linked_ptr<ThreadLocalValueHolderBase> > value_holders; + // Clean up the ThreadLocalValues data structure while holding the lock, but + // defer the destruction of the ThreadLocalValueHolderBases. + { + MutexLock lock(&mutex_); + ThreadIdToThreadLocals* const thread_to_thread_locals = + GetThreadLocalsMapLocked(); + for (ThreadIdToThreadLocals::iterator it = + thread_to_thread_locals->begin(); + it != thread_to_thread_locals->end(); + ++it) { + ThreadLocalValues& thread_local_values = it->second; + ThreadLocalValues::iterator value_pos = + thread_local_values.find(thread_local_instance); + if (value_pos != thread_local_values.end()) { + value_holders.push_back(value_pos->second); + thread_local_values.erase(value_pos); + // This 'if' can only be successful at most once, so theoretically we + // could break out of the loop here, but we don't bother doing so. + } + } + } + // Outside the lock, let the destructor for 'value_holders' deallocate the + // ThreadLocalValueHolderBases. + } + + static void OnThreadExit(DWORD thread_id) { + GTEST_CHECK_(thread_id != 0) << ::GetLastError(); + std::vector<linked_ptr<ThreadLocalValueHolderBase> > value_holders; + // Clean up the ThreadIdToThreadLocals data structure while holding the + // lock, but defer the destruction of the ThreadLocalValueHolderBases. + { + MutexLock lock(&mutex_); + ThreadIdToThreadLocals* const thread_to_thread_locals = + GetThreadLocalsMapLocked(); + ThreadIdToThreadLocals::iterator thread_local_pos = + thread_to_thread_locals->find(thread_id); + if (thread_local_pos != thread_to_thread_locals->end()) { + ThreadLocalValues& thread_local_values = thread_local_pos->second; + for (ThreadLocalValues::iterator value_pos = + thread_local_values.begin(); + value_pos != thread_local_values.end(); + ++value_pos) { + value_holders.push_back(value_pos->second); + } + thread_to_thread_locals->erase(thread_local_pos); + } + } + // Outside the lock, let the destructor for 'value_holders' deallocate the + // ThreadLocalValueHolderBases. + } + + private: + // In a particular thread, maps a ThreadLocal object to its value. + typedef std::map<const ThreadLocalBase*, + linked_ptr<ThreadLocalValueHolderBase> > ThreadLocalValues; + // Stores all ThreadIdToThreadLocals having values in a thread, indexed by + // thread's ID. + typedef std::map<DWORD, ThreadLocalValues> ThreadIdToThreadLocals; + + // Holds the thread id and thread handle that we pass from + // StartWatcherThreadFor to WatcherThreadFunc. + typedef std::pair<DWORD, HANDLE> ThreadIdAndHandle; + + static void StartWatcherThreadFor(DWORD thread_id) { + // The returned handle will be kept in thread_map and closed by + // watcher_thread in WatcherThreadFunc. + HANDLE thread = ::OpenThread(SYNCHRONIZE | THREAD_QUERY_INFORMATION, + FALSE, + thread_id); + GTEST_CHECK_(thread != NULL); + // We need to to pass a valid thread ID pointer into CreateThread for it + // to work correctly under Win98. + DWORD watcher_thread_id; + HANDLE watcher_thread = ::CreateThread( + NULL, // Default security. + 0, // Default stack size + &ThreadLocalRegistryImpl::WatcherThreadFunc, + reinterpret_cast<LPVOID>(new ThreadIdAndHandle(thread_id, thread)), + CREATE_SUSPENDED, + &watcher_thread_id); + GTEST_CHECK_(watcher_thread != NULL); + // Give the watcher thread the same priority as ours to avoid being + // blocked by it. + ::SetThreadPriority(watcher_thread, + ::GetThreadPriority(::GetCurrentThread())); + ::ResumeThread(watcher_thread); + ::CloseHandle(watcher_thread); + } + + // Monitors exit from a given thread and notifies those + // ThreadIdToThreadLocals about thread termination. + static DWORD WINAPI WatcherThreadFunc(LPVOID param) { + const ThreadIdAndHandle* tah = + reinterpret_cast<const ThreadIdAndHandle*>(param); + GTEST_CHECK_( + ::WaitForSingleObject(tah->second, INFINITE) == WAIT_OBJECT_0); + OnThreadExit(tah->first); + ::CloseHandle(tah->second); + delete tah; + return 0; + } + + // Returns map of thread local instances. + static ThreadIdToThreadLocals* GetThreadLocalsMapLocked() { + mutex_.AssertHeld(); + static ThreadIdToThreadLocals* map = new ThreadIdToThreadLocals; + return map; + } + + // Protects access to GetThreadLocalsMapLocked() and its return value. + static Mutex mutex_; + // Protects access to GetThreadMapLocked() and its return value. + static Mutex thread_map_mutex_; +}; + +Mutex ThreadLocalRegistryImpl::mutex_(Mutex::kStaticMutex); +Mutex ThreadLocalRegistryImpl::thread_map_mutex_(Mutex::kStaticMutex); + +ThreadLocalValueHolderBase* ThreadLocalRegistry::GetValueOnCurrentThread( + const ThreadLocalBase* thread_local_instance) { + return ThreadLocalRegistryImpl::GetValueOnCurrentThread( + thread_local_instance); +} + +void ThreadLocalRegistry::OnThreadLocalDestroyed( + const ThreadLocalBase* thread_local_instance) { + ThreadLocalRegistryImpl::OnThreadLocalDestroyed(thread_local_instance); +} + +#endif // GTEST_IS_THREADSAFE && GTEST_OS_WINDOWS + #if GTEST_USES_POSIX_RE // Implements RE. Currently only needed for death tests. @@ -222,7 +631,7 @@ bool AtomMatchesChar(bool escaped, char pattern_char, char ch) { } // Helper function used by ValidateRegex() to format error messages. -String FormatRegexSyntaxError(const char* regex, int index) { +std::string FormatRegexSyntaxError(const char* regex, int index) { return (Message() << "Syntax error at index " << index << " in simple regular expression \"" << regex << "\": ").GetString(); } @@ -429,15 +838,15 @@ const char kUnknownFile[] = "unknown file"; // Formats a source file path and a line number as they would appear // in an error message from the compiler used to compile this code. GTEST_API_ ::std::string FormatFileLocation(const char* file, int line) { - const char* const file_name = file == NULL ? kUnknownFile : file; + const std::string file_name(file == NULL ? kUnknownFile : file); if (line < 0) { - return String::Format("%s:", file_name).c_str(); + return file_name + ":"; } #ifdef _MSC_VER - return String::Format("%s(%d):", file_name, line).c_str(); + return file_name + "(" + StreamableToString(line) + "):"; #else - return String::Format("%s:%d:", file_name, line).c_str(); + return file_name + ":" + StreamableToString(line) + ":"; #endif // _MSC_VER } @@ -448,12 +857,12 @@ GTEST_API_ ::std::string FormatFileLocation(const char* file, int line) { // to the file location it produces, unlike FormatFileLocation(). GTEST_API_ ::std::string FormatCompilerIndependentFileLocation( const char* file, int line) { - const char* const file_name = file == NULL ? kUnknownFile : file; + const std::string file_name(file == NULL ? kUnknownFile : file); if (line < 0) return file_name; else - return String::Format("%s:%d", file_name, line).c_str(); + return file_name + ":" + StreamableToString(line); } @@ -477,10 +886,7 @@ GTestLog::~GTestLog() { } // Disable Microsoft deprecation warnings for POSIX functions called from // this class (creat, dup, dup2, and close) -#ifdef _MSC_VER -# pragma warning(push) -# pragma warning(disable: 4996) -#endif // _MSC_VER +GTEST_DISABLE_MSC_WARNINGS_PUSH_(4996) #if GTEST_HAS_STREAM_REDIRECTION @@ -488,8 +894,7 @@ GTestLog::~GTestLog() { class CapturedStream { public: // The ctor redirects the stream to a temporary file. - CapturedStream(int fd) : fd_(fd), uncaptured_fd_(dup(fd)) { - + explicit CapturedStream(int fd) : fd_(fd), uncaptured_fd_(dup(fd)) { # if GTEST_OS_WINDOWS char temp_dir_path[MAX_PATH + 1] = { '\0' }; // NOLINT char temp_file_path[MAX_PATH + 1] = { '\0' }; // NOLINT @@ -506,10 +911,29 @@ class CapturedStream { << temp_file_path; filename_ = temp_file_path; # else - // There's no guarantee that a test has write access to the - // current directory, so we create the temporary file in the /tmp - // directory instead. + // There's no guarantee that a test has write access to the current + // directory, so we create the temporary file in the /tmp directory + // instead. We use /tmp on most systems, and /sdcard on Android. + // That's because Android doesn't have /tmp. +# if GTEST_OS_LINUX_ANDROID + // Note: Android applications are expected to call the framework's + // Context.getExternalStorageDirectory() method through JNI to get + // the location of the world-writable SD Card directory. However, + // this requires a Context handle, which cannot be retrieved + // globally from native code. Doing so also precludes running the + // code as part of a regular standalone executable, which doesn't + // run in a Dalvik process (e.g. when running it through 'adb shell'). + // + // The location /sdcard is directly accessible from native code + // and is the only location (unofficially) supported by the Android + // team. It's generally a symlink to the real SD Card mount point + // which can be /mnt/sdcard, /mnt/sdcard0, /system/media/sdcard, or + // other OEM-customized locations. Never rely on these, and always + // use /sdcard. + char name_template[] = "/sdcard/gtest_captured_stream.XXXXXX"; +# else char name_template[] = "/tmp/captured_stream.XXXXXX"; +# endif // GTEST_OS_LINUX_ANDROID const int captured_fd = mkstemp(name_template); filename_ = name_template; # endif // GTEST_OS_WINDOWS @@ -522,7 +946,7 @@ class CapturedStream { remove(filename_.c_str()); } - String GetCapturedString() { + std::string GetCapturedString() { if (uncaptured_fd_ != -1) { // Restores the original stream. fflush(NULL); @@ -532,14 +956,14 @@ class CapturedStream { } FILE* const file = posix::FOpen(filename_.c_str(), "r"); - const String content = ReadEntireFile(file); + const std::string content = ReadEntireFile(file); posix::FClose(file); return content; } private: - // Reads the entire content of a file as a String. - static String ReadEntireFile(FILE* file); + // Reads the entire content of a file as an std::string. + static std::string ReadEntireFile(FILE* file); // Returns the size (in bytes) of a file. static size_t GetFileSize(FILE* file); @@ -559,7 +983,7 @@ size_t CapturedStream::GetFileSize(FILE* file) { } // Reads the entire content of a file as a string. -String CapturedStream::ReadEntireFile(FILE* file) { +std::string CapturedStream::ReadEntireFile(FILE* file) { const size_t file_size = GetFileSize(file); char* const buffer = new char[file_size]; @@ -575,15 +999,13 @@ String CapturedStream::ReadEntireFile(FILE* file) { bytes_read += bytes_last_read; } while (bytes_last_read > 0 && bytes_read < file_size); - const String content(buffer, bytes_read); + const std::string content(buffer, bytes_read); delete[] buffer; return content; } -# ifdef _MSC_VER -# pragma warning(pop) -# endif // _MSC_VER +GTEST_DISABLE_MSC_WARNINGS_POP_() static CapturedStream* g_captured_stderr = NULL; static CapturedStream* g_captured_stdout = NULL; @@ -598,8 +1020,8 @@ void CaptureStream(int fd, const char* stream_name, CapturedStream** stream) { } // Stops capturing the output stream and returns the captured string. -String GetCapturedStream(CapturedStream** captured_stream) { - const String content = (*captured_stream)->GetCapturedString(); +std::string GetCapturedStream(CapturedStream** captured_stream) { + const std::string content = (*captured_stream)->GetCapturedString(); delete *captured_stream; *captured_stream = NULL; @@ -618,21 +1040,37 @@ void CaptureStderr() { } // Stops capturing stdout and returns the captured string. -String GetCapturedStdout() { return GetCapturedStream(&g_captured_stdout); } +std::string GetCapturedStdout() { + return GetCapturedStream(&g_captured_stdout); +} // Stops capturing stderr and returns the captured string. -String GetCapturedStderr() { return GetCapturedStream(&g_captured_stderr); } +std::string GetCapturedStderr() { + return GetCapturedStream(&g_captured_stderr); +} #endif // GTEST_HAS_STREAM_REDIRECTION #if GTEST_HAS_DEATH_TEST // A copy of all command line arguments. Set by InitGoogleTest(). -::std::vector<String> g_argvs; +::std::vector<testing::internal::string> g_argvs; + +static const ::std::vector<testing::internal::string>* g_injected_test_argvs = + NULL; // Owned. -// Returns the command line as a vector of strings. -const ::std::vector<String>& GetArgvs() { return g_argvs; } +void SetInjectableArgvs(const ::std::vector<testing::internal::string>* argvs) { + if (g_injected_test_argvs != argvs) + delete g_injected_test_argvs; + g_injected_test_argvs = argvs; +} +const ::std::vector<testing::internal::string>& GetInjectableArgvs() { + if (g_injected_test_argvs != NULL) { + return *g_injected_test_argvs; + } + return g_argvs; +} #endif // GTEST_HAS_DEATH_TEST #if GTEST_OS_WINDOWS_MOBILE @@ -647,8 +1085,8 @@ void Abort() { // Returns the name of the environment variable corresponding to the // given flag. For example, FlagToEnvVar("foo") will return // "GTEST_FOO" in the open-source version. -static String FlagToEnvVar(const char* flag) { - const String full_flag = +static std::string FlagToEnvVar(const char* flag) { + const std::string full_flag = (Message() << GTEST_FLAG_PREFIX_ << flag).GetString(); Message env_var; @@ -705,7 +1143,7 @@ bool ParseInt32(const Message& src_text, const char* str, Int32* value) { // // The value is considered true iff it's not "0". bool BoolFromGTestEnv(const char* flag, bool default_value) { - const String env_var = FlagToEnvVar(flag); + const std::string env_var = FlagToEnvVar(flag); const char* const string_value = posix::GetEnv(env_var.c_str()); return string_value == NULL ? default_value : strcmp(string_value, "0") != 0; @@ -715,7 +1153,7 @@ bool BoolFromGTestEnv(const char* flag, bool default_value) { // variable corresponding to the given flag; if it isn't set or // doesn't represent a valid 32-bit integer, returns default_value. Int32 Int32FromGTestEnv(const char* flag, Int32 default_value) { - const String env_var = FlagToEnvVar(flag); + const std::string env_var = FlagToEnvVar(flag); const char* const string_value = posix::GetEnv(env_var.c_str()); if (string_value == NULL) { // The environment variable is not set. @@ -737,7 +1175,7 @@ Int32 Int32FromGTestEnv(const char* flag, Int32 default_value) { // Reads and returns the string environment variable corresponding to // the given flag; if it's not set, returns default_value. const char* StringFromGTestEnv(const char* flag, const char* default_value) { - const String env_var = FlagToEnvVar(flag); + const std::string env_var = FlagToEnvVar(flag); const char* const value = posix::GetEnv(env_var.c_str()); return value == NULL ? default_value : value; } diff --git a/tests/gtest/src/gtest-printers.cc b/tests/gtest/src/gtest-printers.cc index ed63c7b3b..a2df412f8 100644 --- a/tests/gtest/src/gtest-printers.cc +++ b/tests/gtest/src/gtest-printers.cc @@ -45,6 +45,7 @@ #include "gtest/gtest-printers.h" #include <ctype.h> #include <stdio.h> +#include <cwchar> #include <ostream> // NOLINT #include <string> #include "gtest/internal/gtest-port.h" @@ -55,15 +56,10 @@ namespace { using ::std::ostream; -#if GTEST_OS_WINDOWS_MOBILE // Windows CE does not define _snprintf_s. -# define snprintf _snprintf -#elif _MSC_VER >= 1400 // VC 8.0 and later deprecate snprintf and _snprintf. -# define snprintf _snprintf_s -#elif _MSC_VER -# define snprintf _snprintf -#endif // GTEST_OS_WINDOWS_MOBILE - // Prints a segment of bytes in the given object. +GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ +GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ +GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ void PrintByteSegmentInObjectTo(const unsigned char* obj_bytes, size_t start, size_t count, ostream* os) { char text[5] = ""; @@ -77,7 +73,7 @@ void PrintByteSegmentInObjectTo(const unsigned char* obj_bytes, size_t start, else *os << '-'; } - snprintf(text, sizeof(text), "%02X", obj_bytes[j]); + GTEST_SNPRINTF_(text, sizeof(text), "%02X", obj_bytes[j]); *os << text; } } @@ -184,16 +180,16 @@ static CharFormat PrintAsCharLiteralTo(Char c, ostream* os) { *os << static_cast<char>(c); return kAsIs; } else { - *os << String::Format("\\x%X", static_cast<UnsignedChar>(c)); + *os << "\\x" + String::FormatHexInt(static_cast<UnsignedChar>(c)); return kHexEscape; } } return kSpecialEscape; } -// Prints a char c as if it's part of a string literal, escaping it when +// Prints a wchar_t c as if it's part of a string literal, escaping it when // necessary; returns how c was formatted. -static CharFormat PrintAsWideStringLiteralTo(wchar_t c, ostream* os) { +static CharFormat PrintAsStringLiteralTo(wchar_t c, ostream* os) { switch (c) { case L'\'': *os << "'"; @@ -208,8 +204,9 @@ static CharFormat PrintAsWideStringLiteralTo(wchar_t c, ostream* os) { // Prints a char c as if it's part of a string literal, escaping it when // necessary; returns how c was formatted. -static CharFormat PrintAsNarrowStringLiteralTo(char c, ostream* os) { - return PrintAsWideStringLiteralTo(static_cast<unsigned char>(c), os); +static CharFormat PrintAsStringLiteralTo(char c, ostream* os) { + return PrintAsStringLiteralTo( + static_cast<wchar_t>(static_cast<unsigned char>(c)), os); } // Prints a wide or narrow character c and its code. '\0' is printed @@ -228,7 +225,7 @@ void PrintCharAndCodeTo(Char c, ostream* os) { // obvious). if (c == 0) return; - *os << " (" << String::Format("%d", c).c_str(); + *os << " (" << static_cast<int>(c); // For more convenience, we print c's code again in hexidecimal, // unless c was already printed in the form '\x##' or the code is in @@ -236,8 +233,7 @@ void PrintCharAndCodeTo(Char c, ostream* os) { if (format == kHexEscape || (1 <= c && c <= 9)) { // Do nothing. } else { - *os << String::Format(", 0x%X", - static_cast<UnsignedChar>(c)).c_str(); + *os << ", 0x" << String::FormatHexInt(static_cast<UnsignedChar>(c)); } *os << ")"; } @@ -255,48 +251,69 @@ void PrintTo(wchar_t wc, ostream* os) { PrintCharAndCodeTo<wchar_t>(wc, os); } -// Prints the given array of characters to the ostream. -// The array starts at *begin, the length is len, it may include '\0' characters -// and may not be null-terminated. -static void PrintCharsAsStringTo(const char* begin, size_t len, ostream* os) { - *os << "\""; +// Prints the given array of characters to the ostream. CharType must be either +// char or wchar_t. +// The array starts at begin, the length is len, it may include '\0' characters +// and may not be NUL-terminated. +template <typename CharType> +GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ +GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ +GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ +static void PrintCharsAsStringTo( + const CharType* begin, size_t len, ostream* os) { + const char* const kQuoteBegin = sizeof(CharType) == 1 ? "\"" : "L\""; + *os << kQuoteBegin; bool is_previous_hex = false; for (size_t index = 0; index < len; ++index) { - const char cur = begin[index]; + const CharType cur = begin[index]; if (is_previous_hex && IsXDigit(cur)) { // Previous character is of '\x..' form and this character can be // interpreted as another hexadecimal digit in its number. Break string to // disambiguate. - *os << "\" \""; + *os << "\" " << kQuoteBegin; } - is_previous_hex = PrintAsNarrowStringLiteralTo(cur, os) == kHexEscape; + is_previous_hex = PrintAsStringLiteralTo(cur, os) == kHexEscape; } *os << "\""; } +// Prints a (const) char/wchar_t array of 'len' elements, starting at address +// 'begin'. CharType must be either char or wchar_t. +template <typename CharType> +GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ +GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ +GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ +static void UniversalPrintCharArray( + const CharType* begin, size_t len, ostream* os) { + // The code + // const char kFoo[] = "foo"; + // generates an array of 4, not 3, elements, with the last one being '\0'. + // + // Therefore when printing a char array, we don't print the last element if + // it's '\0', such that the output matches the string literal as it's + // written in the source code. + if (len > 0 && begin[len - 1] == '\0') { + PrintCharsAsStringTo(begin, len - 1, os); + return; + } + + // If, however, the last element in the array is not '\0', e.g. + // const char kFoo[] = { 'f', 'o', 'o' }; + // we must print the entire array. We also print a message to indicate + // that the array is not NUL-terminated. + PrintCharsAsStringTo(begin, len, os); + *os << " (no terminating NUL)"; +} + // Prints a (const) char array of 'len' elements, starting at address 'begin'. void UniversalPrintArray(const char* begin, size_t len, ostream* os) { - PrintCharsAsStringTo(begin, len, os); + UniversalPrintCharArray(begin, len, os); } -// Prints the given array of wide characters to the ostream. -// The array starts at *begin, the length is len, it may include L'\0' -// characters and may not be null-terminated. -static void PrintWideCharsAsStringTo(const wchar_t* begin, size_t len, - ostream* os) { - *os << "L\""; - bool is_previous_hex = false; - for (size_t index = 0; index < len; ++index) { - const wchar_t cur = begin[index]; - if (is_previous_hex && isascii(cur) && IsXDigit(static_cast<char>(cur))) { - // Previous character is of '\x..' form and this character can be - // interpreted as another hexadecimal digit in its number. Break string to - // disambiguate. - *os << "\" L\""; - } - is_previous_hex = PrintAsWideStringLiteralTo(cur, os) == kHexEscape; - } - *os << "\""; +// Prints a (const) wchar_t array of 'len' elements, starting at address +// 'begin'. +void UniversalPrintArray(const wchar_t* begin, size_t len, ostream* os) { + UniversalPrintCharArray(begin, len, os); } // Prints the given C string to the ostream. @@ -322,7 +339,7 @@ void PrintTo(const wchar_t* s, ostream* os) { *os << "NULL"; } else { *os << ImplicitCast_<const void*>(s) << " pointing to "; - PrintWideCharsAsStringTo(s, wcslen(s), os); + PrintCharsAsStringTo(s, std::wcslen(s), os); } } #endif // wchar_t is native @@ -341,13 +358,13 @@ void PrintStringTo(const ::std::string& s, ostream* os) { // Prints a ::wstring object. #if GTEST_HAS_GLOBAL_WSTRING void PrintWideStringTo(const ::wstring& s, ostream* os) { - PrintWideCharsAsStringTo(s.data(), s.size(), os); + PrintCharsAsStringTo(s.data(), s.size(), os); } #endif // GTEST_HAS_GLOBAL_WSTRING #if GTEST_HAS_STD_WSTRING void PrintWideStringTo(const ::std::wstring& s, ostream* os) { - PrintWideCharsAsStringTo(s.data(), s.size(), os); + PrintCharsAsStringTo(s.data(), s.size(), os); } #endif // GTEST_HAS_STD_WSTRING diff --git a/tests/gtest/src/gtest-test-part.cc b/tests/gtest/src/gtest-test-part.cc index 5ddc67c1c..fb0e35425 100644 --- a/tests/gtest/src/gtest-test-part.cc +++ b/tests/gtest/src/gtest-test-part.cc @@ -35,9 +35,9 @@ // Indicates that this translation unit is part of Google Test's // implementation. It must come before gtest-internal-inl.h is -// included, or there will be a compiler error. This trick is to -// prevent a user from accidentally including gtest-internal-inl.h in -// his code. +// included, or there will be a compiler error. This trick exists to +// prevent the accidental inclusion of gtest-internal-inl.h in the +// user's code. #define GTEST_IMPLEMENTATION_ 1 #include "src/gtest-internal-inl.h" #undef GTEST_IMPLEMENTATION_ @@ -48,10 +48,10 @@ using internal::GetUnitTestImpl; // Gets the summary of the failure message by omitting the stack trace // in it. -internal::String TestPartResult::ExtractSummary(const char* message) { +std::string TestPartResult::ExtractSummary(const char* message) { const char* const stack_trace = strstr(message, internal::kStackTraceMarker); - return stack_trace == NULL ? internal::String(message) : - internal::String(message, stack_trace - message); + return stack_trace == NULL ? message : + std::string(message, stack_trace); } // Prints a TestPartResult object. diff --git a/tests/gtest/src/gtest-typed-test.cc b/tests/gtest/src/gtest-typed-test.cc index a5cc88f92..e11d050a4 100644 --- a/tests/gtest/src/gtest-typed-test.cc +++ b/tests/gtest/src/gtest-typed-test.cc @@ -45,6 +45,15 @@ static const char* SkipSpaces(const char* str) { return str; } +static std::vector<std::string> SplitIntoTestNames(const char* src) { + std::vector<std::string> name_vec; + src = SkipSpaces(src); + for (; src != NULL; src = SkipComma(src)) { + name_vec.push_back(StripTrailingSpaces(GetPrefixUntilComma(src))); + } + return name_vec; +} + // Verifies that registered_tests match the test names in // defined_test_names_; returns registered_tests if successful, or // aborts the program otherwise. @@ -53,15 +62,14 @@ const char* TypedTestCasePState::VerifyRegisteredTestNames( typedef ::std::set<const char*>::const_iterator DefinedTestIter; registered_ = true; - // Skip initial whitespace in registered_tests since some - // preprocessors prefix stringizied literals with whitespace. - registered_tests = SkipSpaces(registered_tests); + std::vector<std::string> name_vec = SplitIntoTestNames(registered_tests); Message errors; - ::std::set<String> tests; - for (const char* names = registered_tests; names != NULL; - names = SkipComma(names)) { - const String name = GetPrefixUntilComma(names); + + std::set<std::string> tests; + for (std::vector<std::string>::const_iterator name_it = name_vec.begin(); + name_it != name_vec.end(); ++name_it) { + const std::string& name = *name_it; if (tests.count(name) != 0) { errors << "Test " << name << " is listed more than once.\n"; continue; @@ -93,7 +101,7 @@ const char* TypedTestCasePState::VerifyRegisteredTestNames( } } - const String& errors_str = errors.GetString(); + const std::string& errors_str = errors.GetString(); if (errors_str != "") { fprintf(stderr, "%s %s", FormatFileLocation(file, line).c_str(), errors_str.c_str()); diff --git a/tests/gtest/src/gtest.cc b/tests/gtest/src/gtest.cc index ee5eb1c49..7fd5f298d 100644 --- a/tests/gtest/src/gtest.cc +++ b/tests/gtest/src/gtest.cc @@ -39,10 +39,15 @@ #include <stdarg.h> #include <stdio.h> #include <stdlib.h> +#include <time.h> #include <wchar.h> #include <wctype.h> #include <algorithm> +#include <iomanip> +#include <limits> +#include <list> +#include <map> #include <ostream> // NOLINT #include <sstream> #include <vector> @@ -77,6 +82,7 @@ #elif GTEST_OS_WINDOWS_MOBILE // We are on Windows CE. # include <windows.h> // NOLINT +# undef min #elif GTEST_OS_WINDOWS // We are on Windows proper. @@ -99,6 +105,7 @@ // cpplint thinks that the header is already included, so we want to // silence it. # include <windows.h> // NOLINT +# undef min #else @@ -121,6 +128,8 @@ #if GTEST_CAN_STREAM_RESULTS_ # include <arpa/inet.h> // NOLINT # include <netdb.h> // NOLINT +# include <sys/socket.h> // NOLINT +# include <sys/types.h> // NOLINT #endif // Indicates that this translation unit is part of Google Test's @@ -179,6 +188,10 @@ bool g_help_flag = false; } // namespace internal +static const char* GetDefaultFilter() { + return kUniversalFilter; +} + GTEST_DEFINE_bool_( also_run_disabled_tests, internal::BoolFromGTestEnv("also_run_disabled_tests", false), @@ -201,11 +214,11 @@ GTEST_DEFINE_string_( "Whether to use colors in the output. Valid values: yes, no, " "and auto. 'auto' means to use colors if the output is " "being sent to a terminal and the TERM environment variable " - "is set to xterm, xterm-color, xterm-256color, linux or cygwin."); + "is set to a terminal type that supports colors."); GTEST_DEFINE_string_( filter, - internal::StringFromGTestEnv("filter", kUniversalFilter), + internal::StringFromGTestEnv("filter", GetDefaultFilter()), "A colon-separated list of glob (not regex) patterns " "for filtering the tests to run, optionally followed by a " "'-' and a : separated list of negative patterns (tests to " @@ -305,7 +318,7 @@ UInt32 Random::Generate(UInt32 range) { // Test. g_init_gtest_count is set to the number of times // InitGoogleTest() has been called. We don't protect this variable // under a mutex as it is only accessed in the main thread. -int g_init_gtest_count = 0; +GTEST_API_ int g_init_gtest_count = 0; static bool GTestIsInitialized() { return g_init_gtest_count != 0; } // Iterates over a vector of TestCases, keeping a running sum of the @@ -360,10 +373,10 @@ void AssertHelper::operator=(const Message& message) const { } // Mutex for linked pointers. -GTEST_DEFINE_STATIC_MUTEX_(g_linked_ptr_mutex); +GTEST_API_ GTEST_DEFINE_STATIC_MUTEX_(g_linked_ptr_mutex); // Application pathname gotten in InitGoogleTest. -String g_executable_path; +std::string g_executable_path; // Returns the current application's name, removing directory path if that // is present. @@ -382,29 +395,29 @@ FilePath GetCurrentExecutableName() { // Functions for processing the gtest_output flag. // Returns the output format, or "" for normal printed output. -String UnitTestOptions::GetOutputFormat() { +std::string UnitTestOptions::GetOutputFormat() { const char* const gtest_output_flag = GTEST_FLAG(output).c_str(); - if (gtest_output_flag == NULL) return String(""); + if (gtest_output_flag == NULL) return std::string(""); const char* const colon = strchr(gtest_output_flag, ':'); return (colon == NULL) ? - String(gtest_output_flag) : - String(gtest_output_flag, colon - gtest_output_flag); + std::string(gtest_output_flag) : + std::string(gtest_output_flag, colon - gtest_output_flag); } // Returns the name of the requested output file, or the default if none // was explicitly specified. -String UnitTestOptions::GetAbsolutePathToOutputFile() { +std::string UnitTestOptions::GetAbsolutePathToOutputFile() { const char* const gtest_output_flag = GTEST_FLAG(output).c_str(); if (gtest_output_flag == NULL) - return String(""); + return ""; const char* const colon = strchr(gtest_output_flag, ':'); if (colon == NULL) - return String(internal::FilePath::ConcatPaths( - internal::FilePath( - UnitTest::GetInstance()->original_working_dir()), - internal::FilePath(kDefaultOutputFile)).ToString() ); + return internal::FilePath::ConcatPaths( + internal::FilePath( + UnitTest::GetInstance()->original_working_dir()), + internal::FilePath(kDefaultOutputFile)).string(); internal::FilePath output_name(colon + 1); if (!output_name.IsAbsolutePath()) @@ -417,12 +430,12 @@ String UnitTestOptions::GetAbsolutePathToOutputFile() { internal::FilePath(colon + 1)); if (!output_name.IsDirectory()) - return output_name.ToString(); + return output_name.string(); internal::FilePath result(internal::FilePath::GenerateUniqueFileName( output_name, internal::GetCurrentExecutableName(), GetOutputFormat().c_str())); - return result.ToString(); + return result.string(); } // Returns true iff the wildcard pattern matches the string. The @@ -447,7 +460,8 @@ bool UnitTestOptions::PatternMatchesString(const char *pattern, } } -bool UnitTestOptions::MatchesFilter(const String& name, const char* filter) { +bool UnitTestOptions::MatchesFilter( + const std::string& name, const char* filter) { const char *cur_pattern = filter; for (;;) { if (PatternMatchesString(cur_pattern, name.c_str())) { @@ -467,28 +481,24 @@ bool UnitTestOptions::MatchesFilter(const String& name, const char* filter) { } } -// TODO(keithray): move String function implementations to gtest-string.cc. - // Returns true iff the user-specified filter matches the test case // name and the test name. -bool UnitTestOptions::FilterMatchesTest(const String &test_case_name, - const String &test_name) { - const String& full_name = String::Format("%s.%s", - test_case_name.c_str(), - test_name.c_str()); +bool UnitTestOptions::FilterMatchesTest(const std::string &test_case_name, + const std::string &test_name) { + const std::string& full_name = test_case_name + "." + test_name.c_str(); // Split --gtest_filter at '-', if there is one, to separate into // positive filter and negative filter portions const char* const p = GTEST_FLAG(filter).c_str(); const char* const dash = strchr(p, '-'); - String positive; - String negative; + std::string positive; + std::string negative; if (dash == NULL) { positive = GTEST_FLAG(filter).c_str(); // Whole string is a positive filter - negative = String(""); + negative = ""; } else { - positive = String(p, dash - p); // Everything up to the dash - negative = String(dash+1); // Everything after the dash + positive = std::string(p, dash); // Everything up to the dash + negative = std::string(dash + 1); // Everything after the dash if (positive.empty()) { // Treat '-test1' as the same as '*-test1' positive = kUniversalFilter; @@ -608,7 +618,7 @@ AssertionResult HasOneFailure(const char* /* results_expr */, const TestPartResultArray& results, TestPartResult::Type type, const string& substr) { - const String expected(type == TestPartResult::kFatalFailure ? + const std::string expected(type == TestPartResult::kFatalFailure ? "1 fatal failure" : "1 non-fatal failure"); Message msg; @@ -731,11 +741,22 @@ int UnitTestImpl::failed_test_count() const { return SumOverTestCaseList(test_cases_, &TestCase::failed_test_count); } +// Gets the number of disabled tests that will be reported in the XML report. +int UnitTestImpl::reportable_disabled_test_count() const { + return SumOverTestCaseList(test_cases_, + &TestCase::reportable_disabled_test_count); +} + // Gets the number of disabled tests. int UnitTestImpl::disabled_test_count() const { return SumOverTestCaseList(test_cases_, &TestCase::disabled_test_count); } +// Gets the number of tests to be printed in the XML report. +int UnitTestImpl::reportable_test_count() const { + return SumOverTestCaseList(test_cases_, &TestCase::reportable_test_count); +} + // Gets the number of all tests. int UnitTestImpl::total_test_count() const { return SumOverTestCaseList(test_cases_, &TestCase::total_test_count); @@ -746,7 +767,7 @@ int UnitTestImpl::test_to_run_count() const { return SumOverTestCaseList(test_cases_, &TestCase::test_to_run_count); } -// Returns the current OS stack trace as a String. +// Returns the current OS stack trace as an std::string. // // The maximum number of stack frames to be included is specified by // the gtest_stack_trace_depth flag. The skip_count parameter @@ -756,9 +777,9 @@ int UnitTestImpl::test_to_run_count() const { // For example, if Foo() calls Bar(), which in turn calls // CurrentOsStackTraceExceptTop(1), Foo() will be included in the // trace but Bar() and CurrentOsStackTraceExceptTop() won't. -String UnitTestImpl::CurrentOsStackTraceExceptTop(int skip_count) { +std::string UnitTestImpl::CurrentOsStackTraceExceptTop(int skip_count) { (void)skip_count; - return String(""); + return ""; } // Returns the current time in milliseconds. @@ -787,21 +808,13 @@ TimeInMillis GetTimeInMillis() { #elif GTEST_OS_WINDOWS && !GTEST_HAS_GETTIMEOFDAY_ __timeb64 now; -# ifdef _MSC_VER - // MSVC 8 deprecates _ftime64(), so we want to suppress warning 4996 // (deprecated function) there. // TODO(kenton@google.com): Use GetTickCount()? Or use // SystemTimeToFileTime() -# pragma warning(push) // Saves the current warning state. -# pragma warning(disable:4996) // Temporarily disables warning 4996. + GTEST_DISABLE_MSC_WARNINGS_PUSH_(4996) _ftime64(&now); -# pragma warning(pop) // Restores the warning state. -# else - - _ftime64(&now); - -# endif // _MSC_VER + GTEST_DISABLE_MSC_WARNINGS_POP_() return static_cast<TimeInMillis>(now.time) * 1000 + now.millitm; #elif GTEST_HAS_GETTIMEOFDAY_ @@ -815,41 +828,7 @@ TimeInMillis GetTimeInMillis() { // Utilities -// class String - -// Returns the input enclosed in double quotes if it's not NULL; -// otherwise returns "(null)". For example, "\"Hello\"" is returned -// for input "Hello". -// -// This is useful for printing a C string in the syntax of a literal. -// -// Known issue: escape sequences are not handled yet. -String String::ShowCStringQuoted(const char* c_str) { - return c_str ? String::Format("\"%s\"", c_str) : String("(null)"); -} - -// Copies at most length characters from str into a newly-allocated -// piece of memory of size length+1. The memory is allocated with new[]. -// A terminating null byte is written to the memory, and a pointer to it -// is returned. If str is NULL, NULL is returned. -static char* CloneString(const char* str, size_t length) { - if (str == NULL) { - return NULL; - } else { - char* const clone = new char[length + 1]; - posix::StrNCpy(clone, str, length); - clone[length] = '\0'; - return clone; - } -} - -// Clones a 0-terminated C string, allocating memory using new. The -// caller is responsible for deleting[] the return value. Returns the -// cloned string, or NULL if the input is NULL. -const char * String::CloneCString(const char* c_str) { - return (c_str == NULL) ? - NULL : CloneString(c_str, strlen(c_str)); -} +// class String. #if GTEST_OS_WINDOWS_MOBILE // Creates a UTF-16 wide string from the given ANSI string, allocating @@ -906,11 +885,6 @@ bool String::CStringEquals(const char * lhs, const char * rhs) { // encoding, and streams the result to the given Message object. static void StreamWideCharsToMessage(const wchar_t* wstr, size_t length, Message* msg) { - // TODO(wan): consider allowing a testing::String object to - // contain '\0'. This will make it behave more like std::string, - // and will allow ToUtf8String() to return the correct encoding - // for '\0' s.t. we can get rid of the conditional here (and in - // several other places). for (size_t i = 0; i != length; ) { // NOLINT if (wstr[i] != L'\0') { *msg << WideStringToUtf8(wstr + i, static_cast<int>(length - i)); @@ -927,6 +901,26 @@ static void StreamWideCharsToMessage(const wchar_t* wstr, size_t length, } // namespace internal +// Constructs an empty Message. +// We allocate the stringstream separately because otherwise each use of +// ASSERT/EXPECT in a procedure adds over 200 bytes to the procedure's +// stack frame leading to huge stack frames in some cases; gcc does not reuse +// the stack space. +Message::Message() : ss_(new ::std::stringstream) { + // By default, we want there to be enough precision when printing + // a double to a Message. + *ss_ << std::setprecision(std::numeric_limits<double>::digits10 + 2); +} + +// These two overloads allow streaming a wide C string to a Message +// using the UTF-8 encoding. +Message& Message::operator <<(const wchar_t* wide_c_str) { + return *this << internal::String::ShowWideCString(wide_c_str); +} +Message& Message::operator <<(wchar_t* wide_c_str) { + return *this << internal::String::ShowWideCString(wide_c_str); +} + #if GTEST_HAS_STD_WSTRING // Converts the given wide string to a narrow string using the UTF-8 // encoding, and streams the result to this Message object. @@ -945,6 +939,12 @@ Message& Message::operator <<(const ::wstring& wstr) { } #endif // GTEST_HAS_GLOBAL_WSTRING +// Gets the text streamed to this object so far as an std::string. +// Each '\0' character in the buffer is replaced with "\\0". +std::string Message::GetString() const { + return internal::StringStreamToString(ss_.get()); +} + // AssertionResult constructors. // Used in EXPECT_TRUE/FALSE(assertion_result). AssertionResult::AssertionResult(const AssertionResult& other) @@ -954,6 +954,13 @@ AssertionResult::AssertionResult(const AssertionResult& other) static_cast< ::std::string*>(NULL)) { } +// Swaps two AssertionResults. +void AssertionResult::swap(AssertionResult& other) { + using std::swap; + swap(success_, other.success_); + swap(message_, other.message_); +} + // Returns the assertion's negation. Used with EXPECT/ASSERT_FALSE. AssertionResult AssertionResult::operator!() const { AssertionResult negation(!success_); @@ -980,6 +987,276 @@ AssertionResult AssertionFailure(const Message& message) { namespace internal { +namespace edit_distance { +std::vector<EditType> CalculateOptimalEdits(const std::vector<size_t>& left, + const std::vector<size_t>& right) { + std::vector<std::vector<double> > costs( + left.size() + 1, std::vector<double>(right.size() + 1)); + std::vector<std::vector<EditType> > best_move( + left.size() + 1, std::vector<EditType>(right.size() + 1)); + + // Populate for empty right. + for (size_t l_i = 0; l_i < costs.size(); ++l_i) { + costs[l_i][0] = static_cast<double>(l_i); + best_move[l_i][0] = kRemove; + } + // Populate for empty left. + for (size_t r_i = 1; r_i < costs[0].size(); ++r_i) { + costs[0][r_i] = static_cast<double>(r_i); + best_move[0][r_i] = kAdd; + } + + for (size_t l_i = 0; l_i < left.size(); ++l_i) { + for (size_t r_i = 0; r_i < right.size(); ++r_i) { + if (left[l_i] == right[r_i]) { + // Found a match. Consume it. + costs[l_i + 1][r_i + 1] = costs[l_i][r_i]; + best_move[l_i + 1][r_i + 1] = kMatch; + continue; + } + + const double add = costs[l_i + 1][r_i]; + const double remove = costs[l_i][r_i + 1]; + const double replace = costs[l_i][r_i]; + if (add < remove && add < replace) { + costs[l_i + 1][r_i + 1] = add + 1; + best_move[l_i + 1][r_i + 1] = kAdd; + } else if (remove < add && remove < replace) { + costs[l_i + 1][r_i + 1] = remove + 1; + best_move[l_i + 1][r_i + 1] = kRemove; + } else { + // We make replace a little more expensive than add/remove to lower + // their priority. + costs[l_i + 1][r_i + 1] = replace + 1.00001; + best_move[l_i + 1][r_i + 1] = kReplace; + } + } + } + + // Reconstruct the best path. We do it in reverse order. + std::vector<EditType> best_path; + for (size_t l_i = left.size(), r_i = right.size(); l_i > 0 || r_i > 0;) { + EditType move = best_move[l_i][r_i]; + best_path.push_back(move); + l_i -= move != kAdd; + r_i -= move != kRemove; + } + std::reverse(best_path.begin(), best_path.end()); + return best_path; +} + +namespace { + +// Helper class to convert string into ids with deduplication. +class InternalStrings { + public: + size_t GetId(const std::string& str) { + IdMap::iterator it = ids_.find(str); + if (it != ids_.end()) return it->second; + size_t id = ids_.size(); + return ids_[str] = id; + } + + private: + typedef std::map<std::string, size_t> IdMap; + IdMap ids_; +}; + +} // namespace + +std::vector<EditType> CalculateOptimalEdits( + const std::vector<std::string>& left, + const std::vector<std::string>& right) { + std::vector<size_t> left_ids, right_ids; + { + InternalStrings intern_table; + for (size_t i = 0; i < left.size(); ++i) { + left_ids.push_back(intern_table.GetId(left[i])); + } + for (size_t i = 0; i < right.size(); ++i) { + right_ids.push_back(intern_table.GetId(right[i])); + } + } + return CalculateOptimalEdits(left_ids, right_ids); +} + +namespace { + +// Helper class that holds the state for one hunk and prints it out to the +// stream. +// It reorders adds/removes when possible to group all removes before all +// adds. It also adds the hunk header before printint into the stream. +class Hunk { + public: + Hunk(size_t left_start, size_t right_start) + : left_start_(left_start), + right_start_(right_start), + adds_(), + removes_(), + common_() {} + + void PushLine(char edit, const char* line) { + switch (edit) { + case ' ': + ++common_; + FlushEdits(); + hunk_.push_back(std::make_pair(' ', line)); + break; + case '-': + ++removes_; + hunk_removes_.push_back(std::make_pair('-', line)); + break; + case '+': + ++adds_; + hunk_adds_.push_back(std::make_pair('+', line)); + break; + } + } + + void PrintTo(std::ostream* os) { + PrintHeader(os); + FlushEdits(); + for (std::list<std::pair<char, const char*> >::const_iterator it = + hunk_.begin(); + it != hunk_.end(); ++it) { + *os << it->first << it->second << "\n"; + } + } + + bool has_edits() const { return adds_ || removes_; } + + private: + void FlushEdits() { + hunk_.splice(hunk_.end(), hunk_removes_); + hunk_.splice(hunk_.end(), hunk_adds_); + } + + // Print a unified diff header for one hunk. + // The format is + // "@@ -<left_start>,<left_length> +<right_start>,<right_length> @@" + // where the left/right parts are ommitted if unnecessary. + void PrintHeader(std::ostream* ss) const { + *ss << "@@ "; + if (removes_) { + *ss << "-" << left_start_ << "," << (removes_ + common_); + } + if (removes_ && adds_) { + *ss << " "; + } + if (adds_) { + *ss << "+" << right_start_ << "," << (adds_ + common_); + } + *ss << " @@\n"; + } + + size_t left_start_, right_start_; + size_t adds_, removes_, common_; + std::list<std::pair<char, const char*> > hunk_, hunk_adds_, hunk_removes_; +}; + +} // namespace + +// Create a list of diff hunks in Unified diff format. +// Each hunk has a header generated by PrintHeader above plus a body with +// lines prefixed with ' ' for no change, '-' for deletion and '+' for +// addition. +// 'context' represents the desired unchanged prefix/suffix around the diff. +// If two hunks are close enough that their contexts overlap, then they are +// joined into one hunk. +std::string CreateUnifiedDiff(const std::vector<std::string>& left, + const std::vector<std::string>& right, + size_t context) { + const std::vector<EditType> edits = CalculateOptimalEdits(left, right); + + size_t l_i = 0, r_i = 0, edit_i = 0; + std::stringstream ss; + while (edit_i < edits.size()) { + // Find first edit. + while (edit_i < edits.size() && edits[edit_i] == kMatch) { + ++l_i; + ++r_i; + ++edit_i; + } + + // Find the first line to include in the hunk. + const size_t prefix_context = std::min(l_i, context); + Hunk hunk(l_i - prefix_context + 1, r_i - prefix_context + 1); + for (size_t i = prefix_context; i > 0; --i) { + hunk.PushLine(' ', left[l_i - i].c_str()); + } + + // Iterate the edits until we found enough suffix for the hunk or the input + // is over. + size_t n_suffix = 0; + for (; edit_i < edits.size(); ++edit_i) { + if (n_suffix >= context) { + // Continue only if the next hunk is very close. + std::vector<EditType>::const_iterator it = edits.begin() + edit_i; + while (it != edits.end() && *it == kMatch) ++it; + if (it == edits.end() || (it - edits.begin()) - edit_i >= context) { + // There is no next edit or it is too far away. + break; + } + } + + EditType edit = edits[edit_i]; + // Reset count when a non match is found. + n_suffix = edit == kMatch ? n_suffix + 1 : 0; + + if (edit == kMatch || edit == kRemove || edit == kReplace) { + hunk.PushLine(edit == kMatch ? ' ' : '-', left[l_i].c_str()); + } + if (edit == kAdd || edit == kReplace) { + hunk.PushLine('+', right[r_i].c_str()); + } + + // Advance indices, depending on edit type. + l_i += edit != kAdd; + r_i += edit != kRemove; + } + + if (!hunk.has_edits()) { + // We are done. We don't want this hunk. + break; + } + + hunk.PrintTo(&ss); + } + return ss.str(); +} + +} // namespace edit_distance + +namespace { + +// The string representation of the values received in EqFailure() are already +// escaped. Split them on escaped '\n' boundaries. Leave all other escaped +// characters the same. +std::vector<std::string> SplitEscapedString(const std::string& str) { + std::vector<std::string> lines; + size_t start = 0, end = str.size(); + if (end > 2 && str[0] == '"' && str[end - 1] == '"') { + ++start; + --end; + } + bool escaped = false; + for (size_t i = start; i + 1 < end; ++i) { + if (escaped) { + escaped = false; + if (str[i] == 'n') { + lines.push_back(str.substr(start, i - start - 1)); + start = i + 1; + } + } else { + escaped = str[i] == '\\'; + } + } + lines.push_back(str.substr(start, end - start)); + return lines; +} + +} // namespace + // Constructs and returns the message for an equality assertion // (e.g. ASSERT_EQ, EXPECT_STREQ, etc) failure. // @@ -997,8 +1274,8 @@ namespace internal { // be inserted into the message. AssertionResult EqFailure(const char* expected_expression, const char* actual_expression, - const String& expected_value, - const String& actual_value, + const std::string& expected_value, + const std::string& actual_value, bool ignoring_case) { Message msg; msg << "Value of: " << actual_expression; @@ -1014,14 +1291,26 @@ AssertionResult EqFailure(const char* expected_expression, msg << "\nWhich is: " << expected_value; } + if (!expected_value.empty() && !actual_value.empty()) { + const std::vector<std::string> expected_lines = + SplitEscapedString(expected_value); + const std::vector<std::string> actual_lines = + SplitEscapedString(actual_value); + if (expected_lines.size() > 1 || actual_lines.size() > 1) { + msg << "\nWith diff:\n" + << edit_distance::CreateUnifiedDiff(expected_lines, actual_lines); + } + } + return AssertionFailure() << msg; } // Constructs a failure message for Boolean assertions such as EXPECT_TRUE. -String GetBoolAssertionFailureMessage(const AssertionResult& assertion_result, - const char* expression_text, - const char* actual_predicate_value, - const char* expected_predicate_value) { +std::string GetBoolAssertionFailureMessage( + const AssertionResult& assertion_result, + const char* expression_text, + const char* actual_predicate_value, + const char* expected_predicate_value) { const char* actual_message = assertion_result.message(); Message msg; msg << "Value of: " << expression_text @@ -1168,8 +1457,8 @@ AssertionResult CmpHelperSTREQ(const char* expected_expression, return EqFailure(expected_expression, actual_expression, - String::ShowCStringQuoted(expected), - String::ShowCStringQuoted(actual), + PrintToString(expected), + PrintToString(actual), false); } @@ -1184,8 +1473,8 @@ AssertionResult CmpHelperSTRCASEEQ(const char* expected_expression, return EqFailure(expected_expression, actual_expression, - String::ShowCStringQuoted(expected), - String::ShowCStringQuoted(actual), + PrintToString(expected), + PrintToString(actual), true); } @@ -1349,7 +1638,7 @@ AssertionResult HRESULTFailureHelper(const char* expr, // want inserts expanded. const DWORD kFlags = FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS; - const DWORD kBufSize = 4096; // String::Format can't exceed this length. + const DWORD kBufSize = 4096; // Gets the system's human readable message string for this HRESULT. char error_text[kBufSize] = { '\0' }; DWORD message_length = ::FormatMessageA(kFlags, @@ -1359,7 +1648,7 @@ AssertionResult HRESULTFailureHelper(const char* expr, error_text, // output buffer kBufSize, // buf size NULL); // no arguments for inserts - // Trims tailing white space (FormatMessage leaves a trailing cr-lf) + // Trims tailing white space (FormatMessage leaves a trailing CR-LF) for (; message_length && IsSpace(error_text[message_length - 1]); --message_length) { error_text[message_length - 1] = '\0'; @@ -1367,10 +1656,10 @@ AssertionResult HRESULTFailureHelper(const char* expr, # endif // GTEST_OS_WINDOWS_MOBILE - const String error_hex(String::Format("0x%08X ", hr)); + const std::string error_hex("0x" + String::FormatHexInt(hr)); return ::testing::AssertionFailure() << "Expected: " << expr << " " << expected << ".\n" - << " Actual: " << error_hex << error_text << "\n"; + << " Actual: " << error_hex << " " << error_text << "\n"; } } // namespace @@ -1427,12 +1716,15 @@ inline UInt32 ChopLowBits(UInt32* bits, int n) { // Converts a Unicode code point to a narrow string in UTF-8 encoding. // code_point parameter is of type UInt32 because wchar_t may not be // wide enough to contain a code point. -// The output buffer str must containt at least 32 characters. -// The function returns the address of the output buffer. // If the code_point is not a valid Unicode code point -// (i.e. outside of Unicode range U+0 to U+10FFFF) it will be output -// as '(Invalid Unicode 0xXXXXXXXX)'. -char* CodePointToUtf8(UInt32 code_point, char* str) { +// (i.e. outside of Unicode range U+0 to U+10FFFF) it will be converted +// to "(Invalid Unicode 0xXXXXXXXX)". +std::string CodePointToUtf8(UInt32 code_point) { + if (code_point > kMaxCodePoint4) { + return "(Invalid Unicode 0x" + String::FormatHexInt(code_point) + ")"; + } + + char str[5]; // Big enough for the largest valid code point. if (code_point <= kMaxCodePoint1) { str[1] = '\0'; str[0] = static_cast<char>(code_point); // 0xxxxxxx @@ -1445,22 +1737,12 @@ char* CodePointToUtf8(UInt32 code_point, char* str) { str[2] = static_cast<char>(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx str[1] = static_cast<char>(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx str[0] = static_cast<char>(0xE0 | code_point); // 1110xxxx - } else if (code_point <= kMaxCodePoint4) { + } else { // code_point <= kMaxCodePoint4 str[4] = '\0'; str[3] = static_cast<char>(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx str[2] = static_cast<char>(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx str[1] = static_cast<char>(0x80 | ChopLowBits(&code_point, 6)); // 10xxxxxx str[0] = static_cast<char>(0xF0 | code_point); // 11110xxx - } else { - // The longest string String::Format can produce when invoked - // with these parameters is 28 character long (not including - // the terminating nul character). We are asking for 32 character - // buffer just in case. This is also enough for strncpy to - // null-terminate the destination string. - posix::StrNCpy( - str, String::Format("(Invalid Unicode 0x%X)", code_point).c_str(), 32); - str[31] = '\0'; // Makes sure no change in the format to strncpy leaves - // the result unterminated. } return str; } @@ -1501,7 +1783,7 @@ inline UInt32 CreateCodePointFromUtf16SurrogatePair(wchar_t first, // as '(Invalid Unicode 0xXXXXXXXX)'. If the string is in UTF16 encoding // and contains invalid UTF-16 surrogate pairs, values in those pairs // will be encoded as individual Unicode characters from Basic Normal Plane. -String WideStringToUtf8(const wchar_t* str, int num_chars) { +std::string WideStringToUtf8(const wchar_t* str, int num_chars) { if (num_chars == -1) num_chars = static_cast<int>(wcslen(str)); @@ -1519,27 +1801,17 @@ String WideStringToUtf8(const wchar_t* str, int num_chars) { unicode_code_point = static_cast<UInt32>(str[i]); } - char buffer[32]; // CodePointToUtf8 requires a buffer this big. - stream << CodePointToUtf8(unicode_code_point, buffer); + stream << CodePointToUtf8(unicode_code_point); } return StringStreamToString(&stream); } -// Converts a wide C string to a String using the UTF-8 encoding. +// Converts a wide C string to an std::string using the UTF-8 encoding. // NULL will be converted to "(null)". -String String::ShowWideCString(const wchar_t * wide_c_str) { - if (wide_c_str == NULL) return String("(null)"); - - return String(internal::WideStringToUtf8(wide_c_str, -1).c_str()); -} - -// Similar to ShowWideCString(), except that this function encloses -// the converted string in double quotes. -String String::ShowWideCStringQuoted(const wchar_t* wide_c_str) { - if (wide_c_str == NULL) return String("(null)"); +std::string String::ShowWideCString(const wchar_t * wide_c_str) { + if (wide_c_str == NULL) return "(null)"; - return String::Format("L\"%s\"", - String::ShowWideCString(wide_c_str).c_str()); + return internal::WideStringToUtf8(wide_c_str, -1); } // Compares two wide C strings. Returns true iff they have the same @@ -1567,8 +1839,8 @@ AssertionResult CmpHelperSTREQ(const char* expected_expression, return EqFailure(expected_expression, actual_expression, - String::ShowWideCStringQuoted(expected), - String::ShowWideCStringQuoted(actual), + PrintToString(expected), + PrintToString(actual), false); } @@ -1583,8 +1855,8 @@ AssertionResult CmpHelperSTRNE(const char* s1_expression, return AssertionFailure() << "Expected: (" << s1_expression << ") != (" << s2_expression << "), actual: " - << String::ShowWideCStringQuoted(s1) - << " vs " << String::ShowWideCStringQuoted(s2); + << PrintToString(s1) + << " vs " << PrintToString(s2); } // Compares two C strings, ignoring case. Returns true iff they have @@ -1635,135 +1907,69 @@ bool String::CaseInsensitiveWideCStringEquals(const wchar_t* lhs, #endif // OS selector } -// Compares this with another String. -// Returns < 0 if this is less than rhs, 0 if this is equal to rhs, or > 0 -// if this is greater than rhs. -int String::Compare(const String & rhs) const { - const char* const lhs_c_str = c_str(); - const char* const rhs_c_str = rhs.c_str(); - - if (lhs_c_str == NULL) { - return rhs_c_str == NULL ? 0 : -1; // NULL < anything except NULL - } else if (rhs_c_str == NULL) { - return 1; - } - - const size_t shorter_str_len = - length() <= rhs.length() ? length() : rhs.length(); - for (size_t i = 0; i != shorter_str_len; i++) { - if (lhs_c_str[i] < rhs_c_str[i]) { - return -1; - } else if (lhs_c_str[i] > rhs_c_str[i]) { - return 1; - } - } - return (length() < rhs.length()) ? -1 : - (length() > rhs.length()) ? 1 : 0; +// Returns true iff str ends with the given suffix, ignoring case. +// Any string is considered to end with an empty suffix. +bool String::EndsWithCaseInsensitive( + const std::string& str, const std::string& suffix) { + const size_t str_len = str.length(); + const size_t suffix_len = suffix.length(); + return (str_len >= suffix_len) && + CaseInsensitiveCStringEquals(str.c_str() + str_len - suffix_len, + suffix.c_str()); } -// Returns true iff this String ends with the given suffix. *Any* -// String is considered to end with a NULL or empty suffix. -bool String::EndsWith(const char* suffix) const { - if (suffix == NULL || CStringEquals(suffix, "")) return true; - - if (c_str() == NULL) return false; - - const size_t this_len = strlen(c_str()); - const size_t suffix_len = strlen(suffix); - return (this_len >= suffix_len) && - CStringEquals(c_str() + this_len - suffix_len, suffix); +// Formats an int value as "%02d". +std::string String::FormatIntWidth2(int value) { + std::stringstream ss; + ss << std::setfill('0') << std::setw(2) << value; + return ss.str(); } -// Returns true iff this String ends with the given suffix, ignoring case. -// Any String is considered to end with a NULL or empty suffix. -bool String::EndsWithCaseInsensitive(const char* suffix) const { - if (suffix == NULL || CStringEquals(suffix, "")) return true; - - if (c_str() == NULL) return false; - - const size_t this_len = strlen(c_str()); - const size_t suffix_len = strlen(suffix); - return (this_len >= suffix_len) && - CaseInsensitiveCStringEquals(c_str() + this_len - suffix_len, suffix); +// Formats an int value as "%X". +std::string String::FormatHexInt(int value) { + std::stringstream ss; + ss << std::hex << std::uppercase << value; + return ss.str(); } -// Formats a list of arguments to a String, using the same format -// spec string as for printf. -// -// We do not use the StringPrintf class as it is not universally -// available. -// -// The result is limited to 4096 characters (including the tailing 0). -// If 4096 characters are not enough to format the input, or if -// there's an error, "<formatting error or buffer exceeded>" is -// returned. -String String::Format(const char * format, ...) { - va_list args; - va_start(args, format); - - char buffer[4096]; - const int kBufferSize = sizeof(buffer)/sizeof(buffer[0]); - - // MSVC 8 deprecates vsnprintf(), so we want to suppress warning - // 4996 (deprecated function) there. -#ifdef _MSC_VER // We are using MSVC. -# pragma warning(push) // Saves the current warning state. -# pragma warning(disable:4996) // Temporarily disables warning 4996. - - const int size = vsnprintf(buffer, kBufferSize, format, args); - -# pragma warning(pop) // Restores the warning state. -#else // We are not using MSVC. - const int size = vsnprintf(buffer, kBufferSize, format, args); -#endif // _MSC_VER - va_end(args); - - // vsnprintf()'s behavior is not portable. When the buffer is not - // big enough, it returns a negative value in MSVC, and returns the - // needed buffer size on Linux. When there is an output error, it - // always returns a negative value. For simplicity, we lump the two - // error cases together. - if (size < 0 || size >= kBufferSize) { - return String("<formatting error or buffer exceeded>"); - } else { - return String(buffer, size); - } +// Formats a byte as "%02X". +std::string String::FormatByte(unsigned char value) { + std::stringstream ss; + ss << std::setfill('0') << std::setw(2) << std::hex << std::uppercase + << static_cast<unsigned int>(value); + return ss.str(); } -// Converts the buffer in a stringstream to a String, converting NUL +// Converts the buffer in a stringstream to an std::string, converting NUL // bytes to "\\0" along the way. -String StringStreamToString(::std::stringstream* ss) { +std::string StringStreamToString(::std::stringstream* ss) { const ::std::string& str = ss->str(); const char* const start = str.c_str(); const char* const end = start + str.length(); - // We need to use a helper stringstream to do this transformation - // because String doesn't support push_back(). - ::std::stringstream helper; + std::string result; + result.reserve(2 * (end - start)); for (const char* ch = start; ch != end; ++ch) { if (*ch == '\0') { - helper << "\\0"; // Replaces NUL with "\\0"; + result += "\\0"; // Replaces NUL with "\\0"; } else { - helper.put(*ch); + result += *ch; } } - return String(helper.str().c_str()); + return result; } // Appends the user-supplied message to the Google-Test-generated message. -String AppendUserMessage(const String& gtest_msg, - const Message& user_msg) { +std::string AppendUserMessage(const std::string& gtest_msg, + const Message& user_msg) { // Appends the user message if it's non-empty. - const String user_msg_string = user_msg.GetString(); + const std::string user_msg_string = user_msg.GetString(); if (user_msg_string.empty()) { return gtest_msg; } - Message msg; - msg << gtest_msg << "\n" << user_msg_string; - - return msg.GetString(); + return gtest_msg + "\n" + user_msg_string; } } // namespace internal @@ -1811,8 +2017,9 @@ void TestResult::AddTestPartResult(const TestPartResult& test_part_result) { // Adds a test property to the list. If a property with the same key as the // supplied property is already represented, the value of this test_property // replaces the old value for that key. -void TestResult::RecordProperty(const TestProperty& test_property) { - if (!ValidateTestProperty(test_property)) { +void TestResult::RecordProperty(const std::string& xml_element, + const TestProperty& test_property) { + if (!ValidateTestProperty(xml_element, test_property)) { return; } internal::MutexLock lock(&test_properites_mutex_); @@ -1826,21 +2033,94 @@ void TestResult::RecordProperty(const TestProperty& test_property) { property_with_matching_key->SetValue(test_property.value()); } -// Adds a failure if the key is a reserved attribute of Google Test -// testcase tags. Returns true if the property is valid. -bool TestResult::ValidateTestProperty(const TestProperty& test_property) { - internal::String key(test_property.key()); - if (key == "name" || key == "status" || key == "time" || key == "classname") { - ADD_FAILURE() - << "Reserved key used in RecordProperty(): " - << key - << " ('name', 'status', 'time', and 'classname' are reserved by " - << GTEST_NAME_ << ")"; +// The list of reserved attributes used in the <testsuites> element of XML +// output. +static const char* const kReservedTestSuitesAttributes[] = { + "disabled", + "errors", + "failures", + "name", + "random_seed", + "tests", + "time", + "timestamp" +}; + +// The list of reserved attributes used in the <testsuite> element of XML +// output. +static const char* const kReservedTestSuiteAttributes[] = { + "disabled", + "errors", + "failures", + "name", + "tests", + "time" +}; + +// The list of reserved attributes used in the <testcase> element of XML output. +static const char* const kReservedTestCaseAttributes[] = { + "classname", + "name", + "status", + "time", + "type_param", + "value_param" +}; + +template <int kSize> +std::vector<std::string> ArrayAsVector(const char* const (&array)[kSize]) { + return std::vector<std::string>(array, array + kSize); +} + +static std::vector<std::string> GetReservedAttributesForElement( + const std::string& xml_element) { + if (xml_element == "testsuites") { + return ArrayAsVector(kReservedTestSuitesAttributes); + } else if (xml_element == "testsuite") { + return ArrayAsVector(kReservedTestSuiteAttributes); + } else if (xml_element == "testcase") { + return ArrayAsVector(kReservedTestCaseAttributes); + } else { + GTEST_CHECK_(false) << "Unrecognized xml_element provided: " << xml_element; + } + // This code is unreachable but some compilers may not realizes that. + return std::vector<std::string>(); +} + +static std::string FormatWordList(const std::vector<std::string>& words) { + Message word_list; + for (size_t i = 0; i < words.size(); ++i) { + if (i > 0 && words.size() > 2) { + word_list << ", "; + } + if (i == words.size() - 1) { + word_list << "and "; + } + word_list << "'" << words[i] << "'"; + } + return word_list.GetString(); +} + +bool ValidateTestPropertyName(const std::string& property_name, + const std::vector<std::string>& reserved_names) { + if (std::find(reserved_names.begin(), reserved_names.end(), property_name) != + reserved_names.end()) { + ADD_FAILURE() << "Reserved key used in RecordProperty(): " << property_name + << " (" << FormatWordList(reserved_names) + << " are reserved by " << GTEST_NAME_ << ")"; return false; } return true; } +// Adds a failure if the key is a reserved attribute of the element named +// xml_element. Returns true if the property is valid. +bool TestResult::ValidateTestProperty(const std::string& xml_element, + const TestProperty& test_property) { + return ValidateTestPropertyName(test_property.key(), + GetReservedAttributesForElement(xml_element)); +} + // Clears the object. void TestResult::Clear() { test_part_results_.clear(); @@ -1916,12 +2196,12 @@ void Test::TearDown() { } // Allows user supplied key value pairs to be recorded for later output. -void Test::RecordProperty(const char* key, const char* value) { - UnitTest::GetInstance()->RecordPropertyForCurrentTest(key, value); +void Test::RecordProperty(const std::string& key, const std::string& value) { + UnitTest::GetInstance()->RecordProperty(key, value); } // Allows user supplied key value pairs to be recorded for later output. -void Test::RecordProperty(const char* key, int value) { +void Test::RecordProperty(const std::string& key, int value) { Message value_message; value_message << value; RecordProperty(key, value_message.GetString().c_str()); @@ -1930,7 +2210,7 @@ void Test::RecordProperty(const char* key, int value) { namespace internal { void ReportFailureInUnknownLocation(TestPartResult::Type result_type, - const String& message) { + const std::string& message) { // This function is a friend of UnitTest and as such has access to // AddTestPartResult. UnitTest::GetInstance()->AddTestPartResult( @@ -1938,7 +2218,7 @@ void ReportFailureInUnknownLocation(TestPartResult::Type result_type, NULL, // No info about the source file where the exception occurred. -1, // We have no info on which line caused the exception. message, - String()); // No stack trace, either. + ""); // No stack trace, either. } } // namespace internal @@ -1969,8 +2249,8 @@ bool Test::HasSameFixtureClass() { const bool this_is_TEST = this_fixture_id == internal::GetTestTypeId(); if (first_is_TEST || this_is_TEST) { - // The user mixed TEST and TEST_F in this test case - we'll tell - // him/her how to fix it. + // Both TEST and TEST_F appear in same test case, which is incorrect. + // Tell the user how to fix this. // Gets the name of the TEST and the name of the TEST_F. Note // that first_is_TEST and this_is_TEST cannot both be true, as @@ -1990,8 +2270,8 @@ bool Test::HasSameFixtureClass() { << "want to change the TEST to TEST_F or move it to another test\n" << "case."; } else { - // The user defined two fixture classes with the same name in - // two namespaces - we'll tell him/her how to fix it. + // Two fixture classes with the same name appear in two different + // namespaces, which is not allowed. Tell the user how to fix this. ADD_FAILURE() << "All tests in the same test case must use the same test fixture\n" << "class. However, in test case " @@ -2015,22 +2295,24 @@ bool Test::HasSameFixtureClass() { // function returns its result via an output parameter pointer because VC++ // prohibits creation of objects with destructors on stack in functions // using __try (see error C2712). -static internal::String* FormatSehExceptionMessage(DWORD exception_code, - const char* location) { +static std::string* FormatSehExceptionMessage(DWORD exception_code, + const char* location) { Message message; message << "SEH exception with code 0x" << std::setbase(16) << exception_code << std::setbase(10) << " thrown in " << location << "."; - return new internal::String(message.GetString()); + return new std::string(message.GetString()); } #endif // GTEST_HAS_SEH +namespace internal { + #if GTEST_HAS_EXCEPTIONS // Adds an "exception thrown" fatal failure to the current test. -static internal::String FormatCxxExceptionMessage(const char* description, - const char* location) { +static std::string FormatCxxExceptionMessage(const char* description, + const char* location) { Message message; if (description != NULL) { message << "C++ exception with description \"" << description << "\""; @@ -2042,23 +2324,15 @@ static internal::String FormatCxxExceptionMessage(const char* description, return message.GetString(); } -static internal::String PrintTestPartResultToString( +static std::string PrintTestPartResultToString( const TestPartResult& test_part_result); -// A failed Google Test assertion will throw an exception of this type when -// GTEST_FLAG(throw_on_failure) is true (if exceptions are enabled). We -// derive it from std::runtime_error, which is for errors presumably -// detectable only at run time. Since std::runtime_error inherits from -// std::exception, many testing frameworks know how to extract and print the -// message inside it. -class GoogleTestFailureException : public ::std::runtime_error { - public: - explicit GoogleTestFailureException(const TestPartResult& failure) - : ::std::runtime_error(PrintTestPartResultToString(failure).c_str()) {} -}; +GoogleTestFailureException::GoogleTestFailureException( + const TestPartResult& failure) + : ::std::runtime_error(PrintTestPartResultToString(failure).c_str()) {} + #endif // GTEST_HAS_EXCEPTIONS -namespace internal { // We put these helper functions in the internal namespace as IBM's xlC // compiler rejects the code if they were declared static. @@ -2078,7 +2352,7 @@ Result HandleSehExceptionsInMethodIfSupported( // We create the exception message on the heap because VC++ prohibits // creation of objects with destructors on stack in functions using __try // (see error C2712). - internal::String* exception_message = FormatSehExceptionMessage( + std::string* exception_message = FormatSehExceptionMessage( GetExceptionCode(), location); internal::ReportFailureInUnknownLocation(TestPartResult::kFatalFailure, *exception_message); @@ -2124,9 +2398,10 @@ Result HandleExceptionsInMethodIfSupported( #if GTEST_HAS_EXCEPTIONS try { return HandleSehExceptionsInMethodIfSupported(object, method, location); - } catch (const GoogleTestFailureException&) { // NOLINT - // This exception doesn't originate in code under test. It makes no - // sense to report it as a test failure. + } catch (const internal::GoogleTestFailureException&) { // NOLINT + // This exception type can only be thrown by a failed Google + // Test assertion with the intention of letting another testing + // framework catch it. Therefore we just re-throw it. throw; } catch (const std::exception& e) { // NOLINT internal::ReportFailureInUnknownLocation( @@ -2185,10 +2460,8 @@ bool Test::HasNonfatalFailure() { // Constructs a TestInfo object. It assumes ownership of the test factory // object. -// TODO(vladl@google.com): Make a_test_case_name and a_name const string&'s -// to signify they cannot be NULLs. -TestInfo::TestInfo(const char* a_test_case_name, - const char* a_name, +TestInfo::TestInfo(const std::string& a_test_case_name, + const std::string& a_name, const char* a_type_param, const char* a_value_param, internal::TypeId fixture_class_id, @@ -2227,7 +2500,8 @@ namespace internal { // The newly created TestInfo instance will assume // ownership of the factory object. TestInfo* MakeAndRegisterTestInfo( - const char* test_case_name, const char* name, + const char* test_case_name, + const char* name, const char* type_param, const char* value_param, TypeId fixture_class_id, @@ -2282,11 +2556,11 @@ class TestNameIs { // Returns true iff the test name of test_info matches name_. bool operator()(const TestInfo * test_info) const { - return test_info && internal::String(test_info->name()).Compare(name_) == 0; + return test_info && test_info->name() == name_; } private: - internal::String name_; + std::string name_; }; } // namespace @@ -2365,10 +2639,21 @@ int TestCase::failed_test_count() const { return CountIf(test_info_list_, TestFailed); } +// Gets the number of disabled tests that will be reported in the XML report. +int TestCase::reportable_disabled_test_count() const { + return CountIf(test_info_list_, TestReportableDisabled); +} + +// Gets the number of disabled tests in this test case. int TestCase::disabled_test_count() const { return CountIf(test_info_list_, TestDisabled); } +// Gets the number of tests to be printed in the XML report. +int TestCase::reportable_test_count() const { + return CountIf(test_info_list_, TestReportable); +} + // Get the number of tests in this test case that should run. int TestCase::test_to_run_count() const { return CountIf(test_info_list_, ShouldRunTest); @@ -2456,6 +2741,7 @@ void TestCase::Run() { // Clears the results of all tests in this test case. void TestCase::ClearResult() { + ad_hoc_test_result_.Clear(); ForEach(test_info_list_, TestInfo::ClearTestResult); } @@ -2476,20 +2762,20 @@ void TestCase::UnshuffleTests() { // // FormatCountableNoun(1, "formula", "formuli") returns "1 formula". // FormatCountableNoun(5, "book", "books") returns "5 books". -static internal::String FormatCountableNoun(int count, - const char * singular_form, - const char * plural_form) { - return internal::String::Format("%d %s", count, - count == 1 ? singular_form : plural_form); +static std::string FormatCountableNoun(int count, + const char * singular_form, + const char * plural_form) { + return internal::StreamableToString(count) + " " + + (count == 1 ? singular_form : plural_form); } // Formats the count of tests. -static internal::String FormatTestCount(int test_count) { +static std::string FormatTestCount(int test_count) { return FormatCountableNoun(test_count, "test", "tests"); } // Formats the count of test cases. -static internal::String FormatTestCaseCount(int test_case_count) { +static std::string FormatTestCaseCount(int test_case_count) { return FormatCountableNoun(test_case_count, "test case", "test cases"); } @@ -2514,8 +2800,10 @@ static const char * TestPartResultTypeToString(TestPartResult::Type type) { } } -// Prints a TestPartResult to a String. -static internal::String PrintTestPartResultToString( +namespace internal { + +// Prints a TestPartResult to an std::string. +static std::string PrintTestPartResultToString( const TestPartResult& test_part_result) { return (Message() << internal::FormatFileLocation(test_part_result.file_name(), @@ -2526,7 +2814,7 @@ static internal::String PrintTestPartResultToString( // Prints a TestPartResult. static void PrintTestPartResult(const TestPartResult& test_part_result) { - const internal::String& result = + const std::string& result = PrintTestPartResultToString(test_part_result); printf("%s\n", result.c_str()); fflush(stdout); @@ -2545,8 +2833,6 @@ static void PrintTestPartResult(const TestPartResult& test_part_result) { // class PrettyUnitTestResultPrinter -namespace internal { - enum GTestColor { COLOR_DEFAULT, COLOR_RED, @@ -2554,7 +2840,8 @@ enum GTestColor { COLOR_YELLOW }; -#if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE +#if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE && \ + !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT // Returns the character attribute for the given color. WORD GetColorAttribute(GTestColor color) { @@ -2598,6 +2885,7 @@ bool ShouldUseColor(bool stdout_is_tty) { String::CStringEquals(term, "xterm-color") || String::CStringEquals(term, "xterm-256color") || String::CStringEquals(term, "screen") || + String::CStringEquals(term, "screen-256color") || String::CStringEquals(term, "linux") || String::CStringEquals(term, "cygwin"); return stdout_is_tty && term_supports_color; @@ -2621,8 +2909,9 @@ void ColoredPrintf(GTestColor color, const char* fmt, ...) { va_list args; va_start(args, fmt); -#if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_SYMBIAN || GTEST_OS_ZOS - const bool use_color = false; +#if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_SYMBIAN || GTEST_OS_ZOS || \ + GTEST_OS_IOS || GTEST_OS_WINDOWS_PHONE || GTEST_OS_WINDOWS_RT + const bool use_color = AlwaysFalse(); #else static const bool in_color_mode = ShouldUseColor(posix::IsATTY(posix::FileNo(stdout)) != 0); @@ -2636,7 +2925,8 @@ void ColoredPrintf(GTestColor color, const char* fmt, ...) { return; } -#if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE +#if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MOBILE && \ + !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT const HANDLE stdout_handle = GetStdHandle(STD_OUTPUT_HANDLE); // Gets the current text color. @@ -2663,6 +2953,11 @@ void ColoredPrintf(GTestColor color, const char* fmt, ...) { va_end(args); } +// Text printed in Google Test's text output and --gunit_list_tests +// output to label the type parameter and value parameter for a test. +static const char kTypeParamLabel[] = "TypeParam"; +static const char kValueParamLabel[] = "GetParam()"; + void PrintFullTestCommentIfPresent(const TestInfo& test_info) { const char* const type_param = test_info.type_param(); const char* const value_param = test_info.value_param(); @@ -2670,12 +2965,12 @@ void PrintFullTestCommentIfPresent(const TestInfo& test_info) { if (type_param != NULL || value_param != NULL) { printf(", where "); if (type_param != NULL) { - printf("TypeParam = %s", type_param); + printf("%s = %s", kTypeParamLabel, type_param); if (value_param != NULL) printf(" and "); } if (value_param != NULL) { - printf("GetParam() = %s", value_param); + printf("%s = %s", kValueParamLabel, value_param); } } } @@ -2707,8 +3002,6 @@ class PrettyUnitTestResultPrinter : public TestEventListener { private: static void PrintFailedTests(const UnitTest& unit_test); - - internal::String test_case_name_; }; // Fired before each iteration of tests starts. @@ -2721,7 +3014,7 @@ void PrettyUnitTestResultPrinter::OnTestIterationStart( // Prints the filter if it's not *. This reminds the user that some // tests may be skipped. - if (!internal::String::CStringEquals(filter, kUniversalFilter)) { + if (!String::CStringEquals(filter, kUniversalFilter)) { ColoredPrintf(COLOR_YELLOW, "Note: %s filter = %s\n", GTEST_NAME_, filter); } @@ -2755,22 +3048,21 @@ void PrettyUnitTestResultPrinter::OnEnvironmentsSetUpStart( } void PrettyUnitTestResultPrinter::OnTestCaseStart(const TestCase& test_case) { - test_case_name_ = test_case.name(); - const internal::String counts = + const std::string counts = FormatCountableNoun(test_case.test_to_run_count(), "test", "tests"); ColoredPrintf(COLOR_GREEN, "[----------] "); - printf("%s from %s", counts.c_str(), test_case_name_.c_str()); + printf("%s from %s", counts.c_str(), test_case.name()); if (test_case.type_param() == NULL) { printf("\n"); } else { - printf(", where TypeParam = %s\n", test_case.type_param()); + printf(", where %s = %s\n", kTypeParamLabel, test_case.type_param()); } fflush(stdout); } void PrettyUnitTestResultPrinter::OnTestStart(const TestInfo& test_info) { ColoredPrintf(COLOR_GREEN, "[ RUN ] "); - PrintTestName(test_case_name_.c_str(), test_info.name()); + PrintTestName(test_info.test_case_name(), test_info.name()); printf("\n"); fflush(stdout); } @@ -2793,7 +3085,7 @@ void PrettyUnitTestResultPrinter::OnTestEnd(const TestInfo& test_info) { } else { ColoredPrintf(COLOR_RED, "[ FAILED ] "); } - PrintTestName(test_case_name_.c_str(), test_info.name()); + PrintTestName(test_info.test_case_name(), test_info.name()); if (test_info.result()->Failed()) PrintFullTestCommentIfPresent(test_info); @@ -2809,12 +3101,11 @@ void PrettyUnitTestResultPrinter::OnTestEnd(const TestInfo& test_info) { void PrettyUnitTestResultPrinter::OnTestCaseEnd(const TestCase& test_case) { if (!GTEST_FLAG(print_time)) return; - test_case_name_ = test_case.name(); - const internal::String counts = + const std::string counts = FormatCountableNoun(test_case.test_to_run_count(), "test", "tests"); ColoredPrintf(COLOR_GREEN, "[----------] "); printf("%s from %s (%s ms total)\n\n", - counts.c_str(), test_case_name_.c_str(), + counts.c_str(), test_case.name(), internal::StreamableToString(test_case.elapsed_time()).c_str()); fflush(stdout); } @@ -2875,7 +3166,7 @@ void PrettyUnitTestResultPrinter::OnTestIterationEnd(const UnitTest& unit_test, num_failures == 1 ? "TEST" : "TESTS"); } - int num_disabled = unit_test.disabled_test_count(); + int num_disabled = unit_test.reportable_disabled_test_count(); if (num_disabled && !GTEST_FLAG(also_run_disabled_tests)) { if (!num_failures) { printf("\n"); // Add a spacer if no FAILURE banner is displayed. @@ -3029,18 +3320,27 @@ class XmlUnitTestResultPrinter : public EmptyTestEventListener { // is_attribute is true, the text is meant to appear as an attribute // value, and normalizable whitespace is preserved by replacing it // with character references. - static String EscapeXml(const char* str, bool is_attribute); + static std::string EscapeXml(const std::string& str, bool is_attribute); // Returns the given string with all characters invalid in XML removed. - static string RemoveInvalidXmlCharacters(const string& str); + static std::string RemoveInvalidXmlCharacters(const std::string& str); // Convenience wrapper around EscapeXml when str is an attribute value. - static String EscapeXmlAttribute(const char* str) { + static std::string EscapeXmlAttribute(const std::string& str) { return EscapeXml(str, true); } // Convenience wrapper around EscapeXml when str is not an attribute value. - static String EscapeXmlText(const char* str) { return EscapeXml(str, false); } + static std::string EscapeXmlText(const char* str) { + return EscapeXml(str, false); + } + + // Verifies that the given attribute belongs to the given element and + // streams the attribute as XML. + static void OutputXmlAttribute(std::ostream* stream, + const std::string& element_name, + const std::string& name, + const std::string& value); // Streams an XML CDATA section, escaping invalid CDATA sequences as needed. static void OutputXmlCDataSection(::std::ostream* stream, const char* data); @@ -3051,19 +3351,21 @@ class XmlUnitTestResultPrinter : public EmptyTestEventListener { const TestInfo& test_info); // Prints an XML representation of a TestCase object - static void PrintXmlTestCase(FILE* out, const TestCase& test_case); + static void PrintXmlTestCase(::std::ostream* stream, + const TestCase& test_case); // Prints an XML summary of unit_test to output stream out. - static void PrintXmlUnitTest(FILE* out, const UnitTest& unit_test); + static void PrintXmlUnitTest(::std::ostream* stream, + const UnitTest& unit_test); // Produces a string representing the test properties in a result as space // delimited XML attributes based on the property key="value" pairs. - // When the String is not empty, it includes a space at the beginning, + // When the std::string is not empty, it includes a space at the beginning, // to delimit this attribute from prior attributes. - static String TestPropertiesAsXmlAttributes(const TestResult& result); + static std::string TestPropertiesAsXmlAttributes(const TestResult& result); // The output file. - const String output_file_; + const std::string output_file_; GTEST_DISALLOW_COPY_AND_ASSIGN_(XmlUnitTestResultPrinter); }; @@ -3105,7 +3407,9 @@ void XmlUnitTestResultPrinter::OnTestIterationEnd(const UnitTest& unit_test, fflush(stderr); exit(EXIT_FAILURE); } - PrintXmlUnitTest(xmlout, unit_test); + std::stringstream stream; + PrintXmlUnitTest(&stream, unit_test); + fprintf(xmlout, "%s", StringStreamToString(&stream).c_str()); fclose(xmlout); } @@ -3121,42 +3425,43 @@ void XmlUnitTestResultPrinter::OnTestIterationEnd(const UnitTest& unit_test, // most invalid characters can be retained using character references. // TODO(wan): It might be nice to have a minimally invasive, human-readable // escaping scheme for invalid characters, rather than dropping them. -String XmlUnitTestResultPrinter::EscapeXml(const char* str, bool is_attribute) { +std::string XmlUnitTestResultPrinter::EscapeXml( + const std::string& str, bool is_attribute) { Message m; - if (str != NULL) { - for (const char* src = str; *src; ++src) { - switch (*src) { - case '<': - m << "<"; - break; - case '>': - m << ">"; - break; - case '&': - m << "&"; - break; - case '\'': - if (is_attribute) - m << "'"; - else - m << '\''; - break; - case '"': - if (is_attribute) - m << """; + for (size_t i = 0; i < str.size(); ++i) { + const char ch = str[i]; + switch (ch) { + case '<': + m << "<"; + break; + case '>': + m << ">"; + break; + case '&': + m << "&"; + break; + case '\'': + if (is_attribute) + m << "'"; + else + m << '\''; + break; + case '"': + if (is_attribute) + m << """; + else + m << '"'; + break; + default: + if (IsValidXmlCharacter(ch)) { + if (is_attribute && IsNormalizableWhitespace(ch)) + m << "&#x" << String::FormatByte(static_cast<unsigned char>(ch)) + << ";"; else - m << '"'; - break; - default: - if (IsValidXmlCharacter(*src)) { - if (is_attribute && IsNormalizableWhitespace(*src)) - m << String::Format("&#x%02X;", unsigned(*src)); - else - m << *src; - } - break; - } + m << ch; + } + break; } } @@ -3166,10 +3471,11 @@ String XmlUnitTestResultPrinter::EscapeXml(const char* str, bool is_attribute) { // Returns the given string with all characters invalid in XML removed. // Currently invalid characters are dropped from the string. An // alternative is to replace them with certain characters such as . or ?. -string XmlUnitTestResultPrinter::RemoveInvalidXmlCharacters(const string& str) { - string output; +std::string XmlUnitTestResultPrinter::RemoveInvalidXmlCharacters( + const std::string& str) { + std::string output; output.reserve(str.size()); - for (string::const_iterator it = str.begin(); it != str.end(); ++it) + for (std::string::const_iterator it = str.begin(); it != str.end(); ++it) if (IsValidXmlCharacter(*it)) output.push_back(*it); @@ -3199,6 +3505,37 @@ std::string FormatTimeInMillisAsSeconds(TimeInMillis ms) { return ss.str(); } +static bool PortableLocaltime(time_t seconds, struct tm* out) { +#if defined(_MSC_VER) + return localtime_s(out, &seconds) == 0; +#elif defined(__MINGW32__) || defined(__MINGW64__) + // MINGW <time.h> provides neither localtime_r nor localtime_s, but uses + // Windows' localtime(), which has a thread-local tm buffer. + struct tm* tm_ptr = localtime(&seconds); // NOLINT + if (tm_ptr == NULL) + return false; + *out = *tm_ptr; + return true; +#else + return localtime_r(&seconds, out) != NULL; +#endif +} + +// Converts the given epoch time in milliseconds to a date string in the ISO +// 8601 format, without the timezone information. +std::string FormatEpochTimeInMillisAsIso8601(TimeInMillis ms) { + struct tm time_struct; + if (!PortableLocaltime(static_cast<time_t>(ms / 1000), &time_struct)) + return ""; + // YYYY-MM-DDThh:mm:ss + return StreamableToString(time_struct.tm_year + 1900) + "-" + + String::FormatIntWidth2(time_struct.tm_mon + 1) + "-" + + String::FormatIntWidth2(time_struct.tm_mday) + "T" + + String::FormatIntWidth2(time_struct.tm_hour) + ":" + + String::FormatIntWidth2(time_struct.tm_min) + ":" + + String::FormatIntWidth2(time_struct.tm_sec); +} + // Streams an XML CDATA section, escaping invalid CDATA sequences as needed. void XmlUnitTestResultPrinter::OutputXmlCDataSection(::std::ostream* stream, const char* data) { @@ -3219,48 +3556,63 @@ void XmlUnitTestResultPrinter::OutputXmlCDataSection(::std::ostream* stream, *stream << "]]>"; } +void XmlUnitTestResultPrinter::OutputXmlAttribute( + std::ostream* stream, + const std::string& element_name, + const std::string& name, + const std::string& value) { + const std::vector<std::string>& allowed_names = + GetReservedAttributesForElement(element_name); + + GTEST_CHECK_(std::find(allowed_names.begin(), allowed_names.end(), name) != + allowed_names.end()) + << "Attribute " << name << " is not allowed for element <" << element_name + << ">."; + + *stream << " " << name << "=\"" << EscapeXmlAttribute(value) << "\""; +} + // Prints an XML representation of a TestInfo object. // TODO(wan): There is also value in printing properties with the plain printer. void XmlUnitTestResultPrinter::OutputXmlTestInfo(::std::ostream* stream, const char* test_case_name, const TestInfo& test_info) { - if (test_info.filtered_out ()) - return; - const TestResult& result = *test_info.result(); - *stream << " <testcase name=\"" - << EscapeXmlAttribute(test_info.name()).c_str() << "\""; + const std::string kTestcase = "testcase"; + + *stream << " <testcase"; + OutputXmlAttribute(stream, kTestcase, "name", test_info.name()); if (test_info.value_param() != NULL) { - *stream << " value_param=\"" << EscapeXmlAttribute(test_info.value_param()) - << "\""; + OutputXmlAttribute(stream, kTestcase, "value_param", + test_info.value_param()); } if (test_info.type_param() != NULL) { - *stream << " type_param=\"" << EscapeXmlAttribute(test_info.type_param()) - << "\""; + OutputXmlAttribute(stream, kTestcase, "type_param", test_info.type_param()); } - *stream << " status=\"" - << (test_info.should_run() ? "run" : "notrun") - << "\" time=\"" - << FormatTimeInMillisAsSeconds(result.elapsed_time()) - << "\" classname=\"" << EscapeXmlAttribute(test_case_name).c_str() - << "\"" << TestPropertiesAsXmlAttributes(result).c_str(); + OutputXmlAttribute(stream, kTestcase, "status", + test_info.should_run() ? "run" : "notrun"); + OutputXmlAttribute(stream, kTestcase, "time", + FormatTimeInMillisAsSeconds(result.elapsed_time())); + OutputXmlAttribute(stream, kTestcase, "classname", test_case_name); + *stream << TestPropertiesAsXmlAttributes(result); int failures = 0; for (int i = 0; i < result.total_part_count(); ++i) { const TestPartResult& part = result.GetTestPartResult(i); if (part.failed()) { - if (++failures == 1) + if (++failures == 1) { *stream << ">\n"; - *stream << " <failure message=\"" - << EscapeXmlAttribute(part.summary()).c_str() - << "\" type=\"\">"; + } const string location = internal::FormatCompilerIndependentFileLocation( part.file_name(), part.line_number()); - const string message = location + "\n" + part.message(); - OutputXmlCDataSection(stream, - RemoveInvalidXmlCharacters(message).c_str()); + const string summary = location + "\n" + part.summary(); + *stream << " <failure message=\"" + << EscapeXmlAttribute(summary.c_str()) + << "\" type=\"\">"; + const string detail = location + "\n" + part.message(); + OutputXmlCDataSection(stream, RemoveInvalidXmlCharacters(detail).c_str()); *stream << "</failure>\n"; } } @@ -3272,52 +3624,73 @@ void XmlUnitTestResultPrinter::OutputXmlTestInfo(::std::ostream* stream, } // Prints an XML representation of a TestCase object -void XmlUnitTestResultPrinter::PrintXmlTestCase(FILE* out, +void XmlUnitTestResultPrinter::PrintXmlTestCase(std::ostream* stream, const TestCase& test_case) { - if (test_case.should_skip_report ()) - return; + const std::string kTestsuite = "testsuite"; + *stream << " <" << kTestsuite; + OutputXmlAttribute(stream, kTestsuite, "name", test_case.name()); + OutputXmlAttribute(stream, kTestsuite, "tests", + StreamableToString(test_case.reportable_test_count())); + OutputXmlAttribute(stream, kTestsuite, "failures", + StreamableToString(test_case.failed_test_count())); + OutputXmlAttribute( + stream, kTestsuite, "disabled", + StreamableToString(test_case.reportable_disabled_test_count())); + OutputXmlAttribute(stream, kTestsuite, "errors", "0"); + OutputXmlAttribute(stream, kTestsuite, "time", + FormatTimeInMillisAsSeconds(test_case.elapsed_time())); + *stream << TestPropertiesAsXmlAttributes(test_case.ad_hoc_test_result()) + << ">\n"; - fprintf(out, - " <testsuite name=\"%s\" tests=\"%d\" failures=\"%d\" " - "disabled=\"%d\" ", - EscapeXmlAttribute(test_case.name()).c_str(), - test_case.total_test_count(), - test_case.failed_test_count(), - test_case.disabled_test_count()); - fprintf(out, - "errors=\"0\" time=\"%s\">\n", - FormatTimeInMillisAsSeconds(test_case.elapsed_time()).c_str()); for (int i = 0; i < test_case.total_test_count(); ++i) { - ::std::stringstream stream; - OutputXmlTestInfo(&stream, test_case.name(), *test_case.GetTestInfo(i)); - fprintf(out, "%s", StringStreamToString(&stream).c_str()); + if (test_case.GetTestInfo(i)->is_reportable()) + OutputXmlTestInfo(stream, test_case.name(), *test_case.GetTestInfo(i)); } - fprintf(out, " </testsuite>\n"); + *stream << " </" << kTestsuite << ">\n"; } // Prints an XML summary of unit_test to output stream out. -void XmlUnitTestResultPrinter::PrintXmlUnitTest(FILE* out, +void XmlUnitTestResultPrinter::PrintXmlUnitTest(std::ostream* stream, const UnitTest& unit_test) { - fprintf(out, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"); - fprintf(out, - "<testsuites tests=\"%d\" failures=\"%d\" disabled=\"%d\" " - "errors=\"0\" time=\"%s\" ", - unit_test.total_test_count(), - unit_test.failed_test_count(), - unit_test.disabled_test_count(), - FormatTimeInMillisAsSeconds(unit_test.elapsed_time()).c_str()); + const std::string kTestsuites = "testsuites"; + + *stream << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"; + *stream << "<" << kTestsuites; + + OutputXmlAttribute(stream, kTestsuites, "tests", + StreamableToString(unit_test.reportable_test_count())); + OutputXmlAttribute(stream, kTestsuites, "failures", + StreamableToString(unit_test.failed_test_count())); + OutputXmlAttribute( + stream, kTestsuites, "disabled", + StreamableToString(unit_test.reportable_disabled_test_count())); + OutputXmlAttribute(stream, kTestsuites, "errors", "0"); + OutputXmlAttribute( + stream, kTestsuites, "timestamp", + FormatEpochTimeInMillisAsIso8601(unit_test.start_timestamp())); + OutputXmlAttribute(stream, kTestsuites, "time", + FormatTimeInMillisAsSeconds(unit_test.elapsed_time())); + if (GTEST_FLAG(shuffle)) { - fprintf(out, "random_seed=\"%d\" ", unit_test.random_seed()); + OutputXmlAttribute(stream, kTestsuites, "random_seed", + StreamableToString(unit_test.random_seed())); + } + + *stream << TestPropertiesAsXmlAttributes(unit_test.ad_hoc_test_result()); + + OutputXmlAttribute(stream, kTestsuites, "name", "AllTests"); + *stream << ">\n"; + + for (int i = 0; i < unit_test.total_test_case_count(); ++i) { + if (unit_test.GetTestCase(i)->reportable_test_count() > 0) + PrintXmlTestCase(stream, *unit_test.GetTestCase(i)); } - fprintf(out, "name=\"AllTests\">\n"); - for (int i = 0; i < unit_test.total_test_case_count(); ++i) - PrintXmlTestCase(out, *unit_test.GetTestCase(i)); - fprintf(out, "</testsuites>\n"); + *stream << "</" << kTestsuites << ">\n"; } // Produces a string representing the test properties in a result as space // delimited XML attributes based on the property key="value" pairs. -String XmlUnitTestResultPrinter::TestPropertiesAsXmlAttributes( +std::string XmlUnitTestResultPrinter::TestPropertiesAsXmlAttributes( const TestResult& result) { Message attributes; for (int i = 0; i < result.test_property_count(); ++i) { @@ -3332,112 +3705,6 @@ String XmlUnitTestResultPrinter::TestPropertiesAsXmlAttributes( #if GTEST_CAN_STREAM_RESULTS_ -// Streams test results to the given port on the given host machine. -class StreamingListener : public EmptyTestEventListener { - public: - // Escapes '=', '&', '%', and '\n' characters in str as "%xx". - static string UrlEncode(const char* str); - - StreamingListener(const string& host, const string& port) - : sockfd_(-1), host_name_(host), port_num_(port) { - MakeConnection(); - Send("gtest_streaming_protocol_version=1.0\n"); - } - - virtual ~StreamingListener() { - if (sockfd_ != -1) - CloseConnection(); - } - - void OnTestProgramStart(const UnitTest& /* unit_test */) { - Send("event=TestProgramStart\n"); - } - - void OnTestProgramEnd(const UnitTest& unit_test) { - // Note that Google Test current only report elapsed time for each - // test iteration, not for the entire test program. - Send(String::Format("event=TestProgramEnd&passed=%d\n", - unit_test.Passed())); - - // Notify the streaming server to stop. - CloseConnection(); - } - - void OnTestIterationStart(const UnitTest& /* unit_test */, int iteration) { - Send(String::Format("event=TestIterationStart&iteration=%d\n", - iteration)); - } - - void OnTestIterationEnd(const UnitTest& unit_test, int /* iteration */) { - Send(String::Format("event=TestIterationEnd&passed=%d&elapsed_time=%sms\n", - unit_test.Passed(), - StreamableToString(unit_test.elapsed_time()).c_str())); - } - - void OnTestCaseStart(const TestCase& test_case) { - Send(String::Format("event=TestCaseStart&name=%s\n", test_case.name())); - } - - void OnTestCaseEnd(const TestCase& test_case) { - Send(String::Format("event=TestCaseEnd&passed=%d&elapsed_time=%sms\n", - test_case.Passed(), - StreamableToString(test_case.elapsed_time()).c_str())); - } - - void OnTestStart(const TestInfo& test_info) { - Send(String::Format("event=TestStart&name=%s\n", test_info.name())); - } - - void OnTestEnd(const TestInfo& test_info) { - Send(String::Format( - "event=TestEnd&passed=%d&elapsed_time=%sms\n", - (test_info.result())->Passed(), - StreamableToString((test_info.result())->elapsed_time()).c_str())); - } - - void OnTestPartResult(const TestPartResult& test_part_result) { - const char* file_name = test_part_result.file_name(); - if (file_name == NULL) - file_name = ""; - Send(String::Format("event=TestPartResult&file=%s&line=%d&message=", - UrlEncode(file_name).c_str(), - test_part_result.line_number())); - Send(UrlEncode(test_part_result.message()) + "\n"); - } - - private: - // Creates a client socket and connects to the server. - void MakeConnection(); - - // Closes the socket. - void CloseConnection() { - GTEST_CHECK_(sockfd_ != -1) - << "CloseConnection() can be called only when there is a connection."; - - close(sockfd_); - sockfd_ = -1; - } - - // Sends a string to the socket. - void Send(const string& message) { - GTEST_CHECK_(sockfd_ != -1) - << "Send() can be called only when there is a connection."; - - const int len = static_cast<int>(message.length()); - if (write(sockfd_, message.c_str(), len) != len) { - GTEST_LOG_(WARNING) - << "stream_result_to: failed to stream to " - << host_name_ << ":" << port_num_; - } - } - - int sockfd_; // socket file descriptor - const string host_name_; - const string port_num_; - - GTEST_DISALLOW_COPY_AND_ASSIGN_(StreamingListener); -}; // class StreamingListener - // Checks if str contains '=', '&', '%' or '\n' characters. If yes, // replaces them by "%xx" where xx is their hexadecimal value. For // example, replaces "=" with "%3D". This algorithm is O(strlen(str)) @@ -3452,7 +3719,7 @@ string StreamingListener::UrlEncode(const char* str) { case '=': case '&': case '\n': - result.append(String::Format("%%%02x", static_cast<unsigned char>(ch))); + result.append("%" + String::FormatByte(static_cast<unsigned char>(ch))); break; default: result.push_back(ch); @@ -3462,7 +3729,7 @@ string StreamingListener::UrlEncode(const char* str) { return result; } -void StreamingListener::MakeConnection() { +void StreamingListener::SocketWriter::MakeConnection() { GTEST_CHECK_(sockfd_ == -1) << "MakeConnection() can't be called when there is already a connection."; @@ -3510,8 +3777,8 @@ void StreamingListener::MakeConnection() { // Pushes the given source file location and message onto a per-thread // trace stack maintained by Google Test. -// L < UnitTest::mutex_ -ScopedTrace::ScopedTrace(const char* file, int line, const Message& message) { +ScopedTrace::ScopedTrace(const char* file, int line, const Message& message) + GTEST_LOCK_EXCLUDED_(&UnitTest::mutex_) { TraceInfo trace; trace.file = file; trace.line = line; @@ -3521,35 +3788,64 @@ ScopedTrace::ScopedTrace(const char* file, int line, const Message& message) { } // Pops the info pushed by the c'tor. -// L < UnitTest::mutex_ -ScopedTrace::~ScopedTrace() { +ScopedTrace::~ScopedTrace() + GTEST_LOCK_EXCLUDED_(&UnitTest::mutex_) { UnitTest::GetInstance()->PopGTestTrace(); } // class OsStackTraceGetter -// Returns the current OS stack trace as a String. Parameters: +// Returns the current OS stack trace as an std::string. Parameters: // // max_depth - the maximum number of stack frames to be included // in the trace. // skip_count - the number of top frames to be skipped; doesn't count // against max_depth. // -// L < mutex_ -// We use "L < mutex_" to denote that the function may acquire mutex_. -String OsStackTraceGetter::CurrentStackTrace(int, int) { - return String(""); +string OsStackTraceGetter::CurrentStackTrace(int /* max_depth */, + int /* skip_count */) + GTEST_LOCK_EXCLUDED_(mutex_) { + return ""; } -// L < mutex_ -void OsStackTraceGetter::UponLeavingGTest() { +void OsStackTraceGetter::UponLeavingGTest() + GTEST_LOCK_EXCLUDED_(mutex_) { } const char* const OsStackTraceGetter::kElidedFramesMarker = "... " GTEST_NAME_ " internal frames ..."; +// A helper class that creates the premature-exit file in its +// constructor and deletes the file in its destructor. +class ScopedPrematureExitFile { + public: + explicit ScopedPrematureExitFile(const char* premature_exit_filepath) + : premature_exit_filepath_(premature_exit_filepath) { + // If a path to the premature-exit file is specified... + if (premature_exit_filepath != NULL && *premature_exit_filepath != '\0') { + // create the file with a single "0" character in it. I/O + // errors are ignored as there's nothing better we can do and we + // don't want to fail the test because of this. + FILE* pfile = posix::FOpen(premature_exit_filepath, "w"); + fwrite("0", 1, 1, pfile); + fclose(pfile); + } + } + + ~ScopedPrematureExitFile() { + if (premature_exit_filepath_ != NULL && *premature_exit_filepath_ != '\0') { + remove(premature_exit_filepath_); + } + } + + private: + const char* const premature_exit_filepath_; + + GTEST_DISALLOW_COPY_AND_ASSIGN_(ScopedPrematureExitFile); +}; + } // namespace internal // class TestEventListeners @@ -3636,7 +3932,7 @@ void TestEventListeners::SuppressEventForwarding() { // We don't protect this under mutex_ as a user is not supposed to // call this before main() starts, from which point on the return // value will never change. -UnitTest * UnitTest::GetInstance() { +UnitTest* UnitTest::GetInstance() { // When compiled with MSVC 7.1 in optimized mode, destroying the // UnitTest object upon exiting the program messes up the exit code, // causing successful tests to appear failed. We have to use a @@ -3686,17 +3982,33 @@ int UnitTest::successful_test_count() const { // Gets the number of failed tests. int UnitTest::failed_test_count() const { return impl()->failed_test_count(); } +// Gets the number of disabled tests that will be reported in the XML report. +int UnitTest::reportable_disabled_test_count() const { + return impl()->reportable_disabled_test_count(); +} + // Gets the number of disabled tests. int UnitTest::disabled_test_count() const { return impl()->disabled_test_count(); } +// Gets the number of tests to be printed in the XML report. +int UnitTest::reportable_test_count() const { + return impl()->reportable_test_count(); +} + // Gets the number of all tests. int UnitTest::total_test_count() const { return impl()->total_test_count(); } // Gets the number of tests that should run. int UnitTest::test_to_run_count() const { return impl()->test_to_run_count(); } +// Gets the time of the test program start, in ms from the start of the +// UNIX epoch. +internal::TimeInMillis UnitTest::start_timestamp() const { + return impl()->start_timestamp(); +} + // Gets the elapsed time, in milliseconds. internal::TimeInMillis UnitTest::elapsed_time() const { return impl()->elapsed_time(); @@ -3715,6 +4027,12 @@ const TestCase* UnitTest::GetTestCase(int i) const { return impl()->GetTestCase(i); } +// Returns the TestResult containing information on test failures and +// properties logged outside of individual test cases. +const TestResult& UnitTest::ad_hoc_test_result() const { + return *impl()->ad_hoc_test_result(); +} + // Gets the i-th test case among all the test cases. i can range from 0 to // total_test_case_count() - 1. If i is not in that range, returns NULL. TestCase* UnitTest::GetMutableTestCase(int i) { @@ -3750,12 +4068,12 @@ Environment* UnitTest::AddEnvironment(Environment* env) { // assertion macros (e.g. ASSERT_TRUE, EXPECT_EQ, etc) eventually call // this to report their results. The user code should use the // assertion macros instead of calling this directly. -// L < mutex_ -void UnitTest::AddTestPartResult(TestPartResult::Type result_type, - const char* file_name, - int line_number, - const internal::String& message, - const internal::String& os_stack_trace) { +void UnitTest::AddTestPartResult( + TestPartResult::Type result_type, + const char* file_name, + int line_number, + const std::string& message, + const std::string& os_stack_trace) GTEST_LOCK_EXCLUDED_(mutex_) { Message msg; msg << message; @@ -3788,7 +4106,7 @@ void UnitTest::AddTestPartResult(TestPartResult::Type result_type, // with another testing framework) and specify the former on the // command line for debugging. if (GTEST_FLAG(break_on_failure)) { -#if GTEST_OS_WINDOWS +#if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT // Using DebugBreak on Windows allows gtest to still break into a debugger // when a failure happens and both the --gtest_break_on_failure and // the --gtest_catch_exceptions flags are specified. @@ -3802,7 +4120,7 @@ void UnitTest::AddTestPartResult(TestPartResult::Type result_type, #endif // GTEST_OS_WINDOWS } else if (GTEST_FLAG(throw_on_failure)) { #if GTEST_HAS_EXCEPTIONS - throw GoogleTestFailureException(result); + throw internal::GoogleTestFailureException(result); #else // We cannot call abort() as it generates a pop-up in debug mode // that cannot be suppressed in VC 7.1 or below. @@ -3812,12 +4130,14 @@ void UnitTest::AddTestPartResult(TestPartResult::Type result_type, } } -// Creates and adds a property to the current TestResult. If a property matching -// the supplied value already exists, updates its value instead. -void UnitTest::RecordPropertyForCurrentTest(const char* key, - const char* value) { - const TestProperty test_property(key, value); - impl_->current_test_result()->RecordProperty(test_property); +// Adds a TestProperty to the current TestResult object when invoked from +// inside a test, to current TestCase's ad_hoc_test_result_ when invoked +// from SetUpTestCase or TearDownTestCase, or to the global property set +// when invoked elsewhere. If the result already contains a property with +// the same key, the value will be updated. +void UnitTest::RecordProperty(const std::string& key, + const std::string& value) { + impl_->RecordProperty(TestProperty(key, value)); } // Runs all tests in this UnitTest object and prints the result. @@ -3826,21 +4146,45 @@ void UnitTest::RecordPropertyForCurrentTest(const char* key, // We don't protect this under mutex_, as we only support calling it // from the main thread. int UnitTest::Run() { + const bool in_death_test_child_process = + internal::GTEST_FLAG(internal_run_death_test).length() > 0; + + // Google Test implements this protocol for catching that a test + // program exits before returning control to Google Test: + // + // 1. Upon start, Google Test creates a file whose absolute path + // is specified by the environment variable + // TEST_PREMATURE_EXIT_FILE. + // 2. When Google Test has finished its work, it deletes the file. + // + // This allows a test runner to set TEST_PREMATURE_EXIT_FILE before + // running a Google-Test-based test program and check the existence + // of the file at the end of the test execution to see if it has + // exited prematurely. + + // If we are in the child process of a death test, don't + // create/delete the premature exit file, as doing so is unnecessary + // and will confuse the parent process. Otherwise, create/delete + // the file upon entering/leaving this function. If the program + // somehow exits before this function has a chance to return, the + // premature-exit file will be left undeleted, causing a test runner + // that understands the premature-exit-file protocol to report the + // test as having failed. + const internal::ScopedPrematureExitFile premature_exit_file( + in_death_test_child_process ? + NULL : internal::posix::GetEnv("TEST_PREMATURE_EXIT_FILE")); + // Captures the value of GTEST_FLAG(catch_exceptions). This value will be // used for the duration of the program. impl()->set_catch_exceptions(GTEST_FLAG(catch_exceptions)); #if GTEST_HAS_SEH - const bool in_death_test_child_process = - internal::GTEST_FLAG(internal_run_death_test).length() > 0; - // Either the user wants Google Test to catch exceptions thrown by the // tests or this is executing in the context of death test child // process. In either case the user does not want to see pop-up dialogs // about crashes - they are expected. if (impl()->catch_exceptions() || in_death_test_child_process) { - -# if !GTEST_OS_WINDOWS_MOBILE +# if !GTEST_OS_WINDOWS_MOBILE && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT // SetErrorMode doesn't exist on CE. SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOALIGNMENTFAULTEXCEPT | SEM_NOGPFAULTERRORBOX | SEM_NOOPENFILEERRORBOX); @@ -3870,7 +4214,6 @@ int UnitTest::Run() { 0x0, // Clear the following flags: _WRITE_ABORT_MSG | _CALL_REPORTFAULT); // pop-up window, core dump. # endif - } #endif // GTEST_HAS_SEH @@ -3888,16 +4231,16 @@ const char* UnitTest::original_working_dir() const { // Returns the TestCase object for the test that's currently running, // or NULL if no test is running. -// L < mutex_ -const TestCase* UnitTest::current_test_case() const { +const TestCase* UnitTest::current_test_case() const + GTEST_LOCK_EXCLUDED_(mutex_) { internal::MutexLock lock(&mutex_); return impl_->current_test_case(); } // Returns the TestInfo object for the test that's currently running, // or NULL if no test is running. -// L < mutex_ -const TestInfo* UnitTest::current_test_info() const { +const TestInfo* UnitTest::current_test_info() const + GTEST_LOCK_EXCLUDED_(mutex_) { internal::MutexLock lock(&mutex_); return impl_->current_test_info(); } @@ -3908,9 +4251,9 @@ int UnitTest::random_seed() const { return impl_->random_seed(); } #if GTEST_HAS_PARAM_TEST // Returns ParameterizedTestCaseRegistry object used to keep track of // value-parameterized tests and instantiate and register them. -// L < mutex_ internal::ParameterizedTestCaseRegistry& - UnitTest::parameterized_test_registry() { + UnitTest::parameterized_test_registry() + GTEST_LOCK_EXCLUDED_(mutex_) { return impl_->parameterized_test_registry(); } #endif // GTEST_HAS_PARAM_TEST @@ -3927,15 +4270,15 @@ UnitTest::~UnitTest() { // Pushes a trace defined by SCOPED_TRACE() on to the per-thread // Google Test trace stack. -// L < mutex_ -void UnitTest::PushGTestTrace(const internal::TraceInfo& trace) { +void UnitTest::PushGTestTrace(const internal::TraceInfo& trace) + GTEST_LOCK_EXCLUDED_(mutex_) { internal::MutexLock lock(&mutex_); impl_->gtest_trace_stack().push_back(trace); } // Pops a trace from the per-thread Google Test trace stack. -// L < mutex_ -void UnitTest::PopGTestTrace() { +void UnitTest::PopGTestTrace() + GTEST_LOCK_EXCLUDED_(mutex_) { internal::MutexLock lock(&mutex_); impl_->gtest_trace_stack().pop_back(); } @@ -3944,17 +4287,10 @@ namespace internal { UnitTestImpl::UnitTestImpl(UnitTest* parent) : parent_(parent), -#ifdef _MSC_VER -# pragma warning(push) // Saves the current warning state. -# pragma warning(disable:4355) // Temporarily disables warning 4355 - // (using this in initializer). + GTEST_DISABLE_MSC_WARNINGS_PUSH_(4355 /* using this in initializer */) default_global_test_part_result_reporter_(this), default_per_thread_test_part_result_reporter_(this), -# pragma warning(pop) // Restores the warning state again. -#else - default_global_test_part_result_reporter_(this), - default_per_thread_test_part_result_reporter_(this), -#endif // _MSC_VER + GTEST_DISABLE_MSC_WARNINGS_POP_() global_test_part_result_repoter_( &default_global_test_part_result_reporter_), per_thread_test_part_result_reporter_( @@ -3971,9 +4307,9 @@ UnitTestImpl::UnitTestImpl(UnitTest* parent) post_flag_parse_init_performed_(false), random_seed_(0), // Will be overridden by the flag before first use. random_(0), // Will be reseeded before first use. + start_timestamp_(0), elapsed_time_(0), #if GTEST_HAS_DEATH_TEST - internal_run_death_test_flag_(NULL), death_test_factory_(new DefaultDeathTestFactory), #endif // Will be overridden by the flag before first use. @@ -3991,6 +4327,28 @@ UnitTestImpl::~UnitTestImpl() { delete os_stack_trace_getter_; } +// Adds a TestProperty to the current TestResult object when invoked in a +// context of a test, to current test case's ad_hoc_test_result when invoke +// from SetUpTestCase/TearDownTestCase, or to the global property set +// otherwise. If the result already contains a property with the same key, +// the value will be updated. +void UnitTestImpl::RecordProperty(const TestProperty& test_property) { + std::string xml_element; + TestResult* test_result; // TestResult appropriate for property recording. + + if (current_test_info_ != NULL) { + xml_element = "testcase"; + test_result = &(current_test_info_->result_); + } else if (current_test_case_ != NULL) { + xml_element = "testsuite"; + test_result = &(current_test_case_->ad_hoc_test_result_); + } else { + xml_element = "testsuites"; + test_result = &ad_hoc_test_result_; + } + test_result->RecordProperty(xml_element, test_property); +} + #if GTEST_HAS_DEATH_TEST // Disables event forwarding if the control is currently in a death test // subprocess. Must not be called before InitGoogleTest. @@ -4003,7 +4361,7 @@ void UnitTestImpl::SuppressTestEventsIfInSubprocess() { // Initializes event listeners performing XML output as specified by // UnitTestOptions. Must not be called before InitGoogleTest. void UnitTestImpl::ConfigureXmlOutput() { - const String& output_format = UnitTestOptions::GetOutputFormat(); + const std::string& output_format = UnitTestOptions::GetOutputFormat(); if (output_format == "xml") { listeners()->SetDefaultXmlGenerator(new XmlUnitTestResultPrinter( UnitTestOptions::GetAbsolutePathToOutputFile().c_str())); @@ -4015,13 +4373,13 @@ void UnitTestImpl::ConfigureXmlOutput() { } #if GTEST_CAN_STREAM_RESULTS_ -// Initializes event listeners for streaming test results in String form. +// Initializes event listeners for streaming test results in string form. // Must not be called before InitGoogleTest. void UnitTestImpl::ConfigureStreamingOutput() { - const string& target = GTEST_FLAG(stream_result_to); + const std::string& target = GTEST_FLAG(stream_result_to); if (!target.empty()) { const size_t pos = target.find(':'); - if (pos != string::npos) { + if (pos != std::string::npos) { listeners()->Append(new StreamingListener(target.substr(0, pos), target.substr(pos+1))); } else { @@ -4075,7 +4433,7 @@ void UnitTestImpl::PostFlagParsingInit() { class TestCaseNameIs { public: // Constructor. - explicit TestCaseNameIs(const String& name) + explicit TestCaseNameIs(const std::string& name) : name_(name) {} // Returns true iff the name of test_case matches name_. @@ -4084,7 +4442,7 @@ class TestCaseNameIs { } private: - String name_; + std::string name_; }; // Finds and returns a TestCase with the given name. If one doesn't @@ -4116,7 +4474,7 @@ TestCase* UnitTestImpl::GetTestCase(const char* test_case_name, new TestCase(test_case_name, type_param, set_up_tc, tear_down_tc); // Is this a death test case? - if (internal::UnitTestOptions::MatchesFilter(String(test_case_name), + if (internal::UnitTestOptions::MatchesFilter(test_case_name, kDeathTestCaseFilter)) { // Yes. Inserts the test case after the last death test case // defined so far. This only works when the test cases haven't @@ -4202,6 +4560,7 @@ bool UnitTestImpl::RunAllTests() { TestEventListener* repeater = listeners()->repeater(); + start_timestamp_ = GetTimeInMillis(); repeater->OnTestProgramStart(*parent_); // How many times to repeat the tests? We don't want to repeat them @@ -4394,14 +4753,12 @@ int UnitTestImpl::FilterTests(ReactionToSharding shard_tests) { int num_selected_tests = 0; for (size_t i = 0; i < test_cases_.size(); i++) { TestCase* const test_case = test_cases_[i]; - const String &test_case_name = test_case->name(); + const std::string &test_case_name = test_case->name(); test_case->set_should_run(false); - bool any_matched_filter = false; - for (size_t j = 0; j < test_case->test_info_list().size(); j++) { TestInfo* const test_info = test_case->test_info_list()[j]; - const String test_name(test_info->name()); + const std::string test_name(test_info->name()); // A test is disabled if test case name or test name matches // kDisableTestFilter. const bool is_disabled = @@ -4415,7 +4772,6 @@ int UnitTestImpl::FilterTests(ReactionToSharding shard_tests) { internal::UnitTestOptions::FilterMatchesTest(test_case_name, test_name); test_info->matches_filter_ = matches_filter; - any_matched_filter |= matches_filter; const bool is_runnable = (GTEST_FLAG(also_run_disabled_tests) || !is_disabled) && @@ -4432,14 +4788,37 @@ int UnitTestImpl::FilterTests(ReactionToSharding shard_tests) { test_info->should_run_ = is_selected; test_case->set_should_run(test_case->should_run() || is_selected); } - - test_case->set_should_skip_report(!any_matched_filter); } return num_selected_tests; } +// Prints the given C-string on a single line by replacing all '\n' +// characters with string "\\n". If the output takes more than +// max_length characters, only prints the first max_length characters +// and "...". +static void PrintOnOneLine(const char* str, int max_length) { + if (str != NULL) { + for (int i = 0; *str != '\0'; ++str) { + if (i >= max_length) { + printf("..."); + break; + } + if (*str == '\n') { + printf("\\n"); + i += 2; + } else { + printf("%c", *str); + ++i; + } + } + } +} + // Prints the names of the tests matching the user-specified filter flag. void UnitTestImpl::ListTestsMatchingFilter() { + // Print at most this many characters for each type/value parameter. + const int kMaxParamLength = 250; + for (size_t i = 0; i < test_cases_.size(); i++) { const TestCase* const test_case = test_cases_[i]; bool printed_test_case_name = false; @@ -4450,9 +4829,23 @@ void UnitTestImpl::ListTestsMatchingFilter() { if (test_info->matches_filter_) { if (!printed_test_case_name) { printed_test_case_name = true; - printf("%s.\n", test_case->name()); + printf("%s.", test_case->name()); + if (test_case->type_param() != NULL) { + printf(" # %s = ", kTypeParamLabel); + // We print the type parameter on a single line to make + // the output easy to parse by a program. + PrintOnOneLine(test_case->type_param(), kMaxParamLength); + } + printf("\n"); + } + printf(" %s", test_info->name()); + if (test_info->value_param() != NULL) { + printf(" # %s = ", kValueParamLabel); + // We print the value parameter on a single line to make the + // output easy to parse by a program. + PrintOnOneLine(test_info->value_param(), kMaxParamLength); } - printf(" %s\n", test_info->name()); + printf("\n"); } } } @@ -4516,7 +4909,7 @@ void UnitTestImpl::UnshuffleTests() { } } -// Returns the current OS stack trace as a String. +// Returns the current OS stack trace as an std::string. // // The maximum number of stack frames to be included is specified by // the gtest_stack_trace_depth flag. The skip_count parameter @@ -4526,8 +4919,8 @@ void UnitTestImpl::UnshuffleTests() { // For example, if Foo() calls Bar(), which in turn calls // GetCurrentOsStackTraceExceptTop(..., 1), Foo() will be included in // the trace but Bar() and GetCurrentOsStackTraceExceptTop() won't. -String GetCurrentOsStackTraceExceptTop(UnitTest* /*unit_test*/, - int skip_count) { +std::string GetCurrentOsStackTraceExceptTop(UnitTest* /*unit_test*/, + int skip_count) { // We pass skip_count + 1 to skip this wrapper function in addition // to what the user really wants to skip. return GetUnitTestImpl()->CurrentOsStackTraceExceptTop(skip_count + 1); @@ -4575,7 +4968,7 @@ const char* ParseFlagValue(const char* str, if (str == NULL || flag == NULL) return NULL; // The flag must start with "--" followed by GTEST_FLAG_PREFIX_. - const String flag_str = String::Format("--%s%s", GTEST_FLAG_PREFIX_, flag); + const std::string flag_str = std::string("--") + GTEST_FLAG_PREFIX_ + flag; const size_t flag_len = flag_str.length(); if (strncmp(str, flag_str.c_str(), flag_len) != 0) return NULL; @@ -4640,7 +5033,7 @@ bool ParseInt32Flag(const char* str, const char* flag, Int32* value) { // // On success, stores the value of the flag in *value, and returns // true. On failure, returns false without changing *value. -bool ParseStringFlag(const char* str, const char* flag, String* value) { +bool ParseStringFlag(const char* str, const char* flag, std::string* value) { // Gets the value of the flag as a string. const char* const value_str = ParseFlagValue(str, flag, false); @@ -4692,7 +5085,7 @@ static void PrintColorEncoded(const char* str) { return; } - ColoredPrintf(color, "%s", String(str, p - str).c_str()); + ColoredPrintf(color, "%s", std::string(str, p).c_str()); const char ch = p[1]; str = p + 2; @@ -4782,7 +5175,7 @@ static const char kColorEncodedHelpMessage[] = template <typename CharType> void ParseGoogleTestFlagsOnlyImpl(int* argc, CharType** argv) { for (int i = 1; i < *argc; i++) { - const String arg_string = StreamableToString(argv[i]); + const std::string arg_string = StreamableToString(argv[i]); const char* const arg = arg_string.c_str(); using internal::ParseBoolFlag; diff --git a/tests/gtest/src/gtest_main.cc b/tests/gtest/src/gtest_main.cc index a09bbe0c6..f30282255 100644 --- a/tests/gtest/src/gtest_main.cc +++ b/tests/gtest/src/gtest_main.cc @@ -27,13 +27,12 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#include <iostream> +#include <stdio.h> #include "gtest/gtest.h" GTEST_API_ int main(int argc, char **argv) { - std::cout << "Running main() from gtest_main.cc\n"; - + printf("Running main() from gtest_main.cc\n"); testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } diff --git a/tests/hash-target.cpp b/tests/hash-target.cpp index ad81ef546..bc4a556a0 100644 --- a/tests/hash-target.cpp +++ b/tests/hash-target.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/hash/CMakeLists.txt b/tests/hash/CMakeLists.txt index 2f441d623..6c7297f2e 100644 --- a/tests/hash/CMakeLists.txt +++ b/tests/hash/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # diff --git a/tests/hash/main.cpp b/tests/hash/main.cpp index a16afb171..654cdcc80 100644 --- a/tests/hash/main.cpp +++ b/tests/hash/main.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/io.h b/tests/io.h index 12845edae..5debd7804 100644 --- a/tests/io.h +++ b/tests/io.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/net_load_tests/CMakeLists.txt b/tests/net_load_tests/CMakeLists.txt index 114d0e680..acd7c9ac9 100644 --- a/tests/net_load_tests/CMakeLists.txt +++ b/tests/net_load_tests/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # @@ -37,6 +37,9 @@ add_executable(net_load_tests_clt ${clt_headers}) target_link_libraries(net_load_tests_clt LINK_PRIVATE + otshell_utils + p2p + cryptonote_core ${GTEST_MAIN_LIBRARIES} ${Boost_CHRONO_LIBRARY} ${Boost_DATE_TIME_LIBRARY} @@ -56,6 +59,9 @@ add_executable(net_load_tests_srv ${srv_headers}) target_link_libraries(net_load_tests_srv LINK_PRIVATE + otshell_utils + p2p + cryptonote_core ${GTEST_MAIN_LIBRARIES} ${Boost_CHRONO_LIBRARY} ${Boost_DATE_TIME_LIBRARY} diff --git a/tests/net_load_tests/clt.cpp b/tests/net_load_tests/clt.cpp index 735f3658b..95702e1a8 100644 --- a/tests/net_load_tests/clt.cpp +++ b/tests/net_load_tests/clt.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -44,7 +44,10 @@ #include "net_load_tests.h" +#include "../../contrib/otshell_utils/utils.hpp" + using namespace net_load_tests; +using namespace nOT::nUtils; namespace { @@ -183,6 +186,11 @@ namespace class net_load_test_clt : public ::testing::Test { + public: + net_load_test_clt() + : m_tcp_server(epee::net_utils::e_connection_type_RPC) // RPC disables network limit for unit tests + { + } protected: virtual void SetUp() { @@ -234,7 +242,7 @@ namespace { // Stop server test_levin_commands_handler commands_handler; - test_tcp_server tcp_server; + test_tcp_server tcp_server(epee::net_utils::e_connection_type_NET); tcp_server.get_config_object().m_pcommands_handler = &commands_handler; tcp_server.get_config_object().m_invoke_timeout = CONNECTION_TIMEOUT; @@ -620,6 +628,8 @@ TEST_F(net_load_test_clt, permament_open_and_close_and_connections_closed_by_ser ASSERT_EQ(RESERVED_CONN_CNT, m_tcp_server.get_config_object().get_connections_count()); } +unsigned int epee::g_test_dbg_lock_sleep = 0; + int main(int argc, char** argv) { epee::debug::get_set_enable_assert(true, false); @@ -628,5 +638,6 @@ int main(int argc, char** argv) epee::log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL); ::testing::InitGoogleTest(&argc, argv); + epee::net_utils::data_logger::get_instance().kill_instance(); return RUN_ALL_TESTS(); } diff --git a/tests/net_load_tests/net_load_tests.h b/tests/net_load_tests/net_load_tests.h index cda950ec4..3f8b9561a 100644 --- a/tests/net_load_tests/net_load_tests.h +++ b/tests/net_load_tests/net_load_tests.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/net_load_tests/srv.cpp b/tests/net_load_tests/srv.cpp index 7c79d101a..12b0e46c8 100644 --- a/tests/net_load_tests/srv.cpp +++ b/tests/net_load_tests/srv.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -213,6 +213,8 @@ namespace }; } +unsigned int epee::g_test_dbg_lock_sleep = 0; + int main(int argc, char** argv) { //set up logging options @@ -221,7 +223,7 @@ int main(int argc, char** argv) size_t thread_count = (std::max)(min_thread_count, std::thread::hardware_concurrency() / 2); - test_tcp_server tcp_server; + test_tcp_server tcp_server(epee::net_utils::e_connection_type_RPC); if (!tcp_server.init_server(srv_port, "127.0.0.1")) return 1; @@ -232,6 +234,6 @@ int main(int argc, char** argv) if (!tcp_server.run_server(thread_count, true)) return 2; - + epee::net_utils::data_logger::get_instance().kill_instance(); return 0; } diff --git a/tests/performance_tests/CMakeLists.txt b/tests/performance_tests/CMakeLists.txt index ed4d3d4b6..a64c9fdc1 100644 --- a/tests/performance_tests/CMakeLists.txt +++ b/tests/performance_tests/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # diff --git a/tests/performance_tests/check_ring_signature.h b/tests/performance_tests/check_ring_signature.h index c0107b7e9..c8ab5fc7c 100644 --- a/tests/performance_tests/check_ring_signature.h +++ b/tests/performance_tests/check_ring_signature.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/cn_slow_hash.h b/tests/performance_tests/cn_slow_hash.h index 0915ef3c3..3f4011637 100644 --- a/tests/performance_tests/cn_slow_hash.h +++ b/tests/performance_tests/cn_slow_hash.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/construct_tx.h b/tests/performance_tests/construct_tx.h index 8f3ba452f..caccc5c2d 100644 --- a/tests/performance_tests/construct_tx.h +++ b/tests/performance_tests/construct_tx.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/derive_public_key.h b/tests/performance_tests/derive_public_key.h index 411ee1e82..7aea03cec 100644 --- a/tests/performance_tests/derive_public_key.h +++ b/tests/performance_tests/derive_public_key.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/derive_secret_key.h b/tests/performance_tests/derive_secret_key.h index dff329779..12b5b8aec 100644 --- a/tests/performance_tests/derive_secret_key.h +++ b/tests/performance_tests/derive_secret_key.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/generate_key_derivation.h b/tests/performance_tests/generate_key_derivation.h index 3c2c9b2fc..3d56a0a11 100644 --- a/tests/performance_tests/generate_key_derivation.h +++ b/tests/performance_tests/generate_key_derivation.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/generate_key_image.h b/tests/performance_tests/generate_key_image.h index 4669e1b62..54e30df4e 100644 --- a/tests/performance_tests/generate_key_image.h +++ b/tests/performance_tests/generate_key_image.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/generate_key_image_helper.h b/tests/performance_tests/generate_key_image_helper.h index efe1ecc43..a839ac89d 100644 --- a/tests/performance_tests/generate_key_image_helper.h +++ b/tests/performance_tests/generate_key_image_helper.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/is_out_to_acc.h b/tests/performance_tests/is_out_to_acc.h index b5819c2c3..17a23054e 100644 --- a/tests/performance_tests/is_out_to_acc.h +++ b/tests/performance_tests/is_out_to_acc.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/main.cpp b/tests/performance_tests/main.cpp index d5a2aa960..588cf6529 100644 --- a/tests/performance_tests/main.cpp +++ b/tests/performance_tests/main.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -42,6 +42,8 @@ #include "generate_key_image_helper.h" #include "is_out_to_acc.h" +unsigned int epee::g_test_dbg_lock_sleep = 0; + int main(int argc, char** argv) { set_process_affinity(1); diff --git a/tests/performance_tests/multi_tx_test_base.h b/tests/performance_tests/multi_tx_test_base.h index 96c63da71..99a40dac5 100644 --- a/tests/performance_tests/multi_tx_test_base.h +++ b/tests/performance_tests/multi_tx_test_base.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/performance_tests.h b/tests/performance_tests/performance_tests.h index 3d6f6f4fe..75ff38671 100644 --- a/tests/performance_tests/performance_tests.h +++ b/tests/performance_tests/performance_tests.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/performance_utils.h b/tests/performance_tests/performance_utils.h index d585a0688..9ac26fb53 100644 --- a/tests/performance_tests/performance_utils.h +++ b/tests/performance_tests/performance_utils.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/performance_tests/single_tx_test_base.h b/tests/performance_tests/single_tx_test_base.h index 22d6c6f36..1b3a07b17 100644 --- a/tests/performance_tests/single_tx_test_base.h +++ b/tests/performance_tests/single_tx_test_base.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/unit_tests/BlockchainDB.cpp b/tests/unit_tests/BlockchainDB.cpp new file mode 100644 index 000000000..b6e311daf --- /dev/null +++ b/tests/unit_tests/BlockchainDB.cpp @@ -0,0 +1,325 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <boost/filesystem.hpp> +#include <boost/algorithm/string/predicate.hpp> +#include <cstdio> +#include <iostream> +#include <chrono> +#include <thread> + +#include "gtest/gtest.h" + +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/lmdb/db_lmdb.h" +#include "blockchain_db/berkeleydb/db_bdb.h" +#include "cryptonote_core/cryptonote_format_utils.h" + +using namespace cryptonote; +using epee::string_tools::pod_to_hex; + +#define ASSERT_HASH_EQ(a,b) ASSERT_EQ(pod_to_hex(a), pod_to_hex(b)) + +namespace { // anonymous namespace + +const std::vector<std::string> t_blocks = + { + "0100d5adc49a053b8818b2b6023cd2d532c6774e164a8fcacd603651cb3ea0cb7f9340b28ec016b4bc4ca301aa0101ff6e08acbb2702eab03067870349139bee7eab2ca2e030a6bb73d4f68ab6a3b6ca937214054cdac0843d028bbe23b57ea9bae53f12da93bb57bf8a2e40598d9fccd10c2921576e987d93cd80b4891302468738e391f07c4f2b356f7957160968e0bfef6e907c3cee2d8c23cbf04b089680c6868f01025a0f41f063e195a966051e3a29e17130a9ce97d48f55285b9bb04bdd55a09ae78088aca3cf0202d0f26169290450fe17e08974789c3458910b4db18361cdc564f8f2d0bdd2cf568090cad2c60e02d6f3483ec45505cc3be841046c7a12bf953ac973939bc7b727e54258e1881d4d80e08d84ddcb0102dae6dfb16d3e28aaaf43e00170b90606b36f35f38f8a3dceb5ee18199dd8f17c80c0caf384a30202385d7e57a4daba4cdd9e550a92dcc188838386e7581f13f09de796cbed4716a42101c052492a077abf41996b50c1b2e67fd7288bcd8c55cdc657b4e22d0804371f6901beb76a82ea17400cd6d7f595f70e1667d2018ed8f5a78d1ce07484222618c3cd" + , "0100f9adc49a057d3113f562eac36f14afa08c22ae20bbbf8cffa31a4466d24850732cb96f80e9762365ee01ab0101ff6f08cc953502be76deb845c431f2ed9a4862457654b914003693b8cd672abc935f0d97b16380c08db7010291819f2873e3efbae65ecd5a736f5e8a26318b591c21e39a03fb536520ac63ba80dac40902439a10fde02e39e48e0b31e57cc084a07eedbefb8cbea0143aedd0442b189caa80c6868f010227b84449de4cd7a48cbdce8974baf0b6646e03384e32055e705c243a86bef8a58088aca3cf0202fa7bd15e4e7e884307ab130bb9d50e33c5fcea6546042a26f948efd5952459ee8090cad2c60e028695583dbb8f8faab87e3ef3f88fa827db097bbf51761d91924f5c5b74c6631780e08d84ddcb010279d2f247b54690e3b491e488acff16014a825fd740c23988a25df7c4670c1f2580c0caf384a302022599dfa3f8788b66295051d85937816e1c320cdb347a0fba5219e3fe60c83b2421010576509c5672025d28fd5d3f38efce24e1f9aaf65dd3056b2504e6e2b7f19f7800" + }; + +const std::vector<size_t> t_sizes = + { + 1122 + , 347 + }; + +const std::vector<difficulty_type> t_diffs = + { + 4003674 + , 4051757 + }; + +const std::vector<uint64_t> t_coins = + { + 1952630229575370 + , 1970220553446486 + }; + +const std::vector<std::vector<std::string>> t_transactions = + { + { + "0100010280e08d84ddcb0106010401110701f254220bb50d901a5523eaed438af5d43f8c6d0e54ba0632eb539884f6b7c02008c0a8a50402f9c7cf807ae74e56f4ec84db2bd93cfb02c2249b38e306f5b54b6e05d00d543b8095f52a02b6abb84e00f47f0a72e37b6b29392d906a38468404c57db3dbc5e8dd306a27a880d293ad0302cfc40a86723e7d459e90e45d47818dc0e81a1f451ace5137a4af8110a89a35ea80b4c4c321026b19c796338607d5a2c1ba240a167134142d72d1640ef07902da64fed0b10cfc8088aca3cf02021f6f655254fee84161118b32e7b6f8c31de5eb88aa00c29a8f57c0d1f95a24dd80d0b8e1981a023321af593163cea2ae37168ab926efd87f195756e3b723e886bdb7e618f751c480a094a58d1d0295ed2b08d1cf44482ae0060a5dcc4b7d810a85dea8c62e274f73862f3d59f8ed80a0e5b9c2910102dc50f2f28d7ceecd9a1147f7106c8d5b4e08b2ec77150f52dd7130ee4f5f50d42101d34f90ac861d0ee9fe3891656a234ea86a8a93bf51a237db65baa00d3f4aa196a9e1d89bc06b40e94ea9a26059efc7ba5b2de7ef7c139831ca62f3fe0bb252008f8c7ee810d3e1e06313edf2db362fc39431755779466b635f12f9f32e44470a3e85e08a28fcd90633efc94aa4ae39153dfaf661089d045521343a3d63e8da08d7916753c66aaebd4eefcfe8e58e5b3d266b752c9ca110749fa33fce7c44270386fcf2bed4f03dd5dadb2dc1fd4c505419f8217b9eaec07521f0d8963e104603c926745039cf38d31de6ed95ace8e8a451f5a36f818c151f517546d55ac0f500e54d07b30ea7452f2e93fa4f60bdb30d71a0a97f97eb121e662006780fbf69002228224a96bff37893d47ec3707b17383906c0cd7d9e7412b3e6c8ccf1419b093c06c26f96e3453b424713cdc5c9575f81cda4e157052df11f4c40809edf420f88a3dd1f7909bbf77c8b184a933389094a88e480e900bcdbf6d1824742ee520fc0032e7d892a2b099b8c6edfd1123ce58a34458ee20cad676a7f7cfd80a28f0cb0888af88838310db372986bdcf9bfcae2324480ca7360d22bff21fb569a530e" + } + , { + } + }; + +// if the return type (blobdata for now) of block_to_blob ever changes +// from std::string, this might break. +bool compare_blocks(const block& a, const block& b) +{ + auto hash_a = pod_to_hex(get_block_hash(a)); + auto hash_b = pod_to_hex(get_block_hash(b)); + + return hash_a == hash_b; +} + +/* +void print_block(const block& blk, const std::string& prefix = "") +{ + std::cerr << prefix << ": " << std::endl + << "\thash - " << pod_to_hex(get_block_hash(blk)) << std::endl + << "\tparent - " << pod_to_hex(blk.prev_id) << std::endl + << "\ttimestamp - " << blk.timestamp << std::endl + ; +} + +// if the return type (blobdata for now) of tx_to_blob ever changes +// from std::string, this might break. +bool compare_txs(const transaction& a, const transaction& b) +{ + auto ab = tx_to_blob(a); + auto bb = tx_to_blob(b); + + return ab == bb; +} +*/ + +// convert hex string to string that has values based on that hex +// thankfully should automatically ignore null-terminator. +std::string h2b(const std::string& s) +{ + bool upper = true; + std::string result; + unsigned char val = 0; + for (char c : s) + { + if (upper) + { + val = 0; + if (c <= 'f' && c >= 'a') + { + val = ((c - 'a') + 10) << 4; + } + else + { + val = (c - '0') << 4; + } + } + else + { + if (c <= 'f' && c >= 'a') + { + val |= (c - 'a') + 10; + } + else + { + val |= c - '0'; + } + result += (char)val; + } + upper = !upper; + } + return result; +} + +template <typename T> +class BlockchainDBTest : public testing::Test +{ +protected: + BlockchainDBTest() : m_db(new T()) + { + for (auto& i : t_blocks) + { + block bl; + blobdata bd = h2b(i); + parse_and_validate_block_from_blob(bd, bl); + m_blocks.push_back(bl); + } + for (auto& i : t_transactions) + { + std::vector<transaction> txs; + for (auto& j : i) + { + transaction tx; + blobdata bd = h2b(j); + parse_and_validate_tx_from_blob(bd, tx); + txs.push_back(tx); + } + m_txs.push_back(txs); + } + } + + ~BlockchainDBTest() { + delete m_db; + remove_files(); + } + + BlockchainDB* m_db; + std::string m_prefix; + std::vector<block> m_blocks; + std::vector<std::vector<transaction> > m_txs; + std::vector<std::string> m_filenames; + + void get_filenames() + { + m_filenames = m_db->get_filenames(); + for (auto& f : m_filenames) + { + std::cerr << "File created by test: " << f << std::endl; + } + } + + void remove_files() + { + // remove each file the db created, making sure it starts with fname. + for (auto& f : m_filenames) + { + if (boost::starts_with(f, m_prefix)) + { + boost::filesystem::remove(f); + } + else + { + std::cerr << "File created by test not to be removed (for safety): " << f << std::endl; + } + } + + // remove directory if it still exists + boost::filesystem::remove_all(m_prefix); + } + + void set_prefix(const std::string& prefix) + { + m_prefix = prefix; + } +}; + +using testing::Types; + +typedef Types<BlockchainLMDB, BlockchainBDB> implementations; + +TYPED_TEST_CASE(BlockchainDBTest, implementations); + +TYPED_TEST(BlockchainDBTest, OpenAndClose) +{ + std::string fname(tmpnam(NULL)); + + this->set_prefix(fname); + + // make sure open does not throw + ASSERT_NO_THROW(this->m_db->open(fname)); + this->get_filenames(); + + // make sure open when already open DOES throw + ASSERT_THROW(this->m_db->open(fname), DB_OPEN_FAILURE); + + ASSERT_NO_THROW(this->m_db->close()); +} + +TYPED_TEST(BlockchainDBTest, AddBlock) +{ + std::string fname(tmpnam(NULL)); + this->set_prefix(fname); + + // make sure open does not throw + ASSERT_NO_THROW(this->m_db->open(fname)); + this->get_filenames(); + + // adding a block with no parent in the blockchain should throw. + // note: this shouldn't be possible, but is a good (and cheap) failsafe. + // + // TODO: need at least one more block to make this reasonable, as the + // BlockchainDB implementation should not check for parent if + // no blocks have been added yet (because genesis has no parent). + //ASSERT_THROW(this->m_db->add_block(this->m_blocks[1], t_sizes[1], t_diffs[1], t_coins[1], this->m_txs[1]), BLOCK_PARENT_DNE); + + ASSERT_NO_THROW(this->m_db->add_block(this->m_blocks[0], t_sizes[0], t_diffs[0], t_coins[0], this->m_txs[0])); + ASSERT_NO_THROW(this->m_db->add_block(this->m_blocks[1], t_sizes[1], t_diffs[1], t_coins[1], this->m_txs[1])); + + block b; + ASSERT_TRUE(this->m_db->block_exists(get_block_hash(this->m_blocks[0]))); + ASSERT_NO_THROW(b = this->m_db->get_block(get_block_hash(this->m_blocks[0]))); + + ASSERT_TRUE(compare_blocks(this->m_blocks[0], b)); + + ASSERT_NO_THROW(b = this->m_db->get_block_from_height(0)); + + ASSERT_TRUE(compare_blocks(this->m_blocks[0], b)); + + // assert that we can't add the same block twice + ASSERT_THROW(this->m_db->add_block(this->m_blocks[0], t_sizes[0], t_diffs[0], t_coins[0], this->m_txs[0]), BLOCK_EXISTS); + + for (auto& h : this->m_blocks[0].tx_hashes) + { + transaction tx; + ASSERT_TRUE(this->m_db->tx_exists(h)); + ASSERT_NO_THROW(tx = this->m_db->get_tx(h)); + + ASSERT_HASH_EQ(h, get_transaction_hash(tx)); + } +} + +TYPED_TEST(BlockchainDBTest, RetrieveBlockData) +{ + std::string fname(tmpnam(NULL)); + this->set_prefix(fname); + + // make sure open does not throw + ASSERT_NO_THROW(this->m_db->open(fname)); + this->get_filenames(); + + ASSERT_NO_THROW(this->m_db->add_block(this->m_blocks[0], t_sizes[0], t_diffs[0], t_coins[0], this->m_txs[0])); + + ASSERT_EQ(t_sizes[0], this->m_db->get_block_size(0)); + ASSERT_EQ(t_diffs[0], this->m_db->get_block_cumulative_difficulty(0)); + ASSERT_EQ(t_diffs[0], this->m_db->get_block_difficulty(0)); + ASSERT_EQ(t_coins[0], this->m_db->get_block_already_generated_coins(0)); + + ASSERT_NO_THROW(this->m_db->add_block(this->m_blocks[1], t_sizes[1], t_diffs[1], t_coins[1], this->m_txs[1])); + ASSERT_EQ(t_diffs[1] - t_diffs[0], this->m_db->get_block_difficulty(1)); + + ASSERT_HASH_EQ(get_block_hash(this->m_blocks[0]), this->m_db->get_block_hash_from_height(0)); + + std::vector<block> blks; + ASSERT_NO_THROW(blks = this->m_db->get_blocks_range(0, 1)); + ASSERT_EQ(2, blks.size()); + + ASSERT_HASH_EQ(get_block_hash(this->m_blocks[0]), get_block_hash(blks[0])); + ASSERT_HASH_EQ(get_block_hash(this->m_blocks[1]), get_block_hash(blks[1])); + + std::vector<crypto::hash> hashes; + ASSERT_NO_THROW(hashes = this->m_db->get_hashes_range(0, 1)); + ASSERT_EQ(2, hashes.size()); + + ASSERT_HASH_EQ(get_block_hash(this->m_blocks[0]), hashes[0]); + ASSERT_HASH_EQ(get_block_hash(this->m_blocks[1]), hashes[1]); +} + +} // anonymous namespace diff --git a/tests/unit_tests/CMakeLists.txt b/tests/unit_tests/CMakeLists.txt index 251f6b66e..e28047edb 100644 --- a/tests/unit_tests/CMakeLists.txt +++ b/tests/unit_tests/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # @@ -29,6 +29,7 @@ set(unit_tests_sources address_from_url.cpp base58.cpp + BlockchainDB.cpp block_reward.cpp chacha8.cpp checkpoints.cpp @@ -56,8 +57,10 @@ add_executable(unit_tests target_link_libraries(unit_tests LINK_PRIVATE cryptonote_core + blockchain_db rpc wallet + p2p ${GTEST_MAIN_LIBRARIES} ${Boost_CHRONO_LIBRARY} ${Boost_REGEX_LIBRARY} diff --git a/tests/unit_tests/address_from_url.cpp b/tests/unit_tests/address_from_url.cpp index 22301d568..fe2f072de 100644 --- a/tests/unit_tests/address_from_url.cpp +++ b/tests/unit_tests/address_from_url.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -86,7 +86,7 @@ TEST(AddressFromURL, Success) bool dnssec_result = false; - std::vector<std::string> addresses = tools::wallet2::addresses_from_url("donate.monero.cc", dnssec_result); + std::vector<std::string> addresses = tools::wallet2::addresses_from_url("donate.getmonero.org", dnssec_result); EXPECT_EQ(1, addresses.size()); if (addresses.size() == 1) diff --git a/tests/unit_tests/base58.cpp b/tests/unit_tests/base58.cpp index cdd8c4d68..7e39bd9b5 100644 --- a/tests/unit_tests/base58.cpp +++ b/tests/unit_tests/base58.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/unit_tests/block_reward.cpp b/tests/unit_tests/block_reward.cpp index 7ae088d97..a572d2b75 100644 --- a/tests/unit_tests/block_reward.cpp +++ b/tests/unit_tests/block_reward.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/unit_tests/chacha8.cpp b/tests/unit_tests/chacha8.cpp index bf204ef40..8a706dae4 100644 --- a/tests/unit_tests/chacha8.cpp +++ b/tests/unit_tests/chacha8.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/unit_tests/checkpoints.cpp b/tests/unit_tests/checkpoints.cpp index 3b4172e2e..2f70791c5 100644 --- a/tests/unit_tests/checkpoints.cpp +++ b/tests/unit_tests/checkpoints.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/unit_tests/decompose_amount_into_digits.cpp b/tests/unit_tests/decompose_amount_into_digits.cpp index 903da5909..9b80e58ab 100644 --- a/tests/unit_tests/decompose_amount_into_digits.cpp +++ b/tests/unit_tests/decompose_amount_into_digits.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/unit_tests/dns_resolver.cpp b/tests/unit_tests/dns_resolver.cpp index e944411f3..6717e990a 100644 --- a/tests/unit_tests/dns_resolver.cpp +++ b/tests/unit_tests/dns_resolver.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -43,13 +43,13 @@ TEST(DNSResolver, IPv4Success) ASSERT_EQ(1, ips.size()); - ASSERT_STREQ("93.184.216.119", ips[0].c_str()); + //ASSERT_STREQ("93.184.216.119", ips[0].c_str()); ips = tools::DNSResolver::instance().get_ipv4("example.com", avail, valid); ASSERT_EQ(1, ips.size()); - ASSERT_STREQ("93.184.216.119", ips[0].c_str()); + //ASSERT_STREQ("93.184.216.119", ips[0].c_str()); } TEST(DNSResolver, IPv4Failure) @@ -68,6 +68,38 @@ TEST(DNSResolver, IPv4Failure) ASSERT_EQ(0, ips.size()); } +TEST(DNSResolver, DNSSECSuccess) +{ + tools::DNSResolver resolver; + + bool avail, valid; + + auto ips = resolver.get_ipv4("example.com", avail, valid); + + ASSERT_EQ(1, ips.size()); + + //ASSERT_STREQ("93.184.216.119", ips[0].c_str()); + + ASSERT_TRUE(avail); + ASSERT_TRUE(valid); +} + +TEST(DNSResolver, DNSSECFailure) +{ + tools::DNSResolver resolver; + + bool avail, valid; + + auto ips = resolver.get_ipv4("dnssec-failed.org", avail, valid); + + ASSERT_EQ(1, ips.size()); + + //ASSERT_STREQ("93.184.216.119", ips[0].c_str()); + + ASSERT_TRUE(avail); + ASSERT_FALSE(valid); +} + // It would be great to include an IPv6 test and assume it'll pass, but not every ISP / resolver plays nicely with IPv6;) /*TEST(DNSResolver, IPv6Success) { @@ -108,12 +140,12 @@ TEST(DNSResolver, GetTXTRecord) { bool avail, valid; - std::vector<std::string> records = tools::DNSResolver::instance().get_txt_record("donate.monero.cc", avail, valid); + std::vector<std::string> records = tools::DNSResolver::instance().get_txt_record("donate.getmonero.org", avail, valid); EXPECT_NE(0, records.size()); for (auto& rec : records) { - std::cout << "TXT record for donate.monero.cc: " << rec << std::endl; + std::cout << "TXT record for donate.getmonero.org: " << rec << std::endl; } } diff --git a/tests/unit_tests/epee_boosted_tcp_server.cpp b/tests/unit_tests/epee_boosted_tcp_server.cpp index 42f04b786..bdeaa2bde 100644 --- a/tests/unit_tests/epee_boosted_tcp_server.cpp +++ b/tests/unit_tests/epee_boosted_tcp_server.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -85,7 +85,7 @@ namespace TEST(boosted_tcp_server, worker_threads_are_exception_resistant) { - test_tcp_server srv; + test_tcp_server srv(epee::net_utils::e_connection_type_RPC); // RPC disables network limit for unit tests ASSERT_TRUE(srv.init_server(test_server_port, test_server_host)); std::mutex mtx; diff --git a/tests/unit_tests/epee_levin_protocol_handler_async.cpp b/tests/unit_tests/epee_levin_protocol_handler_async.cpp index a5fc55d9e..096133105 100644 --- a/tests/unit_tests/epee_levin_protocol_handler_async.cpp +++ b/tests/unit_tests/epee_levin_protocol_handler_async.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/unit_tests/get_xtype_from_string.cpp b/tests/unit_tests/get_xtype_from_string.cpp index 8cc9cf7d3..c77a3abbb 100644 --- a/tests/unit_tests/get_xtype_from_string.cpp +++ b/tests/unit_tests/get_xtype_from_string.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/unit_tests/main.cpp b/tests/unit_tests/main.cpp index 33f337122..471895b84 100644 --- a/tests/unit_tests/main.cpp +++ b/tests/unit_tests/main.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // @@ -32,6 +32,8 @@ #include "include_base_utils.h" +unsigned int epee::g_test_dbg_lock_sleep = 0; + int main(int argc, char** argv) { epee::debug::get_set_enable_assert(true, false); diff --git a/tests/unit_tests/mnemonics.cpp b/tests/unit_tests/mnemonics.cpp index d17bb7f85..d2b1edaa9 100644 --- a/tests/unit_tests/mnemonics.cpp +++ b/tests/unit_tests/mnemonics.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/unit_tests/mul_div.cpp b/tests/unit_tests/mul_div.cpp index ae45d1b6b..84652e1c4 100644 --- a/tests/unit_tests/mul_div.cpp +++ b/tests/unit_tests/mul_div.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/unit_tests/parse_amount.cpp b/tests/unit_tests/parse_amount.cpp index 5e677a0b2..a6ef6f3f7 100644 --- a/tests/unit_tests/parse_amount.cpp +++ b/tests/unit_tests/parse_amount.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/unit_tests/serialization.cpp b/tests/unit_tests/serialization.cpp index 4a254b5a4..d4bb0e2fa 100644 --- a/tests/unit_tests/serialization.cpp +++ b/tests/unit_tests/serialization.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/unit_tests/slow_memmem.cpp b/tests/unit_tests/slow_memmem.cpp index 2613209f8..e03e81e79 100644 --- a/tests/unit_tests/slow_memmem.cpp +++ b/tests/unit_tests/slow_memmem.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/unit_tests/test_format_utils.cpp b/tests/unit_tests/test_format_utils.cpp index 9fa09b4e9..438e4719d 100644 --- a/tests/unit_tests/test_format_utils.cpp +++ b/tests/unit_tests/test_format_utils.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/unit_tests/test_peerlist.cpp b/tests/unit_tests/test_peerlist.cpp index 3fb933db5..b89cfb8a2 100644 --- a/tests/unit_tests/test_peerlist.cpp +++ b/tests/unit_tests/test_peerlist.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/unit_tests/test_protocol_pack.cpp b/tests/unit_tests/test_protocol_pack.cpp index ed2bc7e20..e33737da8 100644 --- a/tests/unit_tests/test_protocol_pack.cpp +++ b/tests/unit_tests/test_protocol_pack.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tests/unit_tests/unit_tests_utils.h b/tests/unit_tests/unit_tests_utils.h index c522ef633..84d2c2613 100644 --- a/tests/unit_tests/unit_tests_utils.h +++ b/tests/unit_tests/unit_tests_utils.h @@ -1,4 +1,4 @@ -// Copyright (c) 2014, The Monero Project +// Copyright (c) 2014-2015, The Monero Project // // All rights reserved. // diff --git a/tools/doxygen-publish.sh b/tools/doxygen-publish.sh new file mode 100755 index 000000000..ad02e2834 --- /dev/null +++ b/tools/doxygen-publish.sh @@ -0,0 +1,25 @@ +#!/bin/bash -e + +# maintainer (ask me any questions): rfree + +if [[ ! -r "Doxyfile" ]] ; then + echo "Error, can not read the Doxyfile - make sure to run this script from top of monero project, where the Doxyfile file is located" + exit 1 +fi + +wwwdir="$HOME/monero-www/" +if [[ ! -w "$wwwdir" ]] ; then + echo "Error, can not write into wwwdir=$wwwdir. It should be a directory readable/connected to your webserver, or a symlink to such directory" + exit 1 +fi + +if [[ ! -d "$wwwdir/doc" ]] ; then + echo "Creating subdirs" + mkdir "$wwwdir/doc" +fi + +echo "Generating:" +doxygen Doxyfile && echo "Backup previous version:" && rm -rf ~/monero-www-previous && mv "$wwwdir/doc" ~/monero-www-previous && cp -ar doc/ "$wwwdir/" && echo "Done, builded and copied to public - the doxygen docs" && echo "size:" && du -Dsh "$wwwdir/" && echo "files:" && find "$wwwdir/" | wc -l + + + diff --git a/utils/.gitignore b/utils/.gitignore index 757fee31c..a09c56df5 100644 --- a/utils/.gitignore +++ b/utils/.gitignore @@ -1 +1 @@ -/.idea
\ No newline at end of file +/.idea diff --git a/utils/build_scripts/windows.bat b/utils/build_scripts/windows.bat index a427593be..e61fbc9fe 100644 --- a/utils/build_scripts/windows.bat +++ b/utils/build_scripts/windows.bat @@ -1,4 +1,4 @@ -:: Copyright (c) 2014, The Monero Project +:: Copyright (c) 2014-2015, The Monero Project :: :: All rights reserved. :: diff --git a/utils/gpg_keys/fluffypony.asc b/utils/gpg_keys/fluffypony.asc index 05a1dea64..06ee0ec28 100644 --- a/utils/gpg_keys/fluffypony.asc +++ b/utils/gpg_keys/fluffypony.asc @@ -47,4 +47,4 @@ KXmJCgcWhhqLyuWjl67gR5yQWrnJPrQ7s3sXKLvDsNdCH4Hd9OumV0lcYR5Hr0MT tjLXdWWtTXXcGF3UDnCjsMN4jV6oqbHN3YK4ZrZUNSm/ZZjgoU3Er6P0V5tFmCLa zltmy11aIY4E8FZOzyaPKDGLkQwO7B2QK6J9sMlB6Sk= =V86E ------END PGP PUBLIC KEY BLOCK-----
\ No newline at end of file +-----END PGP PUBLIC KEY BLOCK----- diff --git a/utils/gpg_keys/moneromooo.asc b/utils/gpg_keys/moneromooo.asc new file mode 100644 index 000000000..80cb7ae0b --- /dev/null +++ b/utils/gpg_keys/moneromooo.asc @@ -0,0 +1,52 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v2 + +mQINBFQym34BEADDKCspvziDW0f+T9i6iOewFO9m2XTWKXlQutCPgTkIlZZUrcTR +K+ApsfPxk+PBWgucQDPv/nJVs0CNaSzqewxk7Swjsf8+YjvRmxSSg/NQEgsiBx/s +M0JYlJfCwwFdk9+5b/eAnS3nhcZs/6lFSKk65YCCkj0NSQ6C07gEQyIFagLFKVqW +kAlJoMTURyFKPbcwLj/helG9kUT4I/1DdMx5GhKyL6ogFdLYH5iySAkblLk7I43T +jFE/1OcObKUtkAUgJSirrTJcQ+gtyRUUd/xyU3dE0s9T4h4mG+mm0zRzir+dtjQu +i1OWCjMEe1FItz7We8dWwFlM9G+eWDU7/0mBH8Z3qviPN2ccjJatoFq2t+/tc708 +GcG8uuPs47lvCleXxcH8FidGiITz2gFYAjh5LMSGipNM+1N+Lya8VJQ6B2MdF8hI +XCGtzeiNJSRoklNSTpUo+qQ+bLjETjfogUvq8HpYU+oGi4qaHf+Hkmp4c5Mv3Gfh +5jijXVdmMphe/ZDt4NxBvdSgONIbCwqHwR23vAw5X1/CAZkuQMQTAEMr7OUATfmO +p7gDvxXOGxzq0sqfPTWTBdCj1OPfunHbbeH8ypwBlNpwVG40fJdya+Dqjwu25qX6 +Xh5vxLzeJTBmlawa97MCliPvzzJgW9qHRVCa9lLloGVYLiUOS0N+dZ/r/QARAQAB +tD5tb25lcm9tb29vLW1vbmVybyA8bW9uZXJvbW9vby1tb25lcm9AdXNlcnMubm9y +ZXBseS5naXRodWIuY29tPokCPwQTAQIAKQUCVDKfzgIbAwUJAdqcAAcLCQgHAwIB +BhUIAgkKCwQWAgMBAh4BAheAAAoJEGhvB0VNbO/DUvMQAJrOo9JZKD8gm1TvSE8r +ERwO59CaDk6IwZXhXFncUEJzgOCNNYUc0ulI2Y1R/abJLarrjcKZQ+yxGPsh6gVL +gRckMpRix694tAND85ebl/y2XTcblwp971QJJZm0RltdpXykymo+P2AFw/5KLhcT +XHrzFkrMXKcY+JhyczhLNMg/BMtiaV0gu8G6pq3i2L9B51wBEJaB0qFbowYvXZA0 +mF7Mi8+jlWH8K1szdkff9YtRtxTPWQmJaOB3mV6jWlZM2Cmq5Jk0U9aEbN1I20Iy +k9r9Vinno/Tv2J8QWtqNmBRWTWSQnkLrp9/rN6jTi0h3rkllI5mjm8Vtuxntkd6a +9wKog8ElPLH6zlkm0FztUt5yL9TI/nH9iFUEcxd0QjF4OyshHW1KNSgmHucXrEXg +fggLh1IMgBcmqfZJw2QgcWAyKQ8LKSC7Y0Tv2kPuCW3Z9OA8pi7AbsmCJUyUKqQl ++a6zPW5qMPmPmbX5BiI5jfiIrduQ+4fURDbrWBw78zg6Dy/Z0qA51AnqZGTSyAr1 +ctOYh2Ju0u6Ph/h7Gk1jsYsVTZ9SXG//jOCvLxESuiLD5rsaX5q092CQEuLDpemk +2bjEREygtKa6bckl5Ny0jijaT1/AK9yHbWOIhWpJWDW4rijWuLHdpvZY49il5JcS +rKwuZnvKtNXD6W6DYIRAdQHOuQINBFQym34BEADHtTHduZFdu76RAzqTjT94F92L +xSSopLSk7/sdLWTc2ERmjDId7dKmqrL1Kh2kqAtHY3Rq8Y839LGmbJCzI1kJyOHF +o9jkEI93sqXcztLjizPVukqClOZNt3NV/nvefH6JSdqWcnC4V1mQr2Ztl0j+51i+ +NYVwGjlsOMlBER+LW/s7egRqAQonrcEB5vsSAzd8mOlNKjRAnDCV+C21GDKxzb80 +tx2VS09AQnLLua4PI7xYvCOYOJfN7RwmB1ctgDQp2e+mdwuKuqy81aYLQpQIfVvz +GYQWtA/kKJSzddwpcEU+3+IiHycqgc113KcZAdeZ0m0ShseFFDyuP9rPSiba+hrN +VnxrxhmyfL65PQ4aAzZR5JNrdU1lwsm+wqBveOxuZXeqJiHgSbSiqh1/Dgt9xDd8 +xRFfhoiiPyjjPRmJ+/iG3KXLzEiMfbyTFzGkX3Z9BJTxemUx8JOSVQXa++t4w39J +UwzwBKNItDhtQqJpCaF43fJ4ykLMJi5gRpgqtb+T3CF0abXNII1IfS8a0fSpd48d +6hzoCVqpvWsI1fOY5Ui0BIgubNhkr4OJDCWBT5zhxjCJ3QiUSKyyqjfw1Fpuf/0Y +CSA9Q9FSCq9qTppJs5ITHVjhWw2zxrJEG+P2+dvryBhV9l4T2xx1oHqlKX8zzLBG +kS8NmnxoRFQs5rZYvQARAQABiQIlBBgBAgAPBQJUMpt+AhsMBQkB2pwAAAoJEGhv +B0VNbO/DRi0P/2HGSAVv8rWq8UF6gVp5VLud94pZkpkIR2GQCkqiva9Ysw1bl1ch +9blojO0ufHZ6hMtirSpOd848767UuJDjeJaoNF02rOqVy7E6QwGqpXmB2E1fbxzr +FCd/omA6Jurvp5T2JawpjSsBZYpyxGgDoQ1tC9NjUlcRi1Kebie+p8cQUwZLteXQ +vgdzUMT70fulyExxbmM4x8BSG0/gTGjkznpvuOlfdWLkQCGuyPxFlsVmAOAOIwh5 +jYTRwU51kW+2bSUG4FwUjDtgRnW1gS43207fA1niPgb65AnQLunrNi6aNaTYc/Fj +pfGV5kuLJEVxpGsGPJTirh+1nbOss0vr7pngy1MeH5Wqk1+LBZPRgJ2KNaOcCiNT +8beMCZxOHkLVxJRj318zT5sgT9DWq+OB+m+XGz1wNx0jbMOPrhnwf5tUru7Aqi4/ +3dibY8q68rewT77R7whAood7Zmi98K5SzRnk5F3jp+Ui47TpqQp/SWwpn7gMpDcN +POY/RnsFrhJjUV9DSF0rm2o2RBtV+TVrIrtEOAjof5jcjDJeU7hV7AK6zOYzksJ/ +l5ajwVyde4plP3/R0PJTYFe2tCDXhhCYc79kgIZO5df7mTygeMU20sTMgs3RMWlg +42Yz4PoruQRT3YeUC1Bsgr0ONqQShlM+68NbCe+3uhZm9WbNg6+oqcNt +=itPR +-----END PGP PUBLIC KEY BLOCK----- diff --git a/utils/gpg_keys/tewinget.asc b/utils/gpg_keys/tewinget.asc index 1153d2c23..b22a8ca04 100644 --- a/utils/gpg_keys/tewinget.asc +++ b/utils/gpg_keys/tewinget.asc @@ -16,4 +16,4 @@ OKlmCmujB2IVCf5If44ZoRANoF6MdvlBq5TZv1p2etzz+sJ8cWrcalRlUdA8DcdJ ENfsqndAsm8ID8fYokiCr49jE1QSypWElE65JDsVgX6Yjzz98BYy/9s15YbE5McG Wvg1RzElvwv588XbqxJhEhDZTQW8Dx0z =3FSH ------END PGP PUBLIC KEY BLOCK-----
\ No newline at end of file +-----END PGP PUBLIC KEY BLOCK----- diff --git a/utils/gpg_keys/tomerkon.asc b/utils/gpg_keys/tomerkon.asc index 4e946a6ec..abcf0aeeb 100644 --- a/utils/gpg_keys/tomerkon.asc +++ b/utils/gpg_keys/tomerkon.asc @@ -27,4 +27,4 @@ rWfuA8Dbq31FJ0hdbkezGEF55+jsRwIqmVGHaNvdTm+GyiFibwkc7p6+ynBco8xv zr98VSNx5fM50XBc+pzhAHxzxhIgrmwx5zsQxMmd8pAh4deZUKIyk40XKrwikjoy KLjUdEc73bEjhr1rlVGj6tBVNspHVx5XbUvmDtD1s7E= =EaRm ------END PGP PUBLIC KEY BLOCK-----
\ No newline at end of file +-----END PGP PUBLIC KEY BLOCK----- diff --git a/utils/gpg_keys/warptangent.asc b/utils/gpg_keys/warptangent.asc new file mode 100644 index 000000000..cacd787b6 --- /dev/null +++ b/utils/gpg_keys/warptangent.asc @@ -0,0 +1,52 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1 + +mQINBFSFIj0BEADMJ+82coHXja8ZCQNhs0uw8xTP0Nu3AFIKrCOZIxNFpJvLugOd +B6IKg3q4YNa5RjhOwVRNfzGk4HsYlo+Wg+pw0pGI1WEYJMQ+gvNACKxDGfgxtDL6 +SgugYxUQqPxud5V6GagyV56e47hfdK0WLmYGsIORoX8nksIAvHbL+T9OVpcdP87s +5NLyJBFZrrLR404wunnlr2WTE/3Nmm663sGKI+hb7Xb9G09pr0ba8+f4uYtEycaS +xEBQbeO9Fs/kt02d2oIeqrxulXBHq9LjhXIjiXpeEXUGd6mJmwenclOnreI04esp +TivsiXRcGFnMFAwqTqlrentmzA0ucYuqcSuiId8cEBi2s6KeIAz9we2z8BDeZ1t8 +mT4Xj+GQN9Ki9UiC6a5/6rsXVDXZlURfC9bs0m64IPsEsx/tdr57rKtPD72psgF2 +FQaZpKuBnHu7NQAxydRYd3+5mhqMl0H2OI3Gt7JxdpDFGXjx02gw5rDylwBeiuUc +YkWnkapBEHW8dlvEgYaEOv7lEMg8UR/B0r2EuVDQAQQeFIJ2HitU1xe102RmR74J +OQ2jwLlLip7rA/95v4neVw0z5diVktQ3WtqhwZhQCcednrDhkYnIcsGyBLDi4Pgw +n2H7ppfpzUt/V21bW0nZTkoHC3H1fHN5ar4Jui1dUECbYU5J2RUTxSvE/wARAQAB +tCN3YXJwdGFuZ2VudCA8d2FycHRhbmdlbnRAaW5ib3guY29tPokCOAQTAQIAIgUC +VIUiPQIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQDkkL6/vk6S1c4A/8 +DUIcwgIiUQJIHVfo7TA362lF2nvBOXhE7kiSdk2CeaWEjm7aAaZjwF9oorrMgvkc +hvgg7EzqtNTGt+mqRHszhM4tu8KGrsayxR3i0hDvjcMSUa/45V10ftqGih0f59RD +vGteM7B+qbttHJnRZJNVKgXwUZSLpD7g29W8GcdL3MXmoSQaOmMbWf5CB60hmDYt +Rd8kuHWN9W5Fm+QbYU5WMP1XCt+QmIE8QxFL5m7qRFE9vN/9Elr0VNJo0vMKiz+F +6Kc2985cNE4eTqEwCJm2UBoM86RqGOL6ifMqv6/c2Mb1uZUVVk28wmGb5ZAdLY5p +RKcNjg9O/urwfzaRVdUXZ8wBAGsUPMDyli8dHx4MmCoCqqwGPT/qs9FE+AlKFtiQ +vQ90aeVTw38J6SwOPRw92o/8uIh5EilpaOa5fSKODd0IORxvXyXWF93IaXs13eER +7v9gmvuiMO64lZ7kFqpE0e5FShiJm7PoqhL0mjZik5ThvAC+ZYCiU8CKTOI8iozM +u20yTsCRsSc98Mf6wEZfc6Eht+i9QXxwgkKOHSzPkBLNcxE2z6XgOAMOLCL9NveC +R73fSMQb+hOU7pOfaM7dHYVnrmIOcpS3HrlQJIspdavZdHBrxfQI6GLX652zW4QK +RPDDVnfLwhUDqbjJgw6bncuBhqXqIvcmRgonqRpcj1G5Ag0EVIUiPQEQAKo8YJZ0 +cFW7p8vnsU3cntccyhsh9F9BcREZJbvhUJJMQj1Q4hACzf7u/m9rims2Ez1KRWHb +8gSfA4VUYZGpLgCMxFB8gPV/fC5hiOFQNuVdO9wPtW3LfJ9PMlulyIzod+CeQeqf +Ve0Vwo01NZQtpIU8Y8ROiljhZq3kWS5Rtd96uBcFXKdcWdsH6IIYF1t3xfJLBjKA +jzsAr+5R8oYmIP/uIguQKvIc4OwC7jVdhi0vfBClJNBuRZcWIqfwKXAh8zqMUNFK +ZELbfnNo1zoIXhpsa2ZsSxUHB60oJFrCPvxASjCkSoIvq0lELA+wlSAlJS1I24ZZ +CON4OiReuY8RguY/E+I+WaARgPNui9LN14vgzqeIWTy5f23URjmVIzjWLzVmJI/U +uoIVrcdUDuiEFaCDxVHPropA7d1KanwLnTTgnEPBwegz5LMPtMO0RfleJnFHEYM2 +ZPfsIxWAsz4cgSeRIp5Dl6NMdtIU3HYWtFN+LAIv3sksroNiBCKqhB+L/o5J5+0t +evzAQvm/TrejM4zoBina19uuc5TqMCl3evQryLgx8KQ5gglNftfrNgS3U6//VdTa +rmm6aWNZM81EJr2KzlfNuSANCAdA72ppQjlsBphm+zWFi7SNrooCKmSt7nRpNhxM +xQLnfEvaFyOHAEOiD1aLSL4QEoy/hGRW3xidABEBAAGJAh8EGAECAAkFAlSFIj0C +GwwACgkQDkkL6/vk6S3NOQ//QUJAyEnJRWM7NgWGH1re7l3GQ/tz3ZdJBdEcZqYN +2dHXMBV32umD6bGluEW3COOJ7WCvcdkENDAAaf4Ygt4qnAUDXOH5xKXrBM0NHA+e +AHL31V7Pa9NmcTbw5OPCVLZop8Gkh4sYKE/tZKrP7yUvVu3t5JayOjQny3riIoZP +vBvjzRu5roqKa1/rYUBB9lu0Qi9L84Q1T48Hn04ke39Rweoa/rd35k4I8c7Q4382 +d9426GncSjfgdHF9fR2hoQSvof5BQa0toRIwW2RkUbAtyJ9fWg01XLLActpRON1S +BthtuuV6Qfx4SQs5g0cHMlgcnu1qNKEkBwYWxzPvYJAjAeAmHZfyePyOgOccwAab +ovE8A6ktbw+LeTC4NwYPSILyQYxxxJN+0pBnMfxUE//9luztVk70X6DCbn0t5nsN +H7JBn6XJkWSsObbYZC2Nvfevk9iJh96Ln6nWHkCviwnUVHjetHuBsTGFAIcM6idT +ixqmu+1GohRvrs0OlVeBlR/Lf93YI+YaeBhQAuBOdi8YgpkDhOSEa4jos2zy3N+U +1e7PLIITICZmxCj703TMT6EkYuJa0iZkZE32CF+svF+4IxPGOxfhZSVF572EMPcH +ggZGV0vR7gu7cJRqLqKG3M8UM60tuYFpjT3+6344Akum+YPH8wjo7hbyun5MZdTB +vL0= +=lURU +-----END PGP PUBLIC KEY BLOCK----- diff --git a/utils/munin_plugins/alt_blocks_count b/utils/munin_plugins/alt_blocks_count index a37468b70..06b83c24a 100644 --- a/utils/munin_plugins/alt_blocks_count +++ b/utils/munin_plugins/alt_blocks_count @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # diff --git a/utils/munin_plugins/difficulty b/utils/munin_plugins/difficulty index 6f0e07c3e..0045f2046 100644 --- a/utils/munin_plugins/difficulty +++ b/utils/munin_plugins/difficulty @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # diff --git a/utils/munin_plugins/grey_peerlist_size b/utils/munin_plugins/grey_peerlist_size index 2df7bed06..65191a7fa 100644 --- a/utils/munin_plugins/grey_peerlist_size +++ b/utils/munin_plugins/grey_peerlist_size @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # diff --git a/utils/munin_plugins/height b/utils/munin_plugins/height index dab798c53..76e261e7f 100644 --- a/utils/munin_plugins/height +++ b/utils/munin_plugins/height @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # diff --git a/utils/munin_plugins/incoming_connections_count b/utils/munin_plugins/incoming_connections_count index bcf587fac..16d98edc1 100644 --- a/utils/munin_plugins/incoming_connections_count +++ b/utils/munin_plugins/incoming_connections_count @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # diff --git a/utils/munin_plugins/outgoing_connections_count b/utils/munin_plugins/outgoing_connections_count index e6e971b86..7bae8d508 100644 --- a/utils/munin_plugins/outgoing_connections_count +++ b/utils/munin_plugins/outgoing_connections_count @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # diff --git a/utils/munin_plugins/tx_count b/utils/munin_plugins/tx_count index 54eb2aca5..386c75249 100644 --- a/utils/munin_plugins/tx_count +++ b/utils/munin_plugins/tx_count @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # diff --git a/utils/munin_plugins/tx_pool_size b/utils/munin_plugins/tx_pool_size index 5295fe263..9d7bc6d23 100644 --- a/utils/munin_plugins/tx_pool_size +++ b/utils/munin_plugins/tx_pool_size @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # diff --git a/utils/munin_plugins/white_peerlist_size b/utils/munin_plugins/white_peerlist_size index 8a8f73bf1..e79dfa04c 100644 --- a/utils/munin_plugins/white_peerlist_size +++ b/utils/munin_plugins/white_peerlist_size @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2014, The Monero Project +# Copyright (c) 2014-2015, The Monero Project # # All rights reserved. # diff --git a/utils/test-static-assert.c b/utils/test-static-assert.c index fcac11ce7..3e612bbc7 100644 --- a/utils/test-static-assert.c +++ b/utils/test-static-assert.c @@ -3,4 +3,4 @@ static_assert(1, "FAIL"); int main(int argc, char *argv[]) { return 0; -}
\ No newline at end of file +} |