diff options
Diffstat (limited to 'contrib')
53 files changed, 1548 insertions, 332 deletions
diff --git a/contrib/depends/Makefile b/contrib/depends/Makefile index afa61b93b..ec0e4cfae 100644 --- a/contrib/depends/Makefile +++ b/contrib/depends/Makefile @@ -22,9 +22,16 @@ host_toolchain:=$(HOST)- endif ifneq ($(DEBUG),) -release_type=debug +release_type=Debug else -release_type=release +release_type=Release +endif + +ifneq ($(TESTS),) +build_tests=ON +release_type=Debug +else +build_tests=OFF endif base_build_dir=$(BASEDIR)/work/build @@ -164,6 +171,8 @@ $(host_prefix)/share/toolchain.cmake : toolchain.cmake.in $(host_prefix)/.stamp_ -e 's|@LDFLAGS@|$(strip $(host_LDFLAGS) $(host_$(release_type)_LDFLAGS))|' \ -e 's|@allow_host_packages@|$(ALLOW_HOST_PACKAGES)|' \ -e 's|@debug@|$(DEBUG)|' \ + -e 's|@release_type@|$(release_type)|' \ + -e 's|@build_tests@|$(build_tests)|' \ -e 's|@depends@|$(host_cmake)|' \ -e 's|@prefix@|$($(host_arch)_$(host_os)_prefix)|'\ -e 's|@sdk@|$(SDK_PATH)|'\ @@ -208,4 +217,6 @@ download-win: @$(MAKE) -s HOST=x86_64-w64-mingw32 download-one download: download-osx download-linux download-win + $(foreach package,$(all_packages),$(eval $(call ext_add_stages,$(package)))) + .PHONY: install cached download-one download-osx download-linux download-win download check-packages check-sources diff --git a/contrib/depends/README.md b/contrib/depends/README.md index c9f8b0783..6a15055b4 100644 --- a/contrib/depends/README.md +++ b/contrib/depends/README.md @@ -2,21 +2,29 @@ To build dependencies for the current arch+OS: - make +```bash +make +``` To build for another arch/OS: - make HOST=host-platform-triplet +```bash +make HOST=host-platform-triplet +``` For example: - make HOST=x86_64-w64-mingw32 -j4 +```bash +make HOST=x86_64-w64-mingw32 -j4 +``` A toolchain will be generated that's suitable for plugging into Monero's cmake. In the above example, a dir named x86_64-w64-mingw32 will be created. To use it for Monero: - cmake -DCMAKE_TOOLCHAIN=`pwd`/contrib/depends/x86_64-w64-mingw32 +```bash +cmake -DCMAKE_TOOLCHAIN=`pwd`/contrib/depends/x86_64-w64-mingw32 +``` Common `host-platform-triplets` for cross compilation are: @@ -31,20 +39,24 @@ No other options are needed, the paths are automatically configured. Dependency Options: The following can be set when running make: make FOO=bar - SOURCES_PATH: downloaded sources will be placed here - BASE_CACHE: built packages will be placed here - SDK_PATH: Path where sdk's can be found (used by OSX) - FALLBACK_DOWNLOAD_PATH: If a source file can't be fetched, try here before giving up - DEBUG: disable some optimizations and enable more runtime checking - HOST_ID_SALT: Optional salt to use when generating host package ids - BUILD_ID_SALT: Optional salt to use when generating build package ids +``` +SOURCES_PATH: downloaded sources will be placed here +BASE_CACHE: built packages will be placed here +SDK_PATH: Path where sdk's can be found (used by OSX) +FALLBACK_DOWNLOAD_PATH: If a source file can't be fetched, try here before giving up +DEBUG: disable some optimizations and enable more runtime checking +HOST_ID_SALT: Optional salt to use when generating host package ids +BUILD_ID_SALT: Optional salt to use when generating build package ids +``` Additional targets: - download: run 'make download' to fetch all sources without building them - download-osx: run 'make download-osx' to fetch all sources needed for osx builds - download-win: run 'make download-win' to fetch all sources needed for win builds - download-linux: run 'make download-linux' to fetch all sources needed for linux builds +``` +download: run 'make download' to fetch all sources without building them +download-osx: run 'make download-osx' to fetch all sources needed for osx builds +download-win: run 'make download-win' to fetch all sources needed for win builds +download-linux: run 'make download-linux' to fetch all sources needed for linux builds +``` #Darwin (macos) builds: diff --git a/contrib/depends/funcs.mk b/contrib/depends/funcs.mk index 15e404e42..469144361 100644 --- a/contrib/depends/funcs.mk +++ b/contrib/depends/funcs.mk @@ -213,6 +213,14 @@ $(1): | $($(1)_cached_checksum) endef +stages = fetched extracted preprocessed configured built staged postprocessed cached cached_checksum + +define ext_add_stages +$(foreach stage,$(stages), + $(1)_$(stage): $($(1)_$(stage)) + .PHONY: $(1)_$(stage)) +endef + # These functions create the build targets for each package. They must be # broken down into small steps so that each part is done for all packages # before moving on to the next step. Otherwise, a package's info diff --git a/contrib/depends/packages.md b/contrib/depends/packages.md index 7c8036250..ae5b47327 100644 --- a/contrib/depends/packages.md +++ b/contrib/depends/packages.md @@ -9,39 +9,43 @@ General tips: ## Identifiers Each package is required to define at least these variables: - $(package)_version: - Version of the upstream library or program. If there is no version, a - placeholder such as 1.0 can be used. +``` +$(package)_version: +Version of the upstream library or program. If there is no version, a +placeholder such as 1.0 can be used. - $(package)_download_path: - Location of the upstream source, without the file-name. Usually http or - ftp. +$(package)_download_path: +Location of the upstream source, without the file-name. Usually http or +ftp. - $(package)_file_name: - The upstream source filename available at the download path. +$(package)_file_name: +The upstream source filename available at the download path. - $(package)_sha256_hash: - The sha256 hash of the upstream file +$(package)_sha256_hash: +The sha256 hash of the upstream file +``` These variables are optional: - $(package)_build_subdir: - cd to this dir before running configure/build/stage commands. - - $(package)_download_file: - The file-name of the upstream source if it differs from how it should be - stored locally. This can be used to avoid storing file-names with strange - characters. - - $(package)_dependencies: - Names of any other packages that this one depends on. - - $(package)_patches: - Filenames of any patches needed to build the package - - $(package)_extra_sources: - Any extra files that will be fetched via $(package)_fetch_cmds. These are - specified so that they can be fetched and verified via 'make download'. +``` +$(package)_build_subdir: +cd to this dir before running configure/build/stage commands. + +$(package)_download_file: +The file-name of the upstream source if it differs from how it should be +stored locally. This can be used to avoid storing file-names with strange +characters. + +$(package)_dependencies: +Names of any other packages that this one depends on. + +$(package)_patches: +Filenames of any patches needed to build the package + +$(package)_extra_sources: +Any extra files that will be fetched via $(package)_fetch_cmds. These are +specified so that they can be fetched and verified via 'make download'. +``` ## Build Variables: @@ -49,47 +53,55 @@ After defining the main identifiers, build variables may be added or customized before running the build commands. They should be added to a function called $(package)_set_vars. For example: - define $(package)_set_vars - ... - endef +``` +define $(package)_set_vars +... +endef +``` Most variables can be prefixed with the host, architecture, or both, to make the modifications specific to that case. For example: - Universal: $(package)_cc=gcc - Linux only: $(package)_linux_cc=gcc - x86_64 only: $(package)_x86_64_cc = gcc - x86_64 linux only: $(package)_x86_64_linux_cc = gcc +``` +Universal: $(package)_cc=gcc +Linux only: $(package)_linux_cc=gcc +x86_64 only: $(package)_x86_64_cc = gcc +x86_64 linux only: $(package)_x86_64_linux_cc = gcc +``` These variables may be set to override or append their default values. - $(package)_cc - $(package)_cxx - $(package)_objc - $(package)_objcxx - $(package)_ar - $(package)_ranlib - $(package)_libtool - $(package)_nm - $(package)_cflags - $(package)_cxxflags - $(package)_ldflags - $(package)_cppflags - $(package)_config_env - $(package)_build_env - $(package)_stage_env - $(package)_build_opts - $(package)_config_opts - -The *_env variables are used to add environment variables to the respective +``` +$(package)_cc +$(package)_cxx +$(package)_objc +$(package)_objcxx +$(package)_ar +$(package)_ranlib +$(package)_libtool +$(package)_nm +$(package)_cflags +$(package)_cxxflags +$(package)_ldflags +$(package)_cppflags +$(package)_config_env +$(package)_build_env +$(package)_stage_env +$(package)_build_opts +$(package)_config_opts +``` + +The `*_env` variables are used to add environment variables to the respective commands. Many variables respect a debug/release suffix as well, in order to use them for only the appropriate build config. For example: - $(package)_cflags_release = -O3 - $(package)_cflags_i686_debug = -g - $(package)_config_opts_release = --disable-debug +``` +$(package)_cflags_release = -O3 +$(package)_cflags_i686_debug = -g +$(package)_config_opts_release = --disable-debug +``` These will be used in addition to the options that do not specify debug/release. All builds are considered to be release unless DEBUG=1 is set by @@ -97,51 +109,57 @@ the user. Other variables may be defined as needed. ## Build commands: - For each build, a unique build dir and staging dir are created. For example, - `work/build/mylib/1.0-1adac830f6e` and `work/staging/mylib/1.0-1adac830f6e`. +For each build, a unique build dir and staging dir are created. For example, +`work/build/mylib/1.0-1adac830f6e` and `work/staging/mylib/1.0-1adac830f6e`. + +The following build commands are available for each recipe: - The following build commands are available for each recipe: +``` +$(package)_fetch_cmds: +Runs from: build dir +Fetch the source file. If undefined, it will be fetched and verified +against its hash. - $(package)_fetch_cmds: - Runs from: build dir - Fetch the source file. If undefined, it will be fetched and verified - against its hash. +$(package)_extract_cmds: +Runs from: build dir +Verify the source file against its hash and extract it. If undefined, the +source is assumed to be a tarball. - $(package)_extract_cmds: - Runs from: build dir - Verify the source file against its hash and extract it. If undefined, the - source is assumed to be a tarball. +$(package)_preprocess_cmds: +Runs from: build dir/$(package)_build_subdir +Preprocess the source as necessary. If undefined, does nothing. - $(package)_preprocess_cmds: - Runs from: build dir/$(package)_build_subdir - Preprocess the source as necessary. If undefined, does nothing. +$(package)_config_cmds: +Runs from: build dir/$(package)_build_subdir +Configure the source. If undefined, does nothing. - $(package)_config_cmds: - Runs from: build dir/$(package)_build_subdir - Configure the source. If undefined, does nothing. +$(package)_build_cmds: +Runs from: build dir/$(package)_build_subdir +Build the source. If undefined, does nothing. - $(package)_build_cmds: - Runs from: build dir/$(package)_build_subdir - Build the source. If undefined, does nothing. +$(package)_stage_cmds: +Runs from: build dir/$(package)_build_subdir +Stage the build results. If undefined, does nothing. +``` - $(package)_stage_cmds: - Runs from: build dir/$(package)_build_subdir - Stage the build results. If undefined, does nothing. +The following variables are available for each recipe: - The following variables are available for each recipe: - - $(1)_staging_dir: package's destination sysroot path - $(1)_staging_prefix_dir: prefix path inside of the package's staging dir - $(1)_extract_dir: path to the package's extracted sources - $(1)_build_dir: path where configure/build/stage commands will be run - $(1)_patch_dir: path where the package's patches (if any) are found +``` +$(1)_staging_dir: package's destination sysroot path +$(1)_staging_prefix_dir: prefix path inside of the package's staging dir +$(1)_extract_dir: path to the package's extracted sources +$(1)_build_dir: path where configure/build/stage commands will be run +$(1)_patch_dir: path where the package's patches (if any) are found +``` Notes on build commands: -For packages built with autotools, $($(package)_autoconf) can be used in the +For packages built with autotools, `$($(package)_autoconf)` can be used in the configure step to (usually) correctly configure automatically. Any -$($(package)_config_opts) will be appended. +`$($(package)_config_opts`) will be appended. Most autotools projects can be properly staged using: - $(MAKE) DESTDIR=$($(package)_staging_dir) install +```bash +$(MAKE) DESTDIR=$($(package)_staging_dir) install +``` diff --git a/contrib/depends/packages/eudev.mk b/contrib/depends/packages/eudev.mk index e754c0f20..a7795b777 100644 --- a/contrib/depends/packages/eudev.mk +++ b/contrib/depends/packages/eudev.mk @@ -23,3 +23,7 @@ endef define $(package)_stage_cmds $(MAKE) DESTDIR=$($(package)_staging_dir) install endef + +define $(package)_postprocess_cmds + rm lib/*.la +endef diff --git a/contrib/depends/packages/expat.mk b/contrib/depends/packages/expat.mk index bd2cea1b6..ef81636a2 100644 --- a/contrib/depends/packages/expat.mk +++ b/contrib/depends/packages/expat.mk @@ -6,6 +6,7 @@ $(package)_sha256_hash=03ad85db965f8ab2d27328abcf0bc5571af6ec0a414874b2066ee3fdd define $(package)_set_vars $(package)_config_opts=--enable-static +$(package)_config_opts=--disable-shared $(package)_config_opts+=--prefix=$(host_prefix) endef @@ -20,3 +21,8 @@ endef define $(package)_stage_cmds $(MAKE) DESTDIR=$($(package)_staging_dir) install endef + +define $(package)_postprocess_cmds + rm lib/*.la +endef + diff --git a/contrib/depends/packages/gtest.mk b/contrib/depends/packages/gtest.mk new file mode 100644 index 000000000..5df07a32e --- /dev/null +++ b/contrib/depends/packages/gtest.mk @@ -0,0 +1,38 @@ +package=gtest +$(package)_version=1.8.1 +$(package)_download_path=https://github.com/google/googletest/archive/ +$(package)_file_name=release-$($(package)_version).tar.gz +$(package)_sha256_hash=9bf1fe5182a604b4135edc1a425ae356c9ad15e9b23f9f12a02e80184c3a249c +$(package)_cxxflags=-std=c++11 +$(package)_cxxflags_linux=-fPIC + +define $(package)_config_cmds + cd googletest && \ + CC="$(host_prefix)/native/bin/$($(package)_cc)" \ + CXX="$(host_prefix)/native/bin/$($(package)_cxx)" \ + AR="$(host_prefix)/native/bin/$($(package)_ar)" \ + RANLIB="$(host_prefix)/native/bin/$($(package)_ranlib)" \ + LIBTOOL="$(host_prefix)/native/bin/$($(package)_libtool)" \ + CXXFLAGS="$($(package)_cxxflags)" \ + CCFLAGS="$($(package)_ccflags)" \ + CPPFLAGS="$($(package)_cppflags)" \ + CFLAGS="$($(package)_cflags) $($(package)_cppflags)" \ + LDLAGS="$($(package)_ldflags)" \ + cmake -DCMAKE_INSTALL_PREFIX=$(build_prefix) \ + -DTOOLCHAIN_PREFIX=$(host_toolchain) \ + -DCMAKE_AR="$(host_prefix)/native/bin/$($(package)_ar)" \ + -DCMAKE_RANLIB="$(host_prefix)/native/bin/$($(package)_ranlib)" \ + -DCMAKE_CXX_FLAGS_DEBUG=ON +endef +# -DCMAKE_TOOLCHAIN_FILE=$(HOST)/share/toolchain.cmake + +define $(package)_build_cmds + cd googletest && CC="$(host_prefix)/native/bin/$($(package)_cc)" $(MAKE) +endef + +define $(package)_stage_cmds + mkdir $($(package)_staging_prefix_dir)/lib $($(package)_staging_prefix_dir)/include &&\ + cp googletest/libgtest.a $($(package)_staging_prefix_dir)/lib/ &&\ + cp googletest/libgtest_main.a $($(package)_staging_prefix_dir)/lib/ &&\ + cp -a googletest/include/* $($(package)_staging_prefix_dir)/include/ +endef diff --git a/contrib/depends/packages/hidapi.mk b/contrib/depends/packages/hidapi.mk index 1c43e525a..a27df04fa 100644 --- a/contrib/depends/packages/hidapi.mk +++ b/contrib/depends/packages/hidapi.mk @@ -1,8 +1,8 @@ package=hidapi -$(package)_version=0.8.0-rc1 -$(package)_download_path=https://github.com/signal11/hidapi/archive +$(package)_version=0.9.0 +$(package)_download_path=https://github.com/libusb/hidapi/archive $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=3c147200bf48a04c1e927cd81589c5ddceff61e6dac137a605f6ac9793f4af61 +$(package)_sha256_hash=630ee1834bdd5c5761ab079fd04f463a89585df8fcae51a7bfe4229b1e02a652 $(package)_linux_dependencies=libusb eudev define $(package)_set_vars @@ -28,3 +28,8 @@ endef define $(package)_stage_cmds $(MAKE) DESTDIR=$($(package)_staging_dir) install endef + +define $(package)_postprocess_cmds + rm lib/*.la +endef + diff --git a/contrib/depends/packages/icu4c.mk b/contrib/depends/packages/icu4c.mk index 2b3845488..58ae637b0 100644 --- a/contrib/depends/packages/icu4c.mk +++ b/contrib/depends/packages/icu4c.mk @@ -1,8 +1,8 @@ package=icu4c -$(package)_version=55.1 -$(package)_download_path=https://github.com/TheCharlatan/icu4c/archive -$(package)_file_name=55.1.tar.gz -$(package)_sha256_hash=1f912c54035533fb4268809701d65c7468d00e292efbc31e6444908450cc46ef +$(package)_version=55.2 +$(package)_download_path=https://github.com/unicode-org/icu/releases/download/release-55-2/ +$(package)_file_name=$(package)-55_2-src.tgz +$(package)_sha256_hash=eda2aa9f9c787748a2e2d310590720ca8bcc6252adf6b4cfb03b65bef9d66759 $(package)_patches=icu-001-dont-build-static-dynamic-twice.patch define $(package)_set_vars @@ -21,11 +21,6 @@ define $(package)_config_cmds $(MAKE) $($(package)_build_opts) endef -#define $(package)_build_cmds -# cd source &&\ - $(MAKE) $($((package)_build_opts) `nproc` -#endef - define $(package)_stage_cmds cd buildb &&\ $(MAKE) $($(package)_build_opts) DESTDIR=$($(package)_staging_dir) install lib/* diff --git a/contrib/depends/packages/ldns.mk b/contrib/depends/packages/ldns.mk index 0b7c3806a..ea4902170 100644 --- a/contrib/depends/packages/ldns.mk +++ b/contrib/depends/packages/ldns.mk @@ -6,8 +6,8 @@ $(package)_sha256_hash=8b88e059452118e8949a2752a55ce59bc71fa5bc414103e17f5b6b06f $(package)_dependencies=openssl define $(package)_set_vars - $(package)_config_opts=--disable-shared --enable-static --disable-dane-ta-usage --with-drill - $(package)_config_opts=--with-ssl=$(host_prefix) + $(package)_config_opts=--disable-shared --enable-static --with-drill + $(package)_config_opts+=--with-ssl=$(host_prefix) $(package)_config_opts_release=--disable-debug-mode $(package)_config_opts_linux=--with-pic endef @@ -25,4 +25,6 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds + rm lib/*.la endef + diff --git a/contrib/depends/packages/libiconv.mk b/contrib/depends/packages/libiconv.mk index dbcb28141..d4995c1b7 100644 --- a/contrib/depends/packages/libiconv.mk +++ b/contrib/depends/packages/libiconv.mk @@ -28,3 +28,7 @@ endef define $(package)_stage_cmds $(MAKE) DESTDIR=$($(package)_staging_dir) install endef + +define $(package)_postprocess_cmds + rm lib/*.la +endef diff --git a/contrib/depends/packages/native_cctools.mk b/contrib/depends/packages/native_cctools.mk index 44d238cc4..bcfe1af6b 100644 --- a/contrib/depends/packages/native_cctools.mk +++ b/contrib/depends/packages/native_cctools.mk @@ -52,6 +52,7 @@ endef define $(package)_stage_cmds $(MAKE) DESTDIR=$($(package)_staging_dir) install && \ + cp $($(package)_extract_dir)/cctools/misc/install_name_tool $($(package)_staging_prefix_dir)/bin/ &&\ cd $($(package)_extract_dir)/toolchain && \ mkdir -p $($(package)_staging_prefix_dir)/lib/clang/$($(package)_clang_version)/include && \ mkdir -p $($(package)_staging_prefix_dir)/bin $($(package)_staging_prefix_dir)/include && \ diff --git a/contrib/depends/packages/native_cmake-unused.mk b/contrib/depends/packages/native_cmake-unused.mk new file mode 100644 index 000000000..c9ab75711 --- /dev/null +++ b/contrib/depends/packages/native_cmake-unused.mk @@ -0,0 +1,23 @@ +package=native_cmake +$(package)_version=3.14.0 +$(package)_version_dot=v3.14 +$(package)_download_path=https://cmake.org/files/$($(package)_version_dot)/ +$(package)_file_name=cmake-$($(package)_version).tar.gz +$(package)_sha256_hash=aa76ba67b3c2af1946701f847073f4652af5cbd9f141f221c97af99127e75502 + +define $(package)_set_vars +$(package)_config_opts= +endef + +define $(package)_config_cmds + ./bootstrap &&\ + ./configure $($(package)_config_opts) +endef + +define $(package)_build_cmd + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef diff --git a/contrib/depends/packages/ncurses.mk b/contrib/depends/packages/ncurses.mk new file mode 100644 index 000000000..4e06c00d9 --- /dev/null +++ b/contrib/depends/packages/ncurses.mk @@ -0,0 +1,58 @@ +package=ncurses +$(package)_version=6.1 +$(package)_download_path=https://ftp.gnu.org/gnu/ncurses +$(package)_file_name=$(package)-$($(package)_version).tar.gz +$(package)_sha256_hash=aa057eeeb4a14d470101eff4597d5833dcef5965331be3528c08d99cebaa0d17 + +define $(package)_set_vars + $(package)_build_opts=CC="$($(package)_cc)" + $(package)_config_env=AR="$($(package)_ar)" RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)" + $(package)_config_env_darwin=RANLIB="$(host_prefix)/native/bin/x86_64-apple-darwin11-ranlib" AR="$(host_prefix)/native/bin/x86_64-apple-darwin11-ar" CC="$(host_prefix)/native/bin/$($(package)_cc)" + $(package)_config_opts=--prefix=$(host_prefix) + $(package)_config_opts+=--disable-shared + $(package)_config_opts+=--with-build-cc=gcc + $(package)_config_opts+=--without-debug + $(package)_config_opts+=--without-ada + $(package)_config_opts+=--without-cxx-binding + $(package)_config_opts+=--without-cxx + $(package)_config_opts+=--without-ticlib + $(package)_config_opts+=--without-tic + $(package)_config_opts+=--without-progs + $(package)_config_opts+=--without-tests + $(package)_config_opts+=--without-tack + $(package)_config_opts+=--without-manpages + $(package)_config_opts+=--disable-tic-depends + $(package)_config_opts+=--disable-big-strings + $(package)_config_opts+=--disable-ext-colors + $(package)_config_opts+=--enable-pc-files + $(package)_config_opts+=--host=$(HOST) + $(pacakge)_config_opts+=--without-shared + $(pacakge)_config_opts+=--without-pthread + $(pacakge)_config_opts+=--disable-rpath + $(pacakge)_config_opts+=--disable-colorfgbg + $(pacakge)_config_opts+=--disable-ext-colors + $(pacakge)_config_opts+=--disable-ext-mouse + $(pacakge)_config_opts+=--disable-symlinks + $(pacakge)_config_opts+=--enable-warnings + $(pacakge)_config_opts+=--enable-assertions + $(pacakge)_config_opts+=--disable-home-terminfo + $(pacakge)_config_opts+=--enable-database + $(pacakge)_config_opts+=--enable-sp-funcs + $(pacakge)_config_opts+=--enable-term-driver + $(pacakge)_config_opts+=--enable-interop + $(pacakge)_config_opts+=--enable-widec + $(package)_build_opts=CFLAGS="$($(package)_cflags) $($(package)_cppflags) -fPIC" +endef + +define $(package)_config_cmds + ./configure $($(package)_config_opts) +endef + +define $(package)_build_cmds + $(MAKE) $($(package)_build_opts) V=1 +endef + +define $(package)_stage_cmds + $(MAKE) install DESTDIR=$($(package)_staging_dir) +endef + diff --git a/contrib/depends/packages/openssl.mk b/contrib/depends/packages/openssl.mk index e920b4409..e39dc1d04 100644 --- a/contrib/depends/packages/openssl.mk +++ b/contrib/depends/packages/openssl.mk @@ -1,8 +1,8 @@ package=openssl -$(package)_version=1.0.2q +$(package)_version=1.0.2r $(package)_download_path=https://www.openssl.org/source $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=5744cfcbcec2b1b48629f7354203bc1e5e9b5466998bbccc5b5fcde3b18eb684 +$(package)_sha256_hash=ae51d08bba8a83958e894946f15303ff894d75c2b8bbd44a852b64e3fe11d0d6 define $(package)_set_vars $(package)_config_env=AR="$($(package)_ar)" RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)" diff --git a/contrib/depends/packages/packages.mk b/contrib/depends/packages/packages.mk index 1db50580b..562f4f7d6 100644 --- a/contrib/depends/packages/packages.mk +++ b/contrib/depends/packages/packages.mk @@ -1,16 +1,16 @@ -packages:=boost openssl zeromq cppzmq expat ldns cppzmq readline libiconv hidapi protobuf libusb +packages:=boost openssl zeromq cppzmq expat ldns libiconv hidapi protobuf libusb native_packages := native_ccache native_protobuf darwin_native_packages = native_biplist native_ds_store native_mac_alias -darwin_packages = sodium-darwin +darwin_packages = sodium-darwin ncurses readline -linux_packages = eudev +linux_packages = eudev ncurses readline unwind sodium qt_packages = qt -ifeq ($(host_os),linux) -packages += unwind -packages += sodium +ifeq ($(build_tests),ON) +packages += gtest endif + ifeq ($(host_os),mingw32) packages += icu4c packages += sodium diff --git a/contrib/depends/packages/protobuf.mk b/contrib/depends/packages/protobuf.mk index 54d3fd924..81fa78a3f 100644 --- a/contrib/depends/packages/protobuf.mk +++ b/contrib/depends/packages/protobuf.mk @@ -25,5 +25,7 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - rm lib/libprotoc.a + rm lib/libprotoc.a &&\ + rm lib/*.la endef + diff --git a/contrib/depends/packages/readline.mk b/contrib/depends/packages/readline.mk index afefc7f07..0e2100749 100644 --- a/contrib/depends/packages/readline.mk +++ b/contrib/depends/packages/readline.mk @@ -3,19 +3,19 @@ $(package)_version=8.0 $(package)_download_path=https://ftp.gnu.org/gnu/readline $(package)_file_name=$(package)-$($(package)_version).tar.gz $(package)_sha256_hash=e339f51971478d369f8a053a330a190781acb9864cf4c541060f12078948e461 +$(package)_dependencies=ncurses define $(package)_set_vars - $(package)_build_opts=CC="$($(package)_cc)" - $(package)_config_env=AR="$($(package)_ar)" RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)" $(package)_config_opts=--prefix=$(host_prefix) - $(package)_config_opts+=--disable-shared --enable-multibye --without-purify --without-curses + $(package)_config_opts+=--exec-prefix=$(host_prefix) + $(package)_config_opts+=--host=$(HOST) + $(package)_config_opts+=--disable-shared --with-curses $(package)_config_opts_release=--disable-debug-mode + $(package)_config_opts_darwin+=RANLIB="$(host_prefix)/native/bin/x86_64-apple-darwin11-ranlib" AR="$(host_prefix)/native/bin/x86_64-apple-darwin11-ar" CC="$(host_prefix)/native/bin/$($(package)_cc)" $(package)_build_opts=CFLAGS="$($(package)_cflags) $($(package)_cppflags) -fPIC" endef define $(package)_config_cmds - export bash_cv_have_mbstate_t=yes &&\ - export bash_cv_wcwidth_broken=yes &&\ ./configure $($(package)_config_opts) endef @@ -24,6 +24,6 @@ define $(package)_build_cmds endef define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) install + $(MAKE) install DESTDIR=$($(package)_staging_dir) prefix=$(host_prefix) exec-prefix=$(host_prefix) endef diff --git a/contrib/depends/packages/sodium-darwin.mk b/contrib/depends/packages/sodium-darwin.mk index 8b6ee3f1d..9f11a9426 100644 --- a/contrib/depends/packages/sodium-darwin.mk +++ b/contrib/depends/packages/sodium-darwin.mk @@ -23,3 +23,8 @@ endef define $(package)_stage_cmds $(MAKE) DESTDIR=$($(package)_staging_dir) install endef + +define $(package)_postprocess_cmds + rm lib/*.la +endef + diff --git a/contrib/depends/packages/sodium.mk b/contrib/depends/packages/sodium.mk index 06aa8f874..b71f4383e 100644 --- a/contrib/depends/packages/sodium.mk +++ b/contrib/depends/packages/sodium.mk @@ -23,3 +23,8 @@ endef define $(package)_stage_cmds $(MAKE) DESTDIR=$($(package)_staging_dir) install endef + +define $(package)_postprocess_cmds + rm lib/*.la +endef + diff --git a/contrib/depends/packages/unwind.mk b/contrib/depends/packages/unwind.mk index 543f868a5..fddbd0561 100644 --- a/contrib/depends/packages/unwind.mk +++ b/contrib/depends/packages/unwind.mk @@ -19,4 +19,6 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds + rm lib/*.la endef + diff --git a/contrib/depends/packages/zeromq.mk b/contrib/depends/packages/zeromq.mk index 01146c26f..f17dbeebe 100644 --- a/contrib/depends/packages/zeromq.mk +++ b/contrib/depends/packages/zeromq.mk @@ -30,5 +30,7 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - rm -rf bin share + rm -rf bin share &&\ + rm lib/*.la endef + diff --git a/contrib/depends/toolchain.cmake.in b/contrib/depends/toolchain.cmake.in index b0af7bd6b..ee0407a5e 100644 --- a/contrib/depends/toolchain.cmake.in +++ b/contrib/depends/toolchain.cmake.in @@ -1,9 +1,17 @@ # Set the system name, either Darwin, Linux, or Windows SET(CMAKE_SYSTEM_NAME @depends@) -SET(CMAKE_BUILD_TYPE release) +SET(CMAKE_BUILD_TYPE @release_type@) -SET(STATIC true) -SET(UNBOUND_STATIC true) +OPTION(STATIC "Link libraries statically" ON) +OPTION(TREZOR_DEBUG "Main trezor debugging switch" OFF) +OPTION(BUILD_TESTS "Build tests." OFF) + +SET(STATIC ON) +SET(UNBOUND_STATIC ON) +SET(ARCH "default") + +SET(BUILD_TESTS @build_tests@) +SET(TREZOR_DEBUG @build_tests@) # where is the target environment SET(CMAKE_FIND_ROOT_PATH @prefix@ /usr) @@ -13,6 +21,8 @@ SET(ENV{PKG_CONFIG_PATH} @prefix@/lib/pkgconfig) SET(LRELEASE_PATH @prefix@/native/bin CACHE FILEPATH "path to lrelease" FORCE) SET(Readline_ROOT_DIR @prefix@) +SET(Readline_INCLUDE_DIR @prefix@/include) +SET(Termcap_LIBRARY @prefix@/lib/libncurses.a) SET(LIBUNWIND_INCLUDE_DIR @prefix@/include) SET(LIBUNWIND_LIBRARIES @prefix@/lib/libunwind.a) diff --git a/contrib/epee/include/net/abstract_tcp_server2.h b/contrib/epee/include/net/abstract_tcp_server2.h index 374a28a2e..b38ab5399 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.h +++ b/contrib/epee/include/net/abstract_tcp_server2.h @@ -70,7 +70,7 @@ namespace net_utils struct i_connection_filter { - virtual bool is_remote_host_allowed(const epee::net_utils::network_address &address)=0; + virtual bool is_remote_host_allowed(const epee::net_utils::network_address &address, time_t *t = NULL)=0; protected: virtual ~i_connection_filter(){} }; @@ -227,8 +227,12 @@ namespace net_utils std::map<std::string, t_connection_type> server_type_map; void create_server_type_map(); - bool init_server(uint32_t port, const std::string address = "0.0.0.0", ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect); - bool init_server(const std::string port, const std::string& address = "0.0.0.0", ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect); + bool init_server(uint32_t port, const std::string& address = "0.0.0.0", + uint32_t port_ipv6 = 0, const std::string& address_ipv6 = "::", bool use_ipv6 = false, bool require_ipv4 = true, + ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect); + bool init_server(const std::string port, const std::string& address = "0.0.0.0", + const std::string port_ipv6 = "", const std::string address_ipv6 = "::", bool use_ipv6 = false, bool require_ipv4 = true, + ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect); /// Run the server's io_service loop. bool run_server(size_t threads_count, bool wait = true, const boost::thread::attributes& attrs = boost::thread::attributes()); @@ -269,6 +273,7 @@ namespace net_utils } int get_binded_port(){return m_port;} + int get_binded_port_ipv6(){return m_port_ipv6;} long get_connections_count() const { @@ -339,7 +344,9 @@ namespace net_utils /// Run the server's io_service loop. bool worker_thread(); /// Handle completion of an asynchronous accept operation. - void handle_accept(const boost::system::error_code& e); + void handle_accept_ipv4(const boost::system::error_code& e); + void handle_accept_ipv6(const boost::system::error_code& e); + void handle_accept(const boost::system::error_code& e, bool ipv6 = false); bool is_thread_worker(); @@ -360,11 +367,16 @@ namespace net_utils /// Acceptor used to listen for incoming connections. boost::asio::ip::tcp::acceptor acceptor_; + boost::asio::ip::tcp::acceptor acceptor_ipv6; epee::net_utils::network_address default_remote; std::atomic<bool> m_stop_signal_sent; uint32_t m_port; + uint32_t m_port_ipv6; std::string m_address; + std::string m_address_ipv6; + bool m_use_ipv6; + bool m_require_ipv4; std::string m_thread_name_prefix; //TODO: change to enum server_type, now used size_t m_threads_count; std::vector<boost::shared_ptr<boost::thread> > m_threads; @@ -376,6 +388,8 @@ namespace net_utils /// The next connection to be accepted connection_ptr new_connection_; + connection_ptr new_connection_ipv6; + boost::mutex connections_mutex; std::set<connection_ptr> connections_; diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index 821594355..19e9c9af9 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -54,6 +54,9 @@ #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net" +#define AGGRESSIVE_TIMEOUT_THRESHOLD 120 // sockets +#define NEW_CONNECTION_TIMEOUT_LOCAL 1200000 // 2 minutes +#define NEW_CONNECTION_TIMEOUT_REMOTE 10000 // 10 seconds #define DEFAULT_TIMEOUT_MS_LOCAL 1800000 // 30 minutes #define DEFAULT_TIMEOUT_MS_REMOTE 300000 // 5 minutes #define TIMEOUT_EXTRA_MS_PER_BYTE 0.2 @@ -142,10 +145,18 @@ PRAGMA_WARNING_DISABLE_VS(4355) boost::system::error_code ec; auto remote_ep = socket().remote_endpoint(ec); CHECK_AND_NO_ASSERT_MES(!ec, false, "Failed to get remote endpoint: " << ec.message() << ':' << ec.value()); - CHECK_AND_NO_ASSERT_MES(remote_ep.address().is_v4(), false, "IPv6 not supported here"); + CHECK_AND_NO_ASSERT_MES(remote_ep.address().is_v4() || remote_ep.address().is_v6(), false, "only IPv4 and IPv6 supported here"); - const unsigned long ip_{boost::asio::detail::socket_ops::host_to_network_long(remote_ep.address().to_v4().to_ulong())}; - return start(is_income, is_multithreaded, ipv4_network_address{uint32_t(ip_), remote_ep.port()}); + if (remote_ep.address().is_v4()) + { + const unsigned long ip_ = boost::asio::detail::socket_ops::host_to_network_long(remote_ep.address().to_v4().to_ulong()); + return start(is_income, is_multithreaded, ipv4_network_address{uint32_t(ip_), remote_ep.port()}); + } + else + { + const auto ip_{remote_ep.address().to_v6()}; + return start(is_income, is_multithreaded, ipv6_network_address{ip_, remote_ep.port()}); + } CATCH_ENTRY_L0("connection<t_protocol_handler>::start()", false); } //--------------------------------------------------------------------------------- @@ -189,7 +200,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) m_protocol_handler.after_init_connection(); - reset_timer(get_default_timeout(), false); + reset_timer(boost::posix_time::milliseconds(m_local ? NEW_CONNECTION_TIMEOUT_LOCAL : NEW_CONNECTION_TIMEOUT_REMOTE), false); // first read on the raw socket to detect SSL for the server buffer_ssl_init_fill = 0; @@ -324,12 +335,14 @@ PRAGMA_WARNING_DISABLE_VS(4355) if (!e) { + double current_speed_down; { CRITICAL_REGION_LOCAL(m_throttle_speed_in_mutex); m_throttle_speed_in.handle_trafic_exact(bytes_transferred); - context.m_current_speed_down = m_throttle_speed_in.get_current_speed(); - context.m_max_speed_down = std::max(context.m_max_speed_down, context.m_current_speed_down); + current_speed_down = m_throttle_speed_in.get_current_speed(); } + context.m_current_speed_down = current_speed_down; + context.m_max_speed_down = std::max(context.m_max_speed_down, current_speed_down); { CRITICAL_REGION_LOCAL( epee::net_utils::network_throttle_manager::network_throttle_manager::m_lock_get_global_throttle_in ); @@ -599,12 +612,14 @@ PRAGMA_WARNING_DISABLE_VS(4355) return false; if(m_was_shutdown) return false; + double current_speed_up; { CRITICAL_REGION_LOCAL(m_throttle_speed_out_mutex); m_throttle_speed_out.handle_trafic_exact(cb); - context.m_current_speed_up = m_throttle_speed_out.get_current_speed(); - context.m_max_speed_up = std::max(context.m_max_speed_up, context.m_current_speed_up); + current_speed_up = m_throttle_speed_out.get_current_speed(); } + context.m_current_speed_up = current_speed_up; + context.m_max_speed_up = std::max(context.m_max_speed_up, current_speed_up); //_info("[sock " << socket().native_handle() << "] SEND " << cb); context.m_last_send = time(NULL); @@ -691,7 +706,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) { unsigned count; try { count = host_count(m_host); } catch (...) { count = 0; } - const unsigned shift = std::min(std::max(count, 1u) - 1, 8u); + const unsigned shift = get_state().sock_count > AGGRESSIVE_TIMEOUT_THRESHOLD ? std::min(std::max(count, 1u) - 1, 8u) : 0; boost::posix_time::milliseconds timeout(0); if (m_local) timeout = boost::posix_time::milliseconds(DEFAULT_TIMEOUT_MS_LOCAL >> shift); @@ -730,8 +745,6 @@ PRAGMA_WARNING_DISABLE_VS(4355) template<class t_protocol_handler> void connection<t_protocol_handler>::reset_timer(boost::posix_time::milliseconds ms, bool add) { - if (m_connection_type != e_connection_type_RPC) - return; MTRACE("Setting " << ms << " expiry"); auto self = safe_shared_from_this(); if(!self) @@ -899,12 +912,14 @@ PRAGMA_WARNING_DISABLE_VS(4355) m_io_service_local_instance(new worker()), io_service_(m_io_service_local_instance->io_service), acceptor_(io_service_), + acceptor_ipv6(io_service_), default_remote(), m_stop_signal_sent(false), m_port(0), m_threads_count(0), m_thread_index(0), m_connection_type( connection_type ), - new_connection_() + new_connection_(), + new_connection_ipv6() { create_server_type_map(); m_thread_name_prefix = "NET"; @@ -915,12 +930,14 @@ PRAGMA_WARNING_DISABLE_VS(4355) m_state(boost::make_shared<typename connection<t_protocol_handler>::shared_state>()), io_service_(extarnal_io_service), acceptor_(io_service_), + acceptor_ipv6(io_service_), default_remote(), m_stop_signal_sent(false), m_port(0), m_threads_count(0), m_thread_index(0), m_connection_type(connection_type), - new_connection_() + new_connection_(), + new_connection_ipv6() { create_server_type_map(); m_thread_name_prefix = "NET"; @@ -942,29 +959,92 @@ PRAGMA_WARNING_DISABLE_VS(4355) } //--------------------------------------------------------------------------------- template<class t_protocol_handler> - bool boosted_tcp_server<t_protocol_handler>::init_server(uint32_t port, const std::string address, ssl_options_t ssl_options) + bool boosted_tcp_server<t_protocol_handler>::init_server(uint32_t port, const std::string& address, + uint32_t port_ipv6, const std::string& address_ipv6, bool use_ipv6, bool require_ipv4, + ssl_options_t ssl_options) { TRY_ENTRY(); m_stop_signal_sent = false; m_port = port; + m_port_ipv6 = port_ipv6; m_address = address; + m_address_ipv6 = address_ipv6; + m_use_ipv6 = use_ipv6; + m_require_ipv4 = require_ipv4; + if (ssl_options) m_state->configure_ssl(std::move(ssl_options)); - // Open the acceptor with the option to reuse the address (i.e. SO_REUSEADDR). - boost::asio::ip::tcp::resolver resolver(io_service_); - boost::asio::ip::tcp::resolver::query query(address, boost::lexical_cast<std::string>(port), boost::asio::ip::tcp::resolver::query::canonical_name); - boost::asio::ip::tcp::endpoint endpoint = *resolver.resolve(query); - acceptor_.open(endpoint.protocol()); - acceptor_.set_option(boost::asio::ip::tcp::acceptor::reuse_address(true)); - acceptor_.bind(endpoint); - acceptor_.listen(); - boost::asio::ip::tcp::endpoint binded_endpoint = acceptor_.local_endpoint(); - m_port = binded_endpoint.port(); - MDEBUG("start accept"); - new_connection_.reset(new connection<t_protocol_handler>(io_service_, m_state, m_connection_type, m_state->ssl_options().support)); - acceptor_.async_accept(new_connection_->socket(), - boost::bind(&boosted_tcp_server<t_protocol_handler>::handle_accept, this, - boost::asio::placeholders::error)); + + std::string ipv4_failed = ""; + std::string ipv6_failed = ""; + try + { + boost::asio::ip::tcp::resolver resolver(io_service_); + boost::asio::ip::tcp::resolver::query query(address, boost::lexical_cast<std::string>(port), boost::asio::ip::tcp::resolver::query::canonical_name); + boost::asio::ip::tcp::endpoint endpoint = *resolver.resolve(query); + acceptor_.open(endpoint.protocol()); + // Open the acceptor with the option to reuse the address (i.e. SO_REUSEADDR). + acceptor_.set_option(boost::asio::ip::tcp::acceptor::reuse_address(true)); + acceptor_.bind(endpoint); + acceptor_.listen(); + boost::asio::ip::tcp::endpoint binded_endpoint = acceptor_.local_endpoint(); + m_port = binded_endpoint.port(); + MDEBUG("start accept (IPv4)"); + new_connection_.reset(new connection<t_protocol_handler>(io_service_, m_state, m_connection_type, m_state->ssl_options().support)); + acceptor_.async_accept(new_connection_->socket(), + boost::bind(&boosted_tcp_server<t_protocol_handler>::handle_accept_ipv4, this, + boost::asio::placeholders::error)); + } + catch (const std::exception &e) + { + ipv4_failed = e.what(); + } + + if (ipv4_failed != "") + { + MERROR("Failed to bind IPv4: " << ipv4_failed); + if (require_ipv4) + { + throw std::runtime_error("Failed to bind IPv4 (set to required)"); + } + } + + if (use_ipv6) + { + try + { + if (port_ipv6 == 0) port_ipv6 = port; // default arg means bind to same port as ipv4 + boost::asio::ip::tcp::resolver resolver(io_service_); + boost::asio::ip::tcp::resolver::query query(address_ipv6, boost::lexical_cast<std::string>(port_ipv6), boost::asio::ip::tcp::resolver::query::canonical_name); + boost::asio::ip::tcp::endpoint endpoint = *resolver.resolve(query); + acceptor_ipv6.open(endpoint.protocol()); + // Open the acceptor with the option to reuse the address (i.e. SO_REUSEADDR). + acceptor_ipv6.set_option(boost::asio::ip::tcp::acceptor::reuse_address(true)); + acceptor_ipv6.set_option(boost::asio::ip::v6_only(true)); + acceptor_ipv6.bind(endpoint); + acceptor_ipv6.listen(); + boost::asio::ip::tcp::endpoint binded_endpoint = acceptor_ipv6.local_endpoint(); + m_port_ipv6 = binded_endpoint.port(); + MDEBUG("start accept (IPv6)"); + new_connection_ipv6.reset(new connection<t_protocol_handler>(io_service_, m_state, m_connection_type, m_state->ssl_options().support)); + acceptor_ipv6.async_accept(new_connection_ipv6->socket(), + boost::bind(&boosted_tcp_server<t_protocol_handler>::handle_accept_ipv6, this, + boost::asio::placeholders::error)); + } + catch (const std::exception &e) + { + ipv6_failed = e.what(); + } + } + + if (use_ipv6 && ipv6_failed != "") + { + MERROR("Failed to bind IPv6: " << ipv6_failed); + if (ipv4_failed != "") + { + throw std::runtime_error("Failed to bind IPv4 and IPv6"); + } + } return true; } @@ -983,15 +1063,23 @@ PRAGMA_WARNING_DISABLE_VS(4355) PUSH_WARNINGS DISABLE_GCC_WARNING(maybe-uninitialized) template<class t_protocol_handler> - bool boosted_tcp_server<t_protocol_handler>::init_server(const std::string port, const std::string& address, ssl_options_t ssl_options) + bool boosted_tcp_server<t_protocol_handler>::init_server(const std::string port, const std::string& address, + const std::string port_ipv6, const std::string address_ipv6, bool use_ipv6, bool require_ipv4, + ssl_options_t ssl_options) { uint32_t p = 0; + uint32_t p_ipv6 = 0; if (port.size() && !string_tools::get_xtype_from_string(p, port)) { MERROR("Failed to convert port no = " << port); return false; } - return this->init_server(p, address, std::move(ssl_options)); + + if (port_ipv6.size() && !string_tools::get_xtype_from_string(p_ipv6, port_ipv6)) { + MERROR("Failed to convert port no = " << port_ipv6); + return false; + } + return this->init_server(p, address, p_ipv6, address_ipv6, use_ipv6, require_ipv4, std::move(ssl_options)); } POP_WARNINGS //--------------------------------------------------------------------------------- @@ -1083,7 +1171,7 @@ POP_WARNINGS { //some problems with the listening socket ?.. _dbg1("Net service stopped without stop request, restarting..."); - if(!this->init_server(m_port, m_address)) + if(!this->init_server(m_port, m_address, m_port_ipv6, m_address_ipv6, m_use_ipv6, m_require_ipv4)) { _dbg1("Reiniting service failed, exit."); return false; @@ -1149,29 +1237,52 @@ POP_WARNINGS } //--------------------------------------------------------------------------------- template<class t_protocol_handler> - void boosted_tcp_server<t_protocol_handler>::handle_accept(const boost::system::error_code& e) + void boosted_tcp_server<t_protocol_handler>::handle_accept_ipv4(const boost::system::error_code& e) + { + this->handle_accept(e, false); + } + //--------------------------------------------------------------------------------- + template<class t_protocol_handler> + void boosted_tcp_server<t_protocol_handler>::handle_accept_ipv6(const boost::system::error_code& e) + { + this->handle_accept(e, true); + } + //--------------------------------------------------------------------------------- + template<class t_protocol_handler> + void boosted_tcp_server<t_protocol_handler>::handle_accept(const boost::system::error_code& e, bool ipv6) { MDEBUG("handle_accept"); + + boost::asio::ip::tcp::acceptor* current_acceptor = &acceptor_; + connection_ptr* current_new_connection = &new_connection_; + auto accept_function_pointer = &boosted_tcp_server<t_protocol_handler>::handle_accept_ipv4; + if (ipv6) + { + current_acceptor = &acceptor_ipv6; + current_new_connection = &new_connection_ipv6; + accept_function_pointer = &boosted_tcp_server<t_protocol_handler>::handle_accept_ipv6; + } + try { if (!e) { - if (m_connection_type == e_connection_type_RPC) { - const char *ssl_message = "unknown"; - switch (new_connection_->get_ssl_support()) - { - case epee::net_utils::ssl_support_t::e_ssl_support_disabled: ssl_message = "disabled"; break; - case epee::net_utils::ssl_support_t::e_ssl_support_enabled: ssl_message = "enabled"; break; - case epee::net_utils::ssl_support_t::e_ssl_support_autodetect: ssl_message = "autodetection"; break; - } - MDEBUG("New server for RPC connections, SSL " << ssl_message); - new_connection_->setRpcStation(); // hopefully this is not needed actually - } - connection_ptr conn(std::move(new_connection_)); - new_connection_.reset(new connection<t_protocol_handler>(io_service_, m_state, m_connection_type, conn->get_ssl_support())); - acceptor_.async_accept(new_connection_->socket(), - boost::bind(&boosted_tcp_server<t_protocol_handler>::handle_accept, this, - boost::asio::placeholders::error)); + if (m_connection_type == e_connection_type_RPC) { + const char *ssl_message = "unknown"; + switch ((*current_new_connection)->get_ssl_support()) + { + case epee::net_utils::ssl_support_t::e_ssl_support_disabled: ssl_message = "disabled"; break; + case epee::net_utils::ssl_support_t::e_ssl_support_enabled: ssl_message = "enabled"; break; + case epee::net_utils::ssl_support_t::e_ssl_support_autodetect: ssl_message = "autodetection"; break; + } + MDEBUG("New server for RPC connections, SSL " << ssl_message); + (*current_new_connection)->setRpcStation(); // hopefully this is not needed actually + } + connection_ptr conn(std::move((*current_new_connection))); + (*current_new_connection).reset(new connection<t_protocol_handler>(io_service_, m_state, m_connection_type, conn->get_ssl_support())); + current_acceptor->async_accept((*current_new_connection)->socket(), + boost::bind(accept_function_pointer, this, + boost::asio::placeholders::error)); boost::asio::socket_base::keep_alive opt(true); conn->socket().set_option(opt); @@ -1203,10 +1314,10 @@ POP_WARNINGS assert(m_state != nullptr); // always set in constructor _erro("Some problems at accept: " << e.message() << ", connections_count = " << m_state->sock_count); misc_utils::sleep_no_w(100); - new_connection_.reset(new connection<t_protocol_handler>(io_service_, m_state, m_connection_type, new_connection_->get_ssl_support())); - acceptor_.async_accept(new_connection_->socket(), - boost::bind(&boosted_tcp_server<t_protocol_handler>::handle_accept, this, - boost::asio::placeholders::error)); + (*current_new_connection).reset(new connection<t_protocol_handler>(io_service_, m_state, m_connection_type, (*current_new_connection)->get_ssl_support())); + current_acceptor->async_accept((*current_new_connection)->socket(), + boost::bind(accept_function_pointer, this, + boost::asio::placeholders::error)); } //--------------------------------------------------------------------------------- template<class t_protocol_handler> @@ -1340,23 +1451,84 @@ POP_WARNINGS epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ CRITICAL_REGION_LOCAL(connections_mutex); connections_.erase(new_connection_l); }); boost::asio::ip::tcp::socket& sock_ = new_connection_l->socket(); - ////////////////////////////////////////////////////////////////////////// + bool try_ipv6 = false; + boost::asio::ip::tcp::resolver resolver(io_service_); boost::asio::ip::tcp::resolver::query query(boost::asio::ip::tcp::v4(), adr, port, boost::asio::ip::tcp::resolver::query::canonical_name); - boost::asio::ip::tcp::resolver::iterator iterator = resolver.resolve(query); + boost::system::error_code resolve_error; + boost::asio::ip::tcp::resolver::iterator iterator; + try + { + //resolving ipv4 address as ipv6 throws, catch here and move on + iterator = resolver.resolve(query, resolve_error); + } + catch (const boost::system::system_error& e) + { + if (!m_use_ipv6 || (resolve_error != boost::asio::error::host_not_found && + resolve_error != boost::asio::error::host_not_found_try_again)) + { + throw; + } + try_ipv6 = true; + } + catch (...) + { + throw; + } + + std::string bind_ip_to_use; + boost::asio::ip::tcp::resolver::iterator end; if(iterator == end) { - _erro("Failed to resolve " << adr); - return false; + if (!m_use_ipv6) + { + _erro("Failed to resolve " << adr); + return false; + } + else + { + try_ipv6 = true; + MINFO("Resolving address as IPv4 failed, trying IPv6"); + } + } + else + { + bind_ip_to_use = bind_ip; + } + + if (try_ipv6) + { + boost::asio::ip::tcp::resolver::query query6(boost::asio::ip::tcp::v6(), adr, port, boost::asio::ip::tcp::resolver::query::canonical_name); + + iterator = resolver.resolve(query6, resolve_error); + + if(iterator == end) + { + _erro("Failed to resolve " << adr); + return false; + } + else + { + if (bind_ip == "0.0.0.0") + { + bind_ip_to_use = "::"; + } + else + { + bind_ip_to_use = ""; + } + + } + } - ////////////////////////////////////////////////////////////////////////// + LOG_ERROR("Trying connect to " << adr << ":" << port << ", bind_ip = " << bind_ip_to_use); //boost::asio::ip::tcp::endpoint remote_endpoint(boost::asio::ip::address::from_string(addr.c_str()), port); boost::asio::ip::tcp::endpoint remote_endpoint(*iterator); - auto try_connect_result = try_connect(new_connection_l, adr, port, sock_, remote_endpoint, bind_ip, conn_timeout, ssl_support); + auto try_connect_result = try_connect(new_connection_l, adr, port, sock_, remote_endpoint, bind_ip_to_use, conn_timeout, ssl_support); if (try_connect_result == CONNECT_FAILURE) return false; if (ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect && try_connect_result == CONNECT_NO_SSL) @@ -1364,7 +1536,7 @@ POP_WARNINGS // we connected, but could not connect with SSL, try without MERROR("SSL handshake failed on an autodetect connection, reconnecting without SSL"); new_connection_l->disable_ssl(); - try_connect_result = try_connect(new_connection_l, adr, port, sock_, remote_endpoint, bind_ip, conn_timeout, epee::net_utils::ssl_support_t::e_ssl_support_disabled); + try_connect_result = try_connect(new_connection_l, adr, port, sock_, remote_endpoint, bind_ip_to_use, conn_timeout, epee::net_utils::ssl_support_t::e_ssl_support_disabled); if (try_connect_result != CONNECT_SUCCESS) return false; } @@ -1404,17 +1576,59 @@ POP_WARNINGS epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ CRITICAL_REGION_LOCAL(connections_mutex); connections_.erase(new_connection_l); }); boost::asio::ip::tcp::socket& sock_ = new_connection_l->socket(); - ////////////////////////////////////////////////////////////////////////// + bool try_ipv6 = false; + boost::asio::ip::tcp::resolver resolver(io_service_); boost::asio::ip::tcp::resolver::query query(boost::asio::ip::tcp::v4(), adr, port, boost::asio::ip::tcp::resolver::query::canonical_name); - boost::asio::ip::tcp::resolver::iterator iterator = resolver.resolve(query); + boost::system::error_code resolve_error; + boost::asio::ip::tcp::resolver::iterator iterator; + try + { + //resolving ipv4 address as ipv6 throws, catch here and move on + iterator = resolver.resolve(query, resolve_error); + } + catch (const boost::system::system_error& e) + { + if (!m_use_ipv6 || (resolve_error != boost::asio::error::host_not_found && + resolve_error != boost::asio::error::host_not_found_try_again)) + { + throw; + } + try_ipv6 = true; + } + catch (...) + { + throw; + } + boost::asio::ip::tcp::resolver::iterator end; if(iterator == end) { - _erro("Failed to resolve " << adr); - return false; + if (!try_ipv6) + { + _erro("Failed to resolve " << adr); + return false; + } + else + { + MINFO("Resolving address as IPv4 failed, trying IPv6"); + } } - ////////////////////////////////////////////////////////////////////////// + + if (try_ipv6) + { + boost::asio::ip::tcp::resolver::query query6(boost::asio::ip::tcp::v6(), adr, port, boost::asio::ip::tcp::resolver::query::canonical_name); + + iterator = resolver.resolve(query6, resolve_error); + + if(iterator == end) + { + _erro("Failed to resolve " << adr); + return false; + } + } + + boost::asio::ip::tcp::endpoint remote_endpoint(*iterator); sock_.open(remote_endpoint.protocol()); diff --git a/contrib/epee/include/net/connection_basic.hpp b/contrib/epee/include/net/connection_basic.hpp index d3f5f4f24..2acc6cdda 100644 --- a/contrib/epee/include/net/connection_basic.hpp +++ b/contrib/epee/include/net/connection_basic.hpp @@ -186,8 +186,6 @@ class connection_basic { // not-templated base class for rapid developmet of som void sleep_before_packet(size_t packet_size, int phase, int q_len); // execute a sleep ; phase is not really used now(?) static void save_limit_to_file(int limit); ///< for dr-monero static double get_sleep_time(size_t cb); - - static void set_save_graph(bool save_graph); }; } // nameserver diff --git a/contrib/epee/include/net/http_protocol_handler.inl b/contrib/epee/include/net/http_protocol_handler.inl index ae8e43477..790d0f3b1 100644 --- a/contrib/epee/include/net/http_protocol_handler.inl +++ b/contrib/epee/include/net/http_protocol_handler.inl @@ -577,6 +577,10 @@ namespace net_utils if (query_info.m_http_method != http::http_method_options) { res = handle_request(query_info, response); + if (response.m_response_code == 500) + { + m_want_close = true; // close on all "Internal server error"s + } } else { diff --git a/contrib/epee/include/net/http_server_handlers_map2.h b/contrib/epee/include/net/http_server_handlers_map2.h index 4b2053091..07ed8157b 100644 --- a/contrib/epee/include/net/http_server_handlers_map2.h +++ b/contrib/epee/include/net/http_server_handlers_map2.h @@ -71,7 +71,7 @@ MINFO(m_conn_context << "calling " << s_pattern); \ if(!callback_f(static_cast<command_type::request&>(req), static_cast<command_type::response&>(resp), &m_conn_context)) \ { \ - LOG_ERROR("Failed to " << #callback_f << "()"); \ + MERROR(m_conn_context << "Failed to " << #callback_f << "()"); \ response_info.m_response_code = 500; \ response_info.m_response_comment = "Internal Server Error"; \ return true; \ @@ -99,7 +99,7 @@ MINFO(m_conn_context << "calling " << s_pattern); \ if(!callback_f(static_cast<command_type::request&>(req), static_cast<command_type::response&>(resp), &m_conn_context)) \ { \ - LOG_ERROR("Failed to " << #callback_f << "()"); \ + MERROR(m_conn_context << "Failed to " << #callback_f << "()"); \ response_info.m_response_code = 500; \ response_info.m_response_comment = "Internal Server Error"; \ return true; \ diff --git a/contrib/epee/include/net/http_server_impl_base.h b/contrib/epee/include/net/http_server_impl_base.h index fc2dcbf67..6cd19f17b 100644 --- a/contrib/epee/include/net/http_server_impl_base.h +++ b/contrib/epee/include/net/http_server_impl_base.h @@ -57,6 +57,7 @@ namespace epee {} bool init(std::function<void(size_t, uint8_t*)> rng, const std::string& bind_port = "0", const std::string& bind_ip = "0.0.0.0", + const std::string& bind_ipv6_address = "::", bool use_ipv6 = false, bool require_ipv4 = true, std::vector<std::string> access_control_origins = std::vector<std::string>(), boost::optional<net_utils::http::login> user = boost::none, net_utils::ssl_options_t ssl_options = net_utils::ssl_support_t::e_ssl_support_autodetect) @@ -75,8 +76,12 @@ namespace epee m_net_server.get_config_object().m_user = std::move(user); - MGINFO("Binding on " << bind_ip << ":" << bind_port); - bool res = m_net_server.init_server(bind_port, bind_ip, std::move(ssl_options)); + MGINFO("Binding on " << bind_ip << " (IPv4):" << bind_port); + if (use_ipv6) + { + MGINFO("Binding on " << bind_ipv6_address << " (IPv6):" << bind_port); + } + bool res = m_net_server.init_server(bind_port, bind_ip, bind_port, bind_ipv6_address, use_ipv6, require_ipv4, std::move(ssl_options)); if(!res) { LOG_ERROR("Failed to bind server"); diff --git a/contrib/epee/include/net/levin_protocol_handler_async.h b/contrib/epee/include/net/levin_protocol_handler_async.h index 116b3ace1..8d7ffb2c2 100644 --- a/contrib/epee/include/net/levin_protocol_handler_async.h +++ b/contrib/epee/include/net/levin_protocol_handler_async.h @@ -99,6 +99,8 @@ public: template<class callback_t> bool for_connection(const boost::uuids::uuid &connection_id, const callback_t &cb); size_t get_connections_count(); + size_t get_out_connections_count(); + size_t get_in_connections_count(); void set_handler(levin_commands_handler<t_connection_context>* handler, void (*destroy)(levin_commands_handler<t_connection_context>*) = NULL); async_protocol_handler_config():m_pcommands_handler(NULL), m_pcommands_handler_destroy(NULL), m_max_packet_size(LEVIN_DEFAULT_MAX_PACKET_SIZE), m_invoke_timeout(LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED) @@ -882,6 +884,28 @@ size_t async_protocol_handler_config<t_connection_context>::get_connections_coun } //------------------------------------------------------------------------------------------ template<class t_connection_context> +size_t async_protocol_handler_config<t_connection_context>::get_out_connections_count() +{ + CRITICAL_REGION_LOCAL(m_connects_lock); + size_t count = 0; + for (const auto &c: m_connects) + if (!c.second->m_connection_context.m_is_income) + ++count; + return count; +} +//------------------------------------------------------------------------------------------ +template<class t_connection_context> +size_t async_protocol_handler_config<t_connection_context>::get_in_connections_count() +{ + CRITICAL_REGION_LOCAL(m_connects_lock); + size_t count = 0; + for (const auto &c: m_connects) + if (c.second->m_connection_context.m_is_income) + ++count; + return count; +} +//------------------------------------------------------------------------------------------ +template<class t_connection_context> void async_protocol_handler_config<t_connection_context>::set_handler(levin_commands_handler<t_connection_context>* handler, void (*destroy)(levin_commands_handler<t_connection_context>*)) { if (m_pcommands_handler && m_pcommands_handler_destroy) diff --git a/contrib/epee/include/net/local_ip.h b/contrib/epee/include/net/local_ip.h index 52c5855b9..7523f9d81 100644 --- a/contrib/epee/include/net/local_ip.h +++ b/contrib/epee/include/net/local_ip.h @@ -27,10 +27,38 @@ #pragma once +#include <string> +#include <boost/algorithm/string/predicate.hpp> +#include <boost/asio/ip/address_v6.hpp> + namespace epee { namespace net_utils { + + inline + bool is_ipv6_local(const std::string& ip) + { + auto addr = boost::asio::ip::make_address_v6(ip); + + // ipv6 link-local unicast addresses are fe80::/10 + bool is_link_local = addr.is_link_local(); + + auto addr_bytes = addr.to_bytes(); + + // ipv6 unique local unicast addresses start with fc00::/7 -- (fcXX or fdXX) + bool is_unique_local_unicast = (addr_bytes[0] == 0xfc || addr_bytes[0] == 0xfd); + + return is_link_local || is_unique_local_unicast; + } + + inline + bool is_ipv6_loopback(const std::string& ip) + { + // ipv6 loopback is ::1 + return boost::asio::ip::address_v6::from_string(ip).is_loopback(); + } + inline bool is_ip_local(uint32_t ip) { diff --git a/contrib/epee/include/net/net_fwd.h b/contrib/epee/include/net/net_fwd.h new file mode 100644 index 000000000..ba4fe6259 --- /dev/null +++ b/contrib/epee/include/net/net_fwd.h @@ -0,0 +1,38 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +namespace epee +{ + namespace net_utils + { + struct ssl_authentication_t; + class ssl_options_t; + } +} diff --git a/contrib/epee/include/net/net_helper.h b/contrib/epee/include/net/net_helper.h index 6387c4c34..e315555fc 100644 --- a/contrib/epee/include/net/net_helper.h +++ b/contrib/epee/include/net/net_helper.h @@ -193,7 +193,6 @@ namespace net_utils return CONNECT_FAILURE; } } - m_ssl_options.support = ssl_support_t::e_ssl_support_enabled; } return CONNECT_SUCCESS; }else @@ -223,7 +222,6 @@ namespace net_utils return false; if (m_ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect) { - m_ssl_options.support = epee::net_utils::ssl_support_t::e_ssl_support_enabled; if (try_connect_result == CONNECT_NO_SSL) { MERROR("SSL handshake failed on an autodetect connection, reconnecting without SSL"); @@ -396,7 +394,7 @@ namespace net_utils if (!m_connected || !m_ssl_socket->next_layer().is_open()) return false; if (ssl) - *ssl = m_ssl_options.support == ssl_support_t::e_ssl_support_enabled; + *ssl = m_ssl_options.support != ssl_support_t::e_ssl_support_disabled; return true; } @@ -651,7 +649,7 @@ namespace net_utils bool write(const void* data, size_t sz, boost::system::error_code& ec) { bool success; - if(m_ssl_options.support == ssl_support_t::e_ssl_support_enabled) + if(m_ssl_options.support != ssl_support_t::e_ssl_support_disabled) success = boost::asio::write(*m_ssl_socket, boost::asio::buffer(data, sz), ec); else success = boost::asio::write(m_ssl_socket->next_layer(), boost::asio::buffer(data, sz), ec); @@ -660,7 +658,7 @@ namespace net_utils void async_write(const void* data, size_t sz, boost::system::error_code& ec) { - if(m_ssl_options.support == ssl_support_t::e_ssl_support_enabled) + if(m_ssl_options.support != ssl_support_t::e_ssl_support_disabled) boost::asio::async_write(*m_ssl_socket, boost::asio::buffer(data, sz), boost::lambda::var(ec) = boost::lambda::_1); else boost::asio::async_write(m_ssl_socket->next_layer(), boost::asio::buffer(data, sz), boost::lambda::var(ec) = boost::lambda::_1); @@ -668,7 +666,7 @@ namespace net_utils void async_read(char* buff, size_t sz, boost::asio::detail::transfer_at_least_t transfer_at_least, handler_obj& hndlr) { - if(m_ssl_options.support != ssl_support_t::e_ssl_support_enabled) + if(m_ssl_options.support == ssl_support_t::e_ssl_support_disabled) boost::asio::async_read(m_ssl_socket->next_layer(), boost::asio::buffer(buff, sz), transfer_at_least, hndlr); else boost::asio::async_read(*m_ssl_socket, boost::asio::buffer(buff, sz), transfer_at_least, hndlr); diff --git a/contrib/epee/include/net/net_parse_helpers.h b/contrib/epee/include/net/net_parse_helpers.h index 708cce0ff..1d156d19c 100644 --- a/contrib/epee/include/net/net_parse_helpers.h +++ b/contrib/epee/include/net/net_parse_helpers.h @@ -94,7 +94,7 @@ namespace net_utils return true; } - inline + inline bool parse_uri(const std::string uri, http::uri_content& content) { @@ -128,11 +128,51 @@ namespace net_utils return true; } + inline + bool parse_url_ipv6(const std::string url_str, http::url_content& content) + { + STATIC_REGEXP_EXPR_1(rexp_match_uri, "^((.*?)://)?(\\[(.*)\\](:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal); + // 12 3 4 5 6 7 - inline + content.port = 0; + boost::smatch result; + if(!(boost::regex_search(url_str, result, rexp_match_uri, boost::match_default) && result[0].matched)) + { + LOG_PRINT_L1("[PARSE URI] regex not matched for uri: " << rexp_match_uri); + //content.m_path = uri; + return false; + } + if(result[2].matched) + { + content.schema = result[2]; + } + if(result[4].matched) + { + content.host = result[4]; + } + else // if host not matched, matching should be considered failed + { + return false; + } + if(result[6].matched) + { + content.port = boost::lexical_cast<uint64_t>(result[6]); + } + if(result[7].matched) + { + content.uri = result[7]; + return parse_uri(result[7], content.m_uri_content); + } + + return true; + } + + inline bool parse_url(const std::string url_str, http::url_content& content) { + if (parse_url_ipv6(url_str, content)) return true; + ///iframe_test.html?api_url=http://api.vk.com/api.php&api_id=3289090&api_settings=1&viewer_id=562964060&viewer_type=0&sid=0aad8d1c5713130f9ca0076f2b7b47e532877424961367d81e7fa92455f069be7e21bc3193cbd0be11895&secret=368ebbc0ef&access_token=668bc03f43981d883f73876ffff4aa8564254b359cc745dfa1b3cde7bdab2e94105d8f6d8250717569c0a7&user_id=0&group_id=0&is_app_user=1&auth_key=d2f7a895ca5ff3fdb2a2a8ae23fe679a&language=0&parent_language=0&ad_info=ElsdCQBaQlxiAQRdFUVUXiN2AVBzBx5pU1BXIgZUJlIEAWcgAUoLQg==&referrer=unknown&lc_name=9834b6a3&hash= //STATIC_REGEXP_EXPR_1(rexp_match_uri, "^([^?#]*)(\\?([^#]*))?(#(.*))?", boost::regex::icase | boost::regex::normal); STATIC_REGEXP_EXPR_1(rexp_match_uri, "^((.*?)://)?(([^/:]*)(:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal); diff --git a/contrib/epee/include/net/net_ssl.h b/contrib/epee/include/net/net_ssl.h index 957903ff8..3a97dfdaf 100644 --- a/contrib/epee/include/net/net_ssl.h +++ b/contrib/epee/include/net/net_ssl.h @@ -37,6 +37,8 @@ #include <boost/asio/ssl.hpp> #include <boost/system/error_code.hpp> +#define SSL_FINGERPRINT_SIZE 32 + namespace epee { namespace net_utils @@ -133,6 +135,9 @@ namespace net_utils constexpr size_t get_ssl_magic_size() { return 9; } bool is_ssl(const unsigned char *data, size_t len); bool ssl_support_from_string(ssl_support_t &ssl, boost::string_ref s); + + bool create_ec_ssl_certificate(EVP_PKEY *&pkey, X509 *&cert); + bool create_rsa_ssl_certificate(EVP_PKEY *&pkey, X509 *&cert); } } diff --git a/contrib/epee/include/net/net_utils_base.h b/contrib/epee/include/net/net_utils_base.h index 50536f63b..5ae3e53b3 100644 --- a/contrib/epee/include/net/net_utils_base.h +++ b/contrib/epee/include/net/net_utils_base.h @@ -31,6 +31,7 @@ #include <boost/uuid/uuid.hpp> #include <boost/asio/io_service.hpp> +#include <boost/asio/ip/address_v6.hpp> #include <typeinfo> #include <type_traits> #include "enums.h" @@ -41,7 +42,7 @@ #define MONERO_DEFAULT_LOG_CATEGORY "net" #ifndef MAKE_IP -#define MAKE_IP( a1, a2, a3, a4 ) (a1|(a2<<8)|(a3<<16)|(a4<<24)) +#define MAKE_IP( a1, a2, a3, a4 ) (a1|(a2<<8)|(a3<<16)|(((uint32_t)a4)<<24)) #endif #if BOOST_VERSION >= 107000 @@ -107,6 +108,106 @@ namespace net_utils inline bool operator>=(const ipv4_network_address& lhs, const ipv4_network_address& rhs) noexcept { return !lhs.less(rhs); } + class ipv4_network_subnet + { + uint32_t m_ip; + uint8_t m_mask; + + public: + constexpr ipv4_network_subnet() noexcept + : ipv4_network_subnet(0, 0) + {} + + constexpr ipv4_network_subnet(uint32_t ip, uint8_t mask) noexcept + : m_ip(ip), m_mask(mask) {} + + bool equal(const ipv4_network_subnet& other) const noexcept; + bool less(const ipv4_network_subnet& other) const noexcept; + constexpr bool is_same_host(const ipv4_network_subnet& other) const noexcept + { return subnet() == other.subnet(); } + bool matches(const ipv4_network_address &address) const; + + constexpr uint32_t subnet() const noexcept { return m_ip & ~(0xffffffffull << m_mask); } + std::string str() const; + std::string host_str() const; + bool is_loopback() const; + bool is_local() const; + static constexpr address_type get_type_id() noexcept { return address_type::invalid; } + static constexpr zone get_zone() noexcept { return zone::public_; } + static constexpr bool is_blockable() noexcept { return true; } + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(m_ip) + KV_SERIALIZE(m_mask) + END_KV_SERIALIZE_MAP() + }; + + inline bool operator==(const ipv4_network_subnet& lhs, const ipv4_network_subnet& rhs) noexcept + { return lhs.equal(rhs); } + inline bool operator!=(const ipv4_network_subnet& lhs, const ipv4_network_subnet& rhs) noexcept + { return !lhs.equal(rhs); } + inline bool operator<(const ipv4_network_subnet& lhs, const ipv4_network_subnet& rhs) noexcept + { return lhs.less(rhs); } + inline bool operator<=(const ipv4_network_subnet& lhs, const ipv4_network_subnet& rhs) noexcept + { return !rhs.less(lhs); } + inline bool operator>(const ipv4_network_subnet& lhs, const ipv4_network_subnet& rhs) noexcept + { return rhs.less(lhs); } + inline bool operator>=(const ipv4_network_subnet& lhs, const ipv4_network_subnet& rhs) noexcept + { return !lhs.less(rhs); } + + class ipv6_network_address + { + protected: + boost::asio::ip::address_v6 m_address; + uint16_t m_port; + + public: + ipv6_network_address() + : ipv6_network_address(boost::asio::ip::address_v6::loopback(), 0) + {} + + ipv6_network_address(const boost::asio::ip::address_v6& ip, uint16_t port) + : m_address(ip), m_port(port) + { + } + + bool equal(const ipv6_network_address& other) const noexcept; + bool less(const ipv6_network_address& other) const noexcept; + bool is_same_host(const ipv6_network_address& other) const noexcept + { return m_address == other.m_address; } + + boost::asio::ip::address_v6 ip() const noexcept { return m_address; } + uint16_t port() const noexcept { return m_port; } + std::string str() const; + std::string host_str() const; + bool is_loopback() const; + bool is_local() const; + static constexpr address_type get_type_id() noexcept { return address_type::ipv6; } + static constexpr zone get_zone() noexcept { return zone::public_; } + static constexpr bool is_blockable() noexcept { return true; } + + static const uint8_t ID = 2; + BEGIN_KV_SERIALIZE_MAP() + boost::asio::ip::address_v6::bytes_type bytes = this_ref.m_address.to_bytes(); + epee::serialization::selector<is_store>::serialize_t_val_as_blob(bytes, stg, hparent_section, "addr"); + const_cast<boost::asio::ip::address_v6&>(this_ref.m_address) = boost::asio::ip::address_v6(bytes); + KV_SERIALIZE(m_port) + END_KV_SERIALIZE_MAP() + }; + + inline bool operator==(const ipv6_network_address& lhs, const ipv6_network_address& rhs) noexcept + { return lhs.equal(rhs); } + inline bool operator!=(const ipv6_network_address& lhs, const ipv6_network_address& rhs) noexcept + { return !lhs.equal(rhs); } + inline bool operator<(const ipv6_network_address& lhs, const ipv6_network_address& rhs) noexcept + { return lhs.less(rhs); } + inline bool operator<=(const ipv6_network_address& lhs, const ipv6_network_address& rhs) noexcept + { return !rhs.less(lhs); } + inline bool operator>(const ipv6_network_address& lhs, const ipv6_network_address& rhs) noexcept + { return rhs.less(lhs); } + inline bool operator>=(const ipv6_network_address& lhs, const ipv6_network_address& rhs) noexcept + { return !lhs.less(rhs); } + class network_address { struct interface @@ -214,6 +315,8 @@ namespace net_utils { case address_type::ipv4: return this_ref.template serialize_addr<ipv4_network_address>(is_store_, stg, hparent_section); + case address_type::ipv6: + return this_ref.template serialize_addr<ipv6_network_address>(is_store_, stg, hparent_section); case address_type::tor: return this_ref.template serialize_addr<net::tor_address>(is_store_, stg, hparent_section); case address_type::i2p: @@ -250,7 +353,7 @@ namespace net_utils const network_address m_remote_address; const bool m_is_income; const time_t m_started; - const time_t m_ssl; + const bool m_ssl; time_t m_last_recv; time_t m_last_send; uint64_t m_recv_cnt; @@ -294,6 +397,11 @@ namespace net_utils m_max_speed_up(0) {} + connection_context_base(const connection_context_base& a): connection_context_base() + { + set_details(a.m_connection_id, a.m_remote_address, a.m_is_income, a.m_ssl); + } + connection_context_base& operator=(const connection_context_base& a) { set_details(a.m_connection_id, a.m_remote_address, a.m_is_income, a.m_ssl); diff --git a/contrib/epee/include/rolling_median.h b/contrib/epee/include/rolling_median.h new file mode 100644 index 000000000..8b5a82a84 --- /dev/null +++ b/contrib/epee/include/rolling_median.h @@ -0,0 +1,236 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Adapted from source by AShelly: +// Copyright (c) 2011 ashelly.myopenid.com, licenced under the MIT licence +// https://stackoverflow.com/questions/5527437/rolling-median-in-c-turlach-implementation +// https://stackoverflow.com/questions/1309263/rolling-median-algorithm-in-c +// https://ideone.com/XPbl6 + +#pragma once + +#include <stdlib.h> +#include <stdint.h> + +namespace epee +{ +namespace misc_utils +{ + +template<typename Item> +struct rolling_median_t +{ +private: + Item* data; //circular queue of values + int* pos; //index into `heap` for each value + int* heap; //max/median/min heap holding indexes into `data`. + int N; //allocated size. + int idx; //position in circular queue + int minCt; //count of items in min heap + int maxCt; //count of items in max heap + int sz; //count of items in heap + +private: + + //returns true if heap[i] < heap[j] + bool mmless(int i, int j) const + { + return data[heap[i]] < data[heap[j]]; + } + + //swaps items i&j in heap, maintains indexes + bool mmexchange(int i, int j) + { + const int t = heap[i]; + heap[i] = heap[j]; + heap[j] = t; + pos[heap[i]] = i; + pos[heap[j]] = j; + return 1; + } + + //swaps items i&j if i<j; returns true if swapped + bool mmCmpExch(int i, int j) + { + return mmless(i, j) && mmexchange(i, j); + } + + //maintains minheap property for all items below i. + void minSortDown(int i) + { + for (i *= 2; i <= minCt; i *= 2) + { + if (i < minCt && mmless(i + 1, i)) + ++i; + if (!mmCmpExch(i, i / 2)) + break; + } + } + + //maintains maxheap property for all items below i. (negative indexes) + void maxSortDown(int i) + { + for (i *= 2; i >= -maxCt; i *= 2) + { + if (i > -maxCt && mmless(i, i - 1)) + --i; + if (!mmCmpExch(i / 2, i)) + break; + } + } + + //maintains minheap property for all items above i, including median + //returns true if median changed + bool minSortUp(int i) + { + while (i > 0 && mmCmpExch(i, i / 2)) + i /= 2; + return i == 0; + } + + //maintains maxheap property for all items above i, including median + //returns true if median changed + bool maxSortUp(int i) + { + while (i < 0 && mmCmpExch(i / 2, i)) + i /= 2; + return i == 0; + } + +protected: + rolling_median_t &operator=(const rolling_median_t&) = delete; + rolling_median_t(const rolling_median_t&) = delete; + +public: + //creates new rolling_median_t: to calculate `nItems` running median. + rolling_median_t(size_t N): N(N) + { + int size = N * (sizeof(Item) + sizeof(int) * 2); + data = (Item*)malloc(size); + pos = (int*) (data + N); + heap = pos + N + (N / 2); //points to middle of storage. + clear(); + } + + rolling_median_t(rolling_median_t &&m) + { + free(data); + memcpy(this, &m, sizeof(rolling_median_t)); + m.data = NULL; + } + rolling_median_t &operator=(rolling_median_t &&m) + { + free(data); + memcpy(this, &m, sizeof(rolling_median_t)); + m.data = NULL; + return *this; + } + + ~rolling_median_t() + { + free(data); + } + + void clear() + { + idx = 0; + minCt = 0; + maxCt = 0; + sz = 0; + int nItems = N; + while (nItems--) //set up initial heap fill pattern: median,max,min,max,... + { + pos[nItems] = ((nItems + 1) / 2) * ((nItems & 1) ? -1 : 1); + heap[pos[nItems]] = nItems; + } + } + + int size() const + { + return sz; + } + + //Inserts item, maintains median in O(lg nItems) + void insert(Item v) + { + int p = pos[idx]; + Item old = data[idx]; + data[idx] = v; + idx = (idx + 1) % N; + sz = std::min<int>(sz + 1, N); + if (p > 0) //new item is in minHeap + { + if (minCt < (N - 1) / 2) + { + ++minCt; + } + else if (v > old) + { + minSortDown(p); + return; + } + if (minSortUp(p) && mmCmpExch(0, -1)) + maxSortDown(-1); + } + else if (p < 0) //new item is in maxheap + { + if (maxCt < N / 2) + { + ++maxCt; + } + else if (v < old) + { + maxSortDown(p); + return; + } + if (maxSortUp(p) && minCt && mmCmpExch(1, 0)) + minSortDown(1); + } + else //new item is at median + { + if (maxCt && maxSortUp(-1)) + maxSortDown(-1); + if (minCt && minSortUp(1)) + minSortDown(1); + } + } + + //returns median item (or average of 2 when item count is even) + Item median() const + { + Item v = data[heap[0]]; + if (minCt < maxCt) + { + v = (v + data[heap[-1]]) / 2; + } + return v; + } +}; + +} +} diff --git a/contrib/epee/include/storages/parserse_base_utils.h b/contrib/epee/include/storages/parserse_base_utils.h index b5c4138c5..fe53628a5 100644 --- a/contrib/epee/include/storages/parserse_base_utils.h +++ b/contrib/epee/include/storages/parserse_base_utils.h @@ -31,6 +31,9 @@ #include <algorithm> #include <boost/utility/string_ref.hpp> +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "serialization" + namespace epee { namespace misc_utils @@ -62,6 +65,26 @@ namespace misc_utils 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }; + static const constexpr unsigned char isx[256] = + { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 10, 11, 12, 13, 14, 15, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 10, 11, 12, 13, 14, 15, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + }; + inline bool isspace(char c) { return lut[(uint8_t)c] & 8; @@ -162,6 +185,42 @@ namespace misc_utils val.push_back('\\');break; case '/': //Slash character val.push_back('/');break; + case 'u': //Unicode code point + if (buf_end - it < 4) + { + ASSERT_MES_AND_THROW("Invalid Unicode escape sequence"); + } + else + { + uint32_t dst = 0; + for (int i = 0; i < 4; ++i) + { + const unsigned char tmp = isx[(int)*++it]; + CHECK_AND_ASSERT_THROW_MES(tmp != 0xff, "Bad Unicode encoding"); + dst = dst << 4 | tmp; + } + // encode as UTF-8 + if (dst <= 0x7f) + { + val.push_back(dst); + } + else if (dst <= 0x7ff) + { + val.push_back(0xc0 | (dst >> 6)); + val.push_back(0x80 | (dst & 0x3f)); + } + else if (dst <= 0xffff) + { + val.push_back(0xe0 | (dst >> 12)); + val.push_back(0x80 | ((dst >> 6) & 0x3f)); + val.push_back(0x80 | (dst & 0x3f)); + } + else + { + ASSERT_MES_AND_THROW("Unicode code point is out or range"); + } + } + break; default: val.push_back(*it); LOG_PRINT_L0("Unknown escape sequence :\"\\" << *it << "\""); diff --git a/contrib/epee/include/storages/portable_storage_from_bin.h b/contrib/epee/include/storages/portable_storage_from_bin.h index 2884f8c5e..e0a32b3ca 100644 --- a/contrib/epee/include/storages/portable_storage_from_bin.h +++ b/contrib/epee/include/storages/portable_storage_from_bin.h @@ -136,6 +136,7 @@ namespace epee //for pod types array_entry_t<type_name> sa; size_t size = read_varint(); + CHECK_AND_ASSERT_THROW_MES(size <= m_count, "Size sanity check failed"); sa.reserve(size); //TODO: add some optimization here later while(size--) diff --git a/contrib/epee/include/string_tools.h b/contrib/epee/include/string_tools.h index da47b7d55..1be5eb5e1 100644 --- a/contrib/epee/include/string_tools.h +++ b/contrib/epee/include/string_tools.h @@ -59,26 +59,6 @@ #pragma comment (lib, "Rpcrt4.lib") #endif -static const constexpr unsigned char isx[256] = -{ - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 10, 11, 12, 13, 14, 15, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 10, 11, 12, 13, 14, 15, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, -}; - namespace epee { namespace string_tools @@ -99,10 +79,10 @@ namespace string_tools for(size_t i = 0; i < s.size(); i += 2) { int tmp = *src++; - tmp = isx[tmp]; + tmp = epee::misc_utils::parse::isx[tmp]; if (tmp == 0xff) return false; int t2 = *src++; - t2 = isx[t2]; + t2 = epee::misc_utils::parse::isx[t2]; if (t2 == 0xff) return false; *dst++ = (tmp << 4) | t2; } diff --git a/contrib/epee/src/buffer.cpp b/contrib/epee/src/buffer.cpp index d637b905e..10ea6de56 100644 --- a/contrib/epee/src/buffer.cpp +++ b/contrib/epee/src/buffer.cpp @@ -64,7 +64,8 @@ void buffer::append(const void *data, size_t sz) size_t reserve = (((size() + sz) * 3 / 2) + 4095) & ~4095; new_storage.reserve(reserve); new_storage.resize(size()); - memcpy(new_storage.data(), storage.data() + offset, storage.size() - offset); + if (size() > 0) + memcpy(new_storage.data(), storage.data() + offset, storage.size() - offset); offset = 0; std::swap(storage, new_storage); } diff --git a/contrib/epee/src/connection_basic.cpp b/contrib/epee/src/connection_basic.cpp index 19f2c7b02..82d9e3b53 100644 --- a/contrib/epee/src/connection_basic.cpp +++ b/contrib/epee/src/connection_basic.cpp @@ -284,9 +284,6 @@ double connection_basic::get_sleep_time(size_t cb) { return t; } -void connection_basic::set_save_graph(bool save_graph) { -} - } // namespace } // namespace diff --git a/contrib/epee/src/mlog.cpp b/contrib/epee/src/mlog.cpp index a8bfd114d..4c6ad5516 100644 --- a/contrib/epee/src/mlog.cpp +++ b/contrib/epee/src/mlog.cpp @@ -100,7 +100,7 @@ static const char *get_default_categories(int level) switch (level) { case 0: - categories = "*:WARNING,net:FATAL,net.http:FATAL,net.p2p:FATAL,net.cn:FATAL,global:INFO,verify:FATAL,serialization:FATAL,stacktrace:INFO,logging:INFO,msgwriter:INFO"; + categories = "*:WARNING,net:FATAL,net.http:FATAL,net.ssl:FATAL,net.p2p:FATAL,net.cn:FATAL,global:INFO,verify:FATAL,serialization:FATAL,stacktrace:INFO,logging:INFO,msgwriter:INFO"; break; case 1: categories = "*:INFO,global:INFO,stacktrace:INFO,logging:INFO,msgwriter:INFO,perf.*:DEBUG"; diff --git a/contrib/epee/src/net_helper.cpp b/contrib/epee/src/net_helper.cpp index 3543f5716..719f1c8e0 100644 --- a/contrib/epee/src/net_helper.cpp +++ b/contrib/epee/src/net_helper.cpp @@ -11,10 +11,39 @@ namespace net_utils ////////////////////////////////////////////////////////////////////////// boost::asio::ip::tcp::resolver resolver(GET_IO_SERVICE(timeout)); boost::asio::ip::tcp::resolver::query query(boost::asio::ip::tcp::v4(), addr, port, boost::asio::ip::tcp::resolver::query::canonical_name); - boost::asio::ip::tcp::resolver::iterator iterator = resolver.resolve(query); + + bool try_ipv6 = false; + boost::asio::ip::tcp::resolver::iterator iterator; boost::asio::ip::tcp::resolver::iterator end; - if(iterator == end) // Documentation states that successful call is guaranteed to be non-empty - throw boost::system::system_error{boost::asio::error::fault, "Failed to resolve " + addr}; + boost::system::error_code resolve_error; + try + { + iterator = resolver.resolve(query, resolve_error); + if(iterator == end) // Documentation states that successful call is guaranteed to be non-empty + { + // if IPv4 resolution fails, try IPv6. Unintentional outgoing IPv6 connections should only + // be possible if for some reason a hostname was given and that hostname fails IPv4 resolution, + // so at least for now there should not be a need for a flag "using ipv6 is ok" + try_ipv6 = true; + } + + } + catch (const boost::system::system_error& e) + { + if (resolve_error != boost::asio::error::host_not_found && + resolve_error != boost::asio::error::host_not_found_try_again) + { + throw; + } + try_ipv6 = true; + } + if (try_ipv6) + { + boost::asio::ip::tcp::resolver::query query6(boost::asio::ip::tcp::v6(), addr, port, boost::asio::ip::tcp::resolver::query::canonical_name); + iterator = resolver.resolve(query6); + if (iterator == end) + throw boost::system::system_error{boost::asio::error::fault, "Failed to resolve " + addr}; + } ////////////////////////////////////////////////////////////////////////// diff --git a/contrib/epee/src/net_ssl.cpp b/contrib/epee/src/net_ssl.cpp index 7bedb18ac..7d48d2a64 100644 --- a/contrib/epee/src/net_ssl.cpp +++ b/contrib/epee/src/net_ssl.cpp @@ -78,6 +78,24 @@ namespace }; using openssl_bignum = std::unique_ptr<BIGNUM, openssl_bignum_free>; + struct openssl_ec_key_free + { + void operator()(EC_KEY* ptr) const noexcept + { + EC_KEY_free(ptr); + } + }; + using openssl_ec_key = std::unique_ptr<EC_KEY, openssl_ec_key_free>; + + struct openssl_group_free + { + void operator()(EC_GROUP* ptr) const noexcept + { + EC_GROUP_free(ptr); + } + }; + using openssl_group = std::unique_ptr<EC_GROUP, openssl_group_free>; + boost::system::error_code load_ca_file(boost::asio::ssl::context& ctx, const std::string& path) { SSL_CTX* const ssl_ctx = ctx.native_handle(); // could be moved from context @@ -101,7 +119,7 @@ namespace net_utils // https://stackoverflow.com/questions/256405/programmatically-create-x509-certificate-using-openssl -bool create_ssl_certificate(EVP_PKEY *&pkey, X509 *&cert) +bool create_rsa_ssl_certificate(EVP_PKEY *&pkey, X509 *&cert) { MGINFO("Generating SSL certificate"); pkey = EVP_PKEY_new(); @@ -171,6 +189,87 @@ bool create_ssl_certificate(EVP_PKEY *&pkey, X509 *&cert) return true; } +bool create_ec_ssl_certificate(EVP_PKEY *&pkey, X509 *&cert, int type) +{ + MGINFO("Generating SSL certificate"); + pkey = EVP_PKEY_new(); + if (!pkey) + { + MERROR("Failed to create new private key"); + return false; + } + + openssl_pkey pkey_deleter{pkey}; + openssl_ec_key ec_key{EC_KEY_new()}; + if (!ec_key) + { + MERROR("Error allocating EC private key"); + return false; + } + + EC_GROUP *group = EC_GROUP_new_by_curve_name(type); + if (!group) + { + MERROR("Error getting EC group " << type); + return false; + } + openssl_group group_deleter{group}; + + EC_GROUP_set_asn1_flag(group, OPENSSL_EC_NAMED_CURVE); + EC_GROUP_set_point_conversion_form(group, POINT_CONVERSION_UNCOMPRESSED); + + if (!EC_GROUP_check(group, NULL)) + { + MERROR("Group failed check: " << ERR_reason_error_string(ERR_get_error())); + return false; + } + if (EC_KEY_set_group(ec_key.get(), group) != 1) + { + MERROR("Error setting EC group"); + return false; + } + if (EC_KEY_generate_key(ec_key.get()) != 1) + { + MERROR("Error generating EC private key"); + return false; + } + if (EVP_PKEY_assign_EC_KEY(pkey, ec_key.get()) <= 0) + { + MERROR("Error assigning EC private key"); + return false; + } + + // the key is now managed by the EVP_PKEY structure + (void)ec_key.release(); + + cert = X509_new(); + if (!cert) + { + MERROR("Failed to create new X509 certificate"); + return false; + } + ASN1_INTEGER_set(X509_get_serialNumber(cert), 1); + X509_gmtime_adj(X509_get_notBefore(cert), 0); + X509_gmtime_adj(X509_get_notAfter(cert), 3600 * 24 * 182); // half a year + if (!X509_set_pubkey(cert, pkey)) + { + MERROR("Error setting pubkey on certificate"); + X509_free(cert); + return false; + } + X509_NAME *name = X509_get_subject_name(cert); + X509_set_issuer_name(cert, name); + + if (X509_sign(cert, pkey, EVP_sha256()) == 0) + { + MERROR("Error signing certificate"); + X509_free(cert); + return false; + } + (void)pkey_deleter.release(); + return true; +} + ssl_options_t::ssl_options_t(std::vector<std::vector<std::uint8_t>> fingerprints, std::string ca_path) : fingerprints_(std::move(fingerprints)), ca_path(std::move(ca_path)), @@ -195,7 +294,7 @@ boost::asio::ssl::context ssl_options_t::create_context() const ssl_context.set_options(boost::asio::ssl::context::no_tlsv1_1); // only allow a select handful of tls v1.3 and v1.2 ciphers to be used - SSL_CTX_set_cipher_list(ssl_context.native_handle(), "ECDHE-ECDSA-CHACHA20-POLY1305-SHA256:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-CHACHA20-POLY1305"); + SSL_CTX_set_cipher_list(ssl_context.native_handle(), "ECDHE-ECDSA-CHACHA20-POLY1305-SHA256:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256"); // set options on the SSL context for added security SSL_CTX *ctx = ssl_context.native_handle(); @@ -214,6 +313,10 @@ boost::asio::ssl::context ssl_options_t::create_context() const #ifdef SSL_OP_NO_COMPRESSION SSL_CTX_set_options(ctx, SSL_OP_NO_COMPRESSION); #endif +#ifdef SSL_OP_CIPHER_SERVER_PREFERENCE + SSL_CTX_set_options(ctx, SSL_OP_CIPHER_SERVER_PREFERENCE); +#endif + SSL_CTX_set_ecdh_auto(ctx, 1); switch (verification) { @@ -240,11 +343,29 @@ boost::asio::ssl::context ssl_options_t::create_context() const { EVP_PKEY *pkey; X509 *cert; - CHECK_AND_ASSERT_THROW_MES(create_ssl_certificate(pkey, cert), "Failed to create certificate"); + bool ok = false; + +#ifdef USE_EXTRA_EC_CERT + CHECK_AND_ASSERT_THROW_MES(create_ec_ssl_certificate(pkey, cert, NID_secp256k1), "Failed to create certificate"); + CHECK_AND_ASSERT_THROW_MES(SSL_CTX_use_certificate(ctx, cert), "Failed to use generated certificate"); + if (!SSL_CTX_use_PrivateKey(ctx, pkey)) + MERROR("Failed to use generated EC private key for " << NID_secp256k1); + else + ok = true; + X509_free(cert); + EVP_PKEY_free(pkey); +#endif + + CHECK_AND_ASSERT_THROW_MES(create_rsa_ssl_certificate(pkey, cert), "Failed to create certificate"); CHECK_AND_ASSERT_THROW_MES(SSL_CTX_use_certificate(ctx, cert), "Failed to use generated certificate"); - // don't free the cert, the CTX owns it now - CHECK_AND_ASSERT_THROW_MES(SSL_CTX_use_PrivateKey(ctx, pkey), "Failed to use generated private key"); + if (!SSL_CTX_use_PrivateKey(ctx, pkey)) + MERROR("Failed to use generated RSA private key for RSA"); + else + ok = true; + X509_free(cert); EVP_PKEY_free(pkey); + + CHECK_AND_ASSERT_THROW_MES(ok, "Failed to use any generated certificate"); } else auth.use_ssl_certificate(ssl_context); @@ -321,7 +442,7 @@ bool ssl_options_t::has_fingerprint(boost::asio::ssl::verify_context &ctx) const unsigned int size{ 0 }; // create the digest from the certificate - if (!X509_digest(cert, EVP_sha1(), digest.data(), &size)) { + if (!X509_digest(cert, EVP_sha256(), digest.data(), &size)) { MERROR("Failed to create certificate fingerprint"); return false; } diff --git a/contrib/epee/src/net_utils_base.cpp b/contrib/epee/src/net_utils_base.cpp index 9b781027e..5cc49cc71 100644 --- a/contrib/epee/src/net_utils_base.cpp +++ b/contrib/epee/src/net_utils_base.cpp @@ -21,6 +21,37 @@ namespace epee { namespace net_utils bool ipv4_network_address::is_loopback() const { return net_utils::is_ip_loopback(ip()); } bool ipv4_network_address::is_local() const { return net_utils::is_ip_local(ip()); } + bool ipv6_network_address::equal(const ipv6_network_address& other) const noexcept + { return is_same_host(other) && port() == other.port(); } + + bool ipv6_network_address::less(const ipv6_network_address& other) const noexcept + { return is_same_host(other) ? port() < other.port() : m_address < other.m_address; } + + std::string ipv6_network_address::str() const + { return std::string("[") + host_str() + "]:" + std::to_string(port()); } + + std::string ipv6_network_address::host_str() const { return m_address.to_string(); } + bool ipv6_network_address::is_loopback() const { return m_address.is_loopback(); } + bool ipv6_network_address::is_local() const { return m_address.is_link_local(); } + + + bool ipv4_network_subnet::equal(const ipv4_network_subnet& other) const noexcept + { return is_same_host(other) && m_mask == other.m_mask; } + + bool ipv4_network_subnet::less(const ipv4_network_subnet& other) const noexcept + { return subnet() < other.subnet() ? true : (other.subnet() < subnet() ? false : (m_mask < other.m_mask)); } + + std::string ipv4_network_subnet::str() const + { return string_tools::get_ip_string_from_int32(subnet()) + "/" + std::to_string(m_mask); } + + std::string ipv4_network_subnet::host_str() const { return string_tools::get_ip_string_from_int32(subnet()) + "/" + std::to_string(m_mask); } + bool ipv4_network_subnet::is_loopback() const { return net_utils::is_ip_loopback(subnet()); } + bool ipv4_network_subnet::is_local() const { return net_utils::is_ip_local(subnet()); } + bool ipv4_network_subnet::matches(const ipv4_network_address &address) const + { + return (address.ip() & ~(0xffffffffull << m_mask)) == subnet(); + } + bool network_address::equal(const network_address& other) const { diff --git a/contrib/epee/src/wipeable_string.cpp b/contrib/epee/src/wipeable_string.cpp index 3a6ee5dac..4209b71bf 100644 --- a/contrib/epee/src/wipeable_string.cpp +++ b/contrib/epee/src/wipeable_string.cpp @@ -62,13 +62,15 @@ wipeable_string::wipeable_string(wipeable_string &&other) wipeable_string::wipeable_string(const std::string &other) { grow(other.size()); - memcpy(buffer.data(), other.c_str(), size()); + if (size() > 0) + memcpy(buffer.data(), other.c_str(), size()); } wipeable_string::wipeable_string(std::string &&other) { grow(other.size()); - memcpy(buffer.data(), other.c_str(), size()); + if (size() > 0) + memcpy(buffer.data(), other.c_str(), size()); if (!other.empty()) { memwipe(&other[0], other.size()); // we're kinda left with this again aren't we @@ -79,7 +81,8 @@ wipeable_string::wipeable_string(std::string &&other) wipeable_string::wipeable_string(const char *s) { grow(strlen(s)); - memcpy(buffer.data(), s, size()); + if (size() > 0) + memcpy(buffer.data(), s, size()); } wipeable_string::wipeable_string(const char *s, size_t len) @@ -112,14 +115,18 @@ void wipeable_string::grow(size_t sz, size_t reserved) } size_t old_sz = buffer.size(); std::unique_ptr<char[]> tmp{new char[old_sz]}; - memcpy(tmp.get(), buffer.data(), old_sz * sizeof(char)); if (old_sz > 0) + { + memcpy(tmp.get(), buffer.data(), old_sz * sizeof(char)); memwipe(buffer.data(), old_sz * sizeof(char)); + } buffer.reserve(reserved); buffer.resize(sz); - memcpy(buffer.data(), tmp.get(), old_sz * sizeof(char)); if (old_sz > 0) + { + memcpy(buffer.data(), tmp.get(), old_sz * sizeof(char)); memwipe(tmp.get(), old_sz * sizeof(char)); + } } void wipeable_string::push_back(char c) diff --git a/contrib/gitian/README.md b/contrib/gitian/README.md index b869bde87..32aee5f56 100644 --- a/contrib/gitian/README.md +++ b/contrib/gitian/README.md @@ -26,12 +26,16 @@ Preparing the Gitian builder host The first step is to prepare the host environment that will be used to perform the Gitian builds. This guide explains how to set up the environment, and how to start the builds. -Gitian offers to build with either `kvm`, `docker` or `lxc`. The default build -path chosen is `lxc`, but its setup is more complicated. You need to be logged in as the `gitianuser`. -If this user does not exist yet on your system, create it. Gitian can use -either kvm, lxc or docker as a host environment. This documentation will show -how to build with lxc and docker. While the docker setup is easy, the lxc setup -is more involved. + +* Gitian host OS should be Ubuntu 18.04 "Bionic Beaver". If you are on a mac or windows for example, you can run it in a VM but will be slower. + +* Gitian gives you the option of using any of 3 different virtualization tools: `kvm`, `docker` or `lxc`. This documentation will only show how to build with `lxc` and `docker` (documentation for `kvm` is welcome). Building with `lxc` is the default, but is more complicated, so we recommend docker your first time. + + +## Create the gitianuser account + +You need to create a new user called `gitianuser` and be logged in as that user. The user needs `sudo` access. + LXC --- @@ -76,18 +80,34 @@ This setup is required to enable networking in the container. Docker ------ -Building in docker does not require much setup. Install docker on your host, then type the following: +Prepare for building with docker: + +```bash +sudo apt-get install git make curl docker.io +``` + +Consider adding `gitianuser` to the `docker` group after reading about [the security implications](https://docs.docker.com/v17.09/engine/installation/linux/linux-postinstall/): ```bash -sudo apt-get install git make curl +sudo groupadd docker sudo usermod -aG docker gitianuser ``` +Optionally add yourself to the docker group. Note that this will give docker root access to your system. -Manual and Building +```bash +sudo usermod -aG docker gitianuser +``` + +Manual Building ------------------- The instructions below use the automated script [gitian-build.py](gitian-build.py) which only works in Ubuntu. +======= +The script automatically installs some packages with apt. If you are not running it on a debian-like system, pass `--no-apt` along with the other +arguments to it. It calls all available .yml descriptors, which in turn pass the build configurations for different platforms to gitian. +The instructions below use the automated script [gitian-build.py](gitian-build.py) which is tested to work on Ubuntu. + It calls all available .yml descriptors, which in turn pass the build configurations for different platforms to gitian. Help for the build steps taken can be accessed with `./gitian-build.py --help`. @@ -100,66 +120,95 @@ The `gitian-build.py` script will checkout different release tags, so it's best cp monero/contrib/gitian/gitian-build.py . ``` -Setup the required environment, you only need to do this once: +### Setup the required environment + +Setup for LXC: ```bash -./gitian-build.py --setup fluffypony v0.14.0 +GH_USER=fluffypony +VERSION=v0.14.1.0 + +./gitian-build.py --setup $GH_USER $VERSION ``` -Where `fluffypony` is your Github name and `v0.14.0` is the version tag you want to build. -If you are using docker, run it with: +Where `GH_USER` is your Github user name and `VERSION` is the version tag you want to build. + +Setup for docker: ```bash -./gitian-build.py --setup --docker fluffypony v0.14.0 +./gitian-build.py --setup --docker $GH_USER $VERSION ``` -While gitian and this build script does provide a way for you to sign the build directly, it is recommended to sign in a seperate step. -This script is only there for convenience. Seperate steps for building can still be taken. +While gitian and this build script does provide a way for you to sign the build directly, it is recommended to sign in a separate step. This script is only there for convenience. Separate steps for building can still be taken. In order to sign gitian builds on your host machine, which has your PGP key, -fork the gitian.sigs repository and clone it on your host machine, +fork the [gitian.sigs repository](https://github.com/monero-project/gitian.sigs) and clone it on your host machine, or pass the signed assert file back to your build machine. -``` +```bash git clone git@github.com:monero-project/gitian.sigs.git -git remote add fluffypony git@github.com:fluffypony/gitian.sigs.git +git remote add $GH_USER git@github.com:$GH_USER/gitian.sigs.git ``` -Build Binaries ------------------------------ -To build the most recent tag (pass in `--docker` after setting up with docker): +Build the binaries +------------------ + +**Note:** if you intend to build MacOS binaries, please follow [these instructions](https://github.com/bitcoin-core/docs/blob/master/gitian-building/gitian-building-mac-os-sdk.md) to get the required SDK. + +To build the most recent tag (pass in `--docker` if using docker): ```bash -./gitian-build.py --detach-sign --no-commit -b fluffypony v0.14.0 +./gitian-build.py --detach-sign --no-commit --build $GH_USER $VERSION +``` + +To speed up the build, use `-j 5 --memory 5000` as the first arguments, where `5` is the number of CPU's you allocated to the VM plus one, and 5000 is a little bit less than then the MB's of RAM you allocated. If there is memory corruption on your machine, try to tweak these values. + +If all went well, this produces a number of (uncommitted) `.assert` files in the gitian.sigs directory. + +Checking your work +------------------ + +Take a look in the assert files and note the SHA256 checksums listed there. eg for `v0.14.1.0` you should get this checksum: + ``` +2b95118f53d98d542a85f8732b84ba13b3cd20517ccb40332b0edd0ddf4f8c62 monero-x86_64-linux-gnu.tar.gz +``` + +You should verify that this is really the checksum you get on that file you built. You can also look in the gitian.sigs repo and / or [getmonero.org release checksums](https://web.getmonero.org/downloads/hashes.txt) to see if others got the same checksum for the same version tag. If there is ever a mismatch -- **STOP! Something is wrong**. Contact others on IRC / github to figure out what is going on. -To speed up the build, use `-j 5 -m 5000` as the first arguments, where `5` is the number of CPU's you allocated to the VM plus one, and 5000 is a little bit less than then the MB's of RAM you allocated. If there is memory corruption on your machine, try to tweak these values. -If all went well, this produces a number of (uncommited) `.assert` files in the gitian.sigs repository. +Signing assert files +-------------------- -If you do detached, offline signing, you need to copy these uncommited changes to your host machine, where you can sign them. For example: +If you chose to do detached signing using `--detach-sign` above (recommended), you need to copy these uncommitted changes to your host machine, then sign them using your gpg key like so: ```bash -export NAME=fluffypony -export VERSION=v0.14.0 -gpg --output $VERSION-linux/$NAME/monero-linux-$VERSION-build.assert.sig --detach-sign $VERSION-linux/$NAME/monero-linux-$VERSION-build.assert -gpg --output $VERSION-osx-unsigned/$NAME/monero-osx-$VERSION-build.assert.sig --detach-sign $VERSION-osx-unsigned/$NAME/monero-osx-$VERSION-build.assert -gpg --output $VERSION-win-unsigned/$NAME/monero-win-$VERSION-build.assert.sig --detach-sign $VERSION-win-unsigned/$NAME/monero-win-$VERSION-build.assert +GH_USER=fluffypony +VERSION=v0.14.1.0 + +gpg --detach-sign ${VERSION}-linux/${GH_USER}/monero-linux-*-build.assert +gpg --detach-sign ${VERSION}-win/${GH_USER}/monero-win-*-build.assert +gpg --detach-sign ${VERSION}-osx/${GH_USER}/monero-osx-*-build.assert ``` +<!-- TODO: Replace * above with ${VERSION} once gitian builds correct file name --> + +This will create a `.sig` file for each `.assert` file above (2 files for each platform). + + +Submitting your signed assert files +----------------------------------- Make a pull request (both the `.assert` and `.assert.sig` files) to the [monero-project/gitian.sigs](https://github.com/monero-project/gitian.sigs/) repository: ```bash -git checkout -b v0.14.0 -git commit -S -a -m "Add $NAME v0.14.0" -git push --set-upstream $NAME v0.14.0 +git checkout -b $VERSION +# add your assert and sig files... +git commit -S -a -m "Add $GH_USER $VERSION" +git push --set-upstream $GH_USER $VERSION ``` -```bash - gpg --detach-sign ${VERSION}-linux/${SIGNER}/monero-linux-*-build.assert - gpg --detach-sign ${VERSION}-win-unsigned/${SIGNER}/monero-win-*-build.assert - gpg --detach-sign ${VERSION}-osx-unsigned/${SIGNER}/monero-osx-*-build.assert -``` +**Note:** Please ensure your gpg public key is available to check signatures by adding it to the [gitian.sigs/gitian-pubkeys/](https://github.com/monero-project/gitian.sigs/tree/master/gitian-pubkeys) directory in a pull request. + More Build Options ------------------ diff --git a/contrib/gitian/gitian-build.py b/contrib/gitian/gitian-build.py index cd88ecb20..b654b15c7 100755 --- a/contrib/gitian/gitian-build.py +++ b/contrib/gitian/gitian-build.py @@ -7,21 +7,13 @@ import sys def setup(): global args, workdir - programs = ['ruby', 'git', 'apt-cacher-ng', 'make', 'wget'] + programs = ['apt-cacher-ng', 'ruby', 'git', 'make', 'wget'] if args.kvm: programs += ['python-vm-builder', 'qemu-kvm', 'qemu-utils'] - elif args.docker: - dockers = ['docker.io', 'docker-ce'] - for i in dockers: - return_code = subprocess.call(['sudo', 'apt-get', 'install', '-qq', i]) - if return_code == 0: - break - if return_code != 0: - print('Cannot find any way to install docker', file=sys.stderr) - exit(1) else: programs += ['lxc', 'debootstrap'] - subprocess.check_call(['sudo', 'apt-get', 'install', '-qq'] + programs) + if not args.no_apt: + subprocess.check_call(['sudo', 'apt-get', 'install', '-qq'] + programs) if not os.path.isdir('gitian.sigs'): subprocess.check_call(['git', 'clone', 'https://github.com/monero-project/gitian.sigs.git']) if not os.path.isdir('gitian-builder'): @@ -32,6 +24,8 @@ def setup(): subprocess.check_call(['git', 'checkout', '963322de8420c50502c4cc33d4d7c0d84437b576']) make_image_prog = ['bin/make-base-vm', '--suite', 'bionic', '--arch', 'amd64'] if args.docker: + if not subprocess.call(['docker', '--help'], shell=False, stdout=subprocess.DEVNULL): + print("Please install docker first manually") make_image_prog += ['--docker'] elif not args.kvm: make_image_prog += ['--lxc'] @@ -40,7 +34,7 @@ def setup(): if args.is_bionic and not args.kvm and not args.docker: subprocess.check_call(['sudo', 'sed', '-i', 's/lxcbr0/br0/', '/etc/default/lxc-net']) print('Reboot is required') - exit(0) + sys.exit(0) def build(): global args, workdir @@ -100,7 +94,7 @@ def verify(): def main(): global args, workdir - parser = argparse.ArgumentParser(usage='%(prog)s [options] signer version') + parser = argparse.ArgumentParser(description='Script for running full Gitian builds.', usage='%(prog)s [options] signer version') parser.add_argument('-c', '--commit', action='store_true', dest='commit', help='Indicate that the version argument is for a commit or branch') parser.add_argument('-p', '--pull', action='store_true', dest='pull', help='Indicate that the version argument is the number of a github repository pull request') parser.add_argument('-u', '--url', dest='url', default='https://github.com/monero-project/monero', help='Specify the URL of the repository. Default is %(default)s') @@ -112,11 +106,12 @@ def main(): parser.add_argument('-m', '--memory', dest='memory', default='2000', help='Memory to allocate in MiB. Default %(default)s') parser.add_argument('-k', '--kvm', action='store_true', dest='kvm', help='Use KVM instead of LXC') parser.add_argument('-d', '--docker', action='store_true', dest='docker', help='Use Docker instead of LXC') - parser.add_argument('-S', '--setup', action='store_true', dest='setup', help='Set up the Gitian building environment. Uses LXC. If you want to use KVM, use the --kvm option. Only works on Debian-based systems (Ubuntu, Debian)') + parser.add_argument('-S', '--setup', action='store_true', dest='setup', help='Set up the Gitian building environment. Uses LXC. If you want to use KVM, use the --kvm option. If you run this script on a non-debian based system, pass the --no-apt flag') parser.add_argument('-D', '--detach-sign', action='store_true', dest='detach_sign', help='Create the assert file for detached signing. Will not commit anything.') parser.add_argument('-n', '--no-commit', action='store_false', dest='commit_files', help='Do not commit anything to git') - parser.add_argument('signer', help='GPG signer to sign each build assert file') - parser.add_argument('version', help='Version number, commit, or branch to build.') + parser.add_argument('signer', nargs='?', help='GPG signer to sign each build assert file') + parser.add_argument('version', nargs='?', help='Version number, commit, or branch to build.') + parser.add_argument('-a', '--no-apt', action='store_true', dest='no_apt', help='Indicate that apt is not installed on the system') args = parser.parse_args() workdir = os.getcwd() @@ -128,8 +123,8 @@ def main(): args.is_bionic = b'bionic' in subprocess.check_output(['lsb_release', '-cs']) if args.buildsign: - args.build=True - args.sign=True + args.build = True + args.sign = True if args.kvm and args.docker: raise Exception('Error: cannot have both kvm and docker') @@ -156,11 +151,11 @@ def main(): if args.signer == '': print(script_name+': Missing signer.') print('Try '+script_name+' --help for more information') - exit(1) + sys.exit(1) if args.version == '': print(script_name+': Missing version.') print('Try '+script_name+' --help for more information') - exit(1) + sys.exit(1) # Add leading 'v' for tags if args.commit and args.pull: diff --git a/contrib/gitian/gitian-linux.yml b/contrib/gitian/gitian-linux.yml index 67f174fec..fd94d43bf 100644 --- a/contrib/gitian/gitian-linux.yml +++ b/contrib/gitian/gitian-linux.yml @@ -129,7 +129,15 @@ script: | chmod +x ${WRAP_DIR}/${prog} done + git config --global core.abbrev 9 cd monero + # Set the version string that gets added to the tar archive name + version="`git describe`" + if [[ $version == *"-"*"-"* ]]; then + version="`git rev-parse --short=9 HEAD`" + version="`echo $version | head -c 9`" + fi + BASEPREFIX=`pwd`/contrib/depends # Build dependencies for each host for i in $HOSTS; do @@ -153,10 +161,11 @@ script: | export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir build && cd build cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DBACKCOMPAT=ON - make - DISTNAME=monero-${i} + make ${MAKEOPTS} + DISTNAME=monero-${i}-${version} mv bin ${DISTNAME} find ${DISTNAME}/ | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}.tar.gz cd .. rm -rf build done + diff --git a/contrib/gitian/gitian-osx.yml b/contrib/gitian/gitian-osx.yml index 7de302353..77ea30072 100644 --- a/contrib/gitian/gitian-osx.yml +++ b/contrib/gitian/gitian-osx.yml @@ -77,7 +77,15 @@ script: | create_per-host_faketime_wrappers "2000-01-01 12:00:00" export PATH=${WRAP_DIR}:${PATH} + git config --global core.abbrev 9 cd monero + # Set the version string that gets added to the tar archive name + version="`git describe`" + if [[ $version == *"-"*"-"* ]]; then + version="`git rev-parse --short=9 HEAD`" + version="`echo $version | head -c 9`" + fi + BASEPREFIX=`pwd`/contrib/depends mkdir -p ${BASEPREFIX}/SDKs @@ -100,8 +108,8 @@ script: | export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir build && cd build cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake - make - DISTNAME=monero-${i} + make ${MAKEOPTS} + DISTNAME=monero-${i}-${version} mv bin ${DISTNAME} find ${DISTNAME}/ | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}.tar.gz cd .. diff --git a/contrib/gitian/gitian-win.yml b/contrib/gitian/gitian-win.yml index 1eb558300..4c559acfe 100644 --- a/contrib/gitian/gitian-win.yml +++ b/contrib/gitian/gitian-win.yml @@ -100,7 +100,15 @@ script: | create_per-host_linker_wrapper "2000-01-01 12:00:00" export PATH=${WRAP_DIR}:${PATH} + git config --global core.abbrev 9 cd monero + # Set the version string that gets added to the tar archive name + version="`git describe`" + if [[ $version == *"-"*"-"* ]]; then + version="`git rev-parse --short=9 HEAD`" + version="`echo $version | head -c 9`" + fi + BASEPREFIX=`pwd`/contrib/depends # Build dependencies for each host for i in $HOSTS; do @@ -125,8 +133,8 @@ script: | export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir build && cd build cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake - make - DISTNAME=monero-${i} + make ${MAKEOPTS} + DISTNAME=monero-${i}-${version} mv bin ${DISTNAME} find ${DISTNAME}/ | sort | zip -X@ ${OUTDIR}/${DISTNAME}.zip cd .. && rm -rf build diff --git a/contrib/rlwrap/monerocommands_bitmonerod.txt b/contrib/rlwrap/monerocommands_bitmonerod.txt index c4f77b37d..e06e4f4f1 100644 --- a/contrib/rlwrap/monerocommands_bitmonerod.txt +++ b/contrib/rlwrap/monerocommands_bitmonerod.txt @@ -27,10 +27,8 @@ save set_log show_hr start_mining -start_save_graph status stop_daemon stop_mining -stop_save_graph sync_info unban |