diff options
| -rw-r--r-- | contrib/depends/packages/libiconv.mk | 15 | ||||
| -rw-r--r-- | contrib/depends/packages/sodium.mk | 2 | ||||
| -rw-r--r-- | contrib/depends/patches/libiconv/fix-whitespace.patch | 13 | ||||
| -rw-r--r-- | contrib/depends/patches/sodium/fix-whitespace.patch | 13 | ||||
| -rw-r--r-- | contrib/gitian/README.md | 146 | ||||
| -rwxr-xr-x | contrib/gitian/gitian-build.py | 192 | ||||
| -rw-r--r-- | contrib/gitian/gitian-linux.yml | 162 | ||||
| -rw-r--r-- | contrib/gitian/gitian-osx.yml | 114 | ||||
| -rw-r--r-- | contrib/gitian/gitian-win.yml | 134 | ||||
| -rwxr-xr-x | contrib/gitian/symbol-check.py | 163 |
10 files changed, 953 insertions, 1 deletions
diff --git a/contrib/depends/packages/libiconv.mk b/contrib/depends/packages/libiconv.mk index 87e30b208..dbcb28141 100644 --- a/contrib/depends/packages/libiconv.mk +++ b/contrib/depends/packages/libiconv.mk @@ -3,9 +3,22 @@ $(package)_version=1.15 $(package)_download_path=https://ftp.gnu.org/gnu/libiconv $(package)_file_name=libiconv-$($(package)_version).tar.gz $(package)_sha256_hash=ccf536620a45458d26ba83887a983b96827001e92a13847b45e4925cc8913178 +$(package)_patches=fix-whitespace.patch + +define $(package)_set_vars + $(package)_config_opts=--disable-nls + $(package)_config_opts=--enable-static + $(package)_config_opts=--disable-shared + $(package)_config_opts_linux=--with-pic +endef + +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub build-aux/ &&\ + patch -p1 < $($(package)_patch_dir)/fix-whitespace.patch +endef define $(package)_config_cmds - $($(package)_autoconf) --disable-nls --enable-static --disable-shared + $($(package)_autoconf) endef define $(package)_build_cmds diff --git a/contrib/depends/packages/sodium.mk b/contrib/depends/packages/sodium.mk index c38121bf7..35f444fd5 100644 --- a/contrib/depends/packages/sodium.mk +++ b/contrib/depends/packages/sodium.mk @@ -3,6 +3,7 @@ $(package)_version=1.0.15 $(package)_download_path=https://download.libsodium.org/libsodium/releases/ $(package)_file_name=libsodium-$($(package)_version).tar.gz $(package)_sha256_hash=fb6a9e879a2f674592e4328c5d9f79f082405ee4bb05cb6e679b90afe9e178f4 +$(package)_patches=fix-whitespace.patch define $(package)_set_vars $(package)_config_opts=--enable-static --disable-shared @@ -11,6 +12,7 @@ endef define $(package)_config_cmds ./autogen.sh &&\ + patch -p1 < $($(package)_patch_dir)/fix-whitespace.patch &&\ $($(package)_autoconf) $($(package)_config_opts) endef diff --git a/contrib/depends/patches/libiconv/fix-whitespace.patch b/contrib/depends/patches/libiconv/fix-whitespace.patch new file mode 100644 index 000000000..531364b45 --- /dev/null +++ b/contrib/depends/patches/libiconv/fix-whitespace.patch @@ -0,0 +1,13 @@ +diff --git a/preload/configure b/preload/configure +index aab5c77..e20b8f0 100755 +--- a/preload/configure ++++ b/preload/configure +@@ -588,7 +588,7 @@ MAKEFLAGS= + PACKAGE_NAME='libiconv' + PACKAGE_TARNAME='libiconv' + PACKAGE_VERSION='0' +-PACKAGE_STRING='libiconv 0' ++PACKAGE_STRING='libiconv0' + PACKAGE_BUGREPORT='' + PACKAGE_URL='' + diff --git a/contrib/depends/patches/sodium/fix-whitespace.patch b/contrib/depends/patches/sodium/fix-whitespace.patch new file mode 100644 index 000000000..c11838611 --- /dev/null +++ b/contrib/depends/patches/sodium/fix-whitespace.patch @@ -0,0 +1,13 @@ +diff --git a/configure b/configure +index b29f769..ca008ae 100755 +--- a/configure ++++ b/configure +@@ -591,7 +591,7 @@ MAKEFLAGS= + PACKAGE_NAME='libsodium' + PACKAGE_TARNAME='libsodium' + PACKAGE_VERSION='1.0.15' +-PACKAGE_STRING='libsodium 1.0.15' ++PACKAGE_STRING='libsodium' + PACKAGE_BUGREPORT='https://github.com/jedisct1/libsodium/issues' + PACKAGE_URL='https://github.com/jedisct1/libsodium' + diff --git a/contrib/gitian/README.md b/contrib/gitian/README.md new file mode 100644 index 000000000..4bd326f22 --- /dev/null +++ b/contrib/gitian/README.md @@ -0,0 +1,146 @@ +Gitian building +================ + +*Setup instructions for a Gitian build of Monero using a VM or physical system.* + +Gitian is the deterministic build process that is used to build the Monero CLI +executables. It provides a way to be reasonably sure that the +executables are really built from the git source. It also makes sure that +the same, tested dependencies are used and statically built into the executable. + +Multiple developers build the source code by following a specific descriptor +("recipe"), cryptographically sign the result, and upload the resulting signature. +These results are compared and only if they match, the build is accepted and provided +for download. + +More independent Gitian builders are needed, which is why this guide exists. +It is preferred you follow these steps yourself instead of using someone else's +VM image to avoid 'contaminating' the build. + +Table of Contents +------------------ + +Please note that these instructions have been forked from bitcoin's gitian build +instructions. Please also consult their documentation, when running into problems. +The signing is left as inherited from bitcoin at the moment. + +- [Preparing the Gitian builder host](#preparing-the-gitian-builder-host) +- [Getting and building the inputs](#getting-and-building-the-inputs) +- [Building Binaries](#building-bitcoin-core) +- [Signing externally](#signing-externally) +- [Uploading signatures](#uploading-signatures) + +Preparing the Gitian builder host +--------------------------------- + +The first step is to prepare the host environment that will be used to perform the Gitian builds. +This guide explains how to set up the environment, and how to start the builds. + +Gitian builds are for now executed on Ubuntu 18.04 "Bionic Beaver". A solution is being worked on to run +it in docker in the future. Please run Ubuntu in either a VM, or on your physical machine. +You need to be logged in as the `gitianuser` in order to build gitian builds. If this user does not exist yet on your system, +create it. + +Note that a version of `lxc-execute` higher or equal to 2.1.1 is required. +You can check the version with `lxc-execute --version`. + +First we need to set up dependencies. Type/paste the following in the terminal: + +```bash +sudo apt-get install git ruby apt-cacher-ng qemu-utils debootstrap lxc python-cheetah parted kpartx bridge-utils make ubuntu-archive-keyring curl firewalld +``` + +Then set up LXC and the rest with the following, which is a complex jumble of settings and workarounds: + +```bash +sudo -s +# the version of lxc-start in Debian needs to run as root, so make sure +# that the build script can execute it without providing a password +echo "%sudo ALL=NOPASSWD: /usr/bin/lxc-start" > /etc/sudoers.d/gitian-lxc +echo "%sudo ALL=NOPASSWD: /usr/bin/lxc-execute" >> /etc/sudoers.d/gitian-lxc +# make /etc/rc.local script that sets up bridge between guest and host +echo '#!/bin/sh -e' > /etc/rc.local +echo 'brctl addbr br0' >> /etc/rc.local +echo 'ip addr add 10.0.3.1/24 broadcast 10.0.3.255 dev br0' >> /etc/rc.local +echo 'ip link set br0 up' >> /etc/rc.local +echo 'firewall-cmd --zone=trusted --add-interface=br0' >> /etc/rc.local +echo 'exit 0' >> /etc/rc.local +chmod +x /etc/rc.local +# make sure that USE_LXC is always set when logging in as gitianuser, +# and configure LXC IP addresses +echo 'export USE_LXC=1' >> /home/gitianuser/.profile +echo 'export GITIAN_HOST_IP=10.0.3.1' >> /home/gitianuser/.profile +echo 'export LXC_GUEST_IP=10.0.3.5' >> /home/gitianuser/.profile +reboot +``` + +This setup is required to enable networking in the container. + + +Manual and Building +------------------- +The instructions below use the automated script [gitian-build.py](https://github.com/betcoin/bitcoin/blob/master/contrib/gitian-build.py) which only works in Ubuntu. +It calls all available descriptors. Help for the build steps taken can be accessed with `./gitian-build.py --help`. + +Initial Gitian Setup +-------------------- +The `gitian-build.py` script will checkout different release tags, so it's best to copy it: + +```bash +cp monero/contrib/gitian/gitian-build.py . +``` + +Setup the required environment, you only need to do this once: + +``` +./gitian-build.py --setup fluffypony 0.14.0 +``` + +Where `fluffypony` is your Github name and `0.14.0` is the version tag you want to build (without `v`). + +While gitian and this build script does provide a way for you to sign the build directly, it is recommended to sign in a seperate step. +This script is only there for convenience. Seperate steps for building can still be taken. +In order to sign gitian builds on your host machine, which has your PGP key, +fork the gitian.sigs repository and clone it on your host machine, +or pass the signed assert file back to your build machine. + +``` +git clone git@github.com:monero-project/gitian.sigs.git +git remote add fluffypony git@github.com:fluffypony/gitian.sigs.git +``` + +Build Binaries +----------------------------- +To build the most recent tag: + + `./gitian-build.py --detach-sign --no-commit -b fluffypony 0.14.0` + +To speed up the build, use `-j 5 -m 5000` as the first arguments, where `5` is the number of CPU's you allocated to the VM plus one, and 5000 is a little bit less than then the MB's of RAM you allocated. If there is memory corruption on your machine, try to tweak these values. + +If all went well, this produces a number of (uncommited) `.assert` files in the gitian.sigs repository. + +If you do detached, offline signing, you need to copy these uncommited changes to your host machine, where you can sign them. For example: + +``` +export NAME=fluffypony +export VERSION=0.14 +gpg --output $VERSION-linux/$NAME/monero-linux-$VERSION-build.assert.sig --detach-sign $VERSION-linux/$NAME/monero-linux-$VERSION-build.assert +gpg --output $VERSION-osx-unsigned/$NAME/monero-osx-$VERSION-build.assert.sig --detach-sign $VERSION-osx-unsigned/$NAME/monero-osx-$VERSION-build.assert +gpg --output $VERSION-win-unsigned/$NAME/monero-win-$VERSION-build.assert.sig --detach-sign $VERSION-win-unsigned/$NAME/monero-win-$VERSION-build.assert +``` + +Make a pull request (both the `.assert` and `.assert.sig` files) to the +[monero-project/gitian.sigs](https://github.com/monero-project/gitian.sigs/) repository: + +``` +git checkout -b 0.14.0 +git commit -S -a -m "Add $NAME 0.14.0" +git push --set-upstream $NAME 0.14.0 +``` + +```bash + gpg --detach-sign ${VERSION}-linux/${SIGNER}/monero-linux-*-build.assert + gpg --detach-sign ${VERSION}-win-unsigned/${SIGNER}/monero-win-*-build.assert + gpg --detach-sign ${VERSION}-osx-unsigned/${SIGNER}/monero-osx-*-build.assert +``` + diff --git a/contrib/gitian/gitian-build.py b/contrib/gitian/gitian-build.py new file mode 100755 index 000000000..99c64e9dd --- /dev/null +++ b/contrib/gitian/gitian-build.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 + +import argparse +import os +import subprocess +import sys + +def setup(): + global args, workdir + programs = ['ruby', 'git', 'apt-cacher-ng', 'make', 'wget'] + if args.kvm: + programs += ['python-vm-builder', 'qemu-kvm', 'qemu-utils'] + elif args.docker: + dockers = ['docker.io', 'docker-ce'] + for i in dockers: + return_code = subprocess.call(['sudo', 'apt-get', 'install', '-qq', i]) + if return_code == 0: + break + if return_code != 0: + print('Cannot find any way to install docker', file=sys.stderr) + exit(1) + else: + programs += ['lxc', 'debootstrap'] + subprocess.check_call(['sudo', 'apt-get', 'install', '-qq'] + programs) + if not os.path.isdir('gitian.sigs'): + subprocess.check_call(['git', 'clone', 'https://github.com/monero-project/gitian.sigs.git']) + if not os.path.isdir('gitian-builder'): + subprocess.check_call(['git', 'clone', 'https://github.com/devrandom/gitian-builder.git']) + if not os.path.isdir('monero'): + subprocess.check_call(['git', 'clone', 'https://github.com/monero-project/monero.git']) + os.chdir('gitian-builder') + subprocess.check_call(['git', 'checkout', '963322de8420c50502c4cc33d4d7c0d84437b576']) + make_image_prog = ['bin/make-base-vm', '--suite', 'bionic', '--arch', 'amd64'] + if args.docker: + make_image_prog += ['--docker'] + elif not args.kvm: + make_image_prog += ['--lxc'] + subprocess.check_call(make_image_prog) + os.chdir(workdir) + if args.is_bionic and not args.kvm and not args.docker: + subprocess.check_call(['sudo', 'sed', '-i', 's/lxcbr0/br0/', '/etc/default/lxc-net']) + print('Reboot is required') + exit(0) + +def build(): + global args, workdir + + os.makedirs('monero-binaries/' + args.version, exist_ok=True) + print('\nBuilding Dependencies\n') + os.chdir('gitian-builder') + os.makedirs('inputs', exist_ok=True) + + subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://downloads.sourceforge.net/project/osslsigncode/osslsigncode/osslsigncode-1.7.1.tar.gz']) + subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/cfields/osslsigncode-Backports-to-1.7.1.patch']) + subprocess.check_output(["echo 'a8c4e9cafba922f89de0df1f2152e7be286aba73f78505169bc351a7938dd911 inputs/osslsigncode-Backports-to-1.7.1.patch' | sha256sum -c"], shell=True) + subprocess.check_output(["echo 'f9a8cdb38b9c309326764ebc937cba1523a3a751a7ab05df3ecc99d18ae466c9 inputs/osslsigncode-1.7.1.tar.gz' | sha256sum -c"], shell=True) + subprocess.check_call(['make', '-C', '../monero/contrib/depends', 'download', 'SOURCES_PATH=' + os.getcwd() + '/cache/common']) + + if args.linux: + print('\nCompiling ' + args.version + ' Linux') + subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero='+args.commit, '--url', 'monero='+args.url, '../monero/contrib/gitian/gitian-linux.yml']) + subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-linux', '--destination', '../gitian.sigs/', '../monero/contrib/gitian/gitian-linux.yml']) + subprocess.check_call('mv build/out/monero-*.tar.gz ../monero-binaries/'+args.version, shell=True) + + if args.windows: + print('\nCompiling ' + args.version + ' Windows') + subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero='+args.commit, '--url', 'monero='+args.url, '../monero/contrib/gitian/gitian-win.yml']) + subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-win', '--destination', '../gitian.sigs/', '../monero/contrib/gitian/gitian-win.yml']) + subprocess.check_call('mv build/out/monero*.zip ../monero-binaries/'+args.version, shell=True) + + if args.macos: + print('\nCompiling ' + args.version + ' MacOS') + subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero='+args.commit, '--url', 'monero'+args.url, '../monero/contrib/gitian/gitian-osx.yml']) + subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx', '--destination', '../gitian.sigs/', '../monero/contrib/gitian/gitian-osx.yml']) + subprocess.check_call('mv build/out/monero*.tar.gz ../monero-binaries/'+args.version, shell=True) + + os.chdir(workdir) + + if args.commit_files: + print('\nCommitting '+args.version+' Unsigned Sigs\n') + os.chdir('gitian.sigs') + subprocess.check_call(['git', 'add', args.version+'-linux/'+args.signer]) + subprocess.check_call(['git', 'add', args.version+'-win/'+args.signer]) + subprocess.check_call(['git', 'add', args.version+'-osx/'+args.signer]) + subprocess.check_call(['git', 'commit', '-m', 'Add '+args.version+' unsigned sigs for '+args.signer]) + os.chdir(workdir) + +def verify(): + global args, workdir + os.chdir('gitian-builder') + + print('\nVerifying v'+args.version+' Linux\n') + subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-linux', '../monero/contrib/gitian/gitian-linux.yml']) + print('\nVerifying v'+args.version+' Windows\n') + subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win', '../monero/contrib/gitian/gitian-win.yml']) + print('\nVerifying v'+args.version+' MacOS\n') + subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx', '../monero/contrib/gitian/gitian-osx.yml']) + os.chdir(workdir) + +def main(): + global args, workdir + + parser = argparse.ArgumentParser(usage='%(prog)s [options] signer version') + parser.add_argument('-c', '--commit', action='store_true', dest='commit', help='Indicate that the version argument is for a commit or branch') + parser.add_argument('-p', '--pull', action='store_true', dest='pull', help='Indicate that the version argument is the number of a github repository pull request') + parser.add_argument('-u', '--url', dest='url', default='https://github.com/monero-project/monero', help='Specify the URL of the repository. Default is %(default)s') + parser.add_argument('-v', '--verify', action='store_true', dest='verify', help='Verify the Gitian build') + parser.add_argument('-b', '--build', action='store_true', dest='build', help='Do a Gitian build') + parser.add_argument('-B', '--buildsign', action='store_true', dest='buildsign', help='Build both signed and unsigned binaries') + parser.add_argument('-o', '--os', dest='os', default='lwm', help='Specify which Operating Systems the build is for. Default is %(default)s. l for Linux, w for Windows, m for MacOS') + parser.add_argument('-j', '--jobs', dest='jobs', default='2', help='Number of processes to use. Default %(default)s') + parser.add_argument('-m', '--memory', dest='memory', default='2000', help='Memory to allocate in MiB. Default %(default)s') + parser.add_argument('-k', '--kvm', action='store_true', dest='kvm', help='Use KVM instead of LXC') + parser.add_argument('-d', '--docker', action='store_true', dest='docker', help='Use Docker instead of LXC') + parser.add_argument('-S', '--setup', action='store_true', dest='setup', help='Set up the Gitian building environment. Uses LXC. If you want to use KVM, use the --kvm option. Only works on Debian-based systems (Ubuntu, Debian)') + parser.add_argument('-D', '--detach-sign', action='store_true', dest='detach_sign', help='Create the assert file for detached signing. Will not commit anything.') + parser.add_argument('-n', '--no-commit', action='store_false', dest='commit_files', help='Do not commit anything to git') + parser.add_argument('signer', help='GPG signer to sign each build assert file') + parser.add_argument('version', help='Version number, commit, or branch to build. If building a commit or branch, the -c option must be specified') + + args = parser.parse_args() + workdir = os.getcwd() + + args.linux = 'l' in args.os + args.windows = 'w' in args.os + args.macos = 'm' in args.os + + args.is_bionic = b'bionic' in subprocess.check_output(['lsb_release', '-cs']) + + if args.buildsign: + args.build=True + args.sign=True + + if args.kvm and args.docker: + raise Exception('Error: cannot have both kvm and docker') + + args.sign_prog = 'true' if args.detach_sign else 'gpg --detach-sign' + + # Set enviroment variable USE_LXC or USE_DOCKER, let gitian-builder know that we use lxc or docker + if args.docker: + os.environ['USE_DOCKER'] = '1' + elif not args.kvm: + os.environ['USE_LXC'] = '1' + if not 'GITIAN_HOST_IP' in os.environ.keys(): + os.environ['GITIAN_HOST_IP'] = '10.0.3.1' + if not 'LXC_GUEST_IP' in os.environ.keys(): + os.environ['LXC_GUEST_IP'] = '10.0.3.5' + + # Disable for MacOS if no SDK found + if args.macos and not os.path.isfile('gitian-builder/inputs/MacOSX10.11.sdk.tar.gz'): + print('Cannot build for MacOS, SDK does not exist. Will build for other OSes') + args.macos = False + + script_name = os.path.basename(sys.argv[0]) + # Signer and version shouldn't be empty + if args.signer == '': + print(script_name+': Missing signer.') + print('Try '+script_name+' --help for more information') + exit(1) + if args.version == '': + print(script_name+': Missing version.') + print('Try '+script_name+' --help for more information') + exit(1) + + # Add leading 'v' for tags + if args.commit and args.pull: + raise Exception('Cannot have both commit and pull') + args.commit = ('' if args.commit else) + args.version + + if args.setup: + setup() + + os.chdir('monero') + if args.pull: + subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge']) + os.chdir('../gitian-builder/inputs/monero') + subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge']) + args.commit = subprocess.check_output(['git', 'show', '-s', '--format=%H', 'FETCH_HEAD'], universal_newlines=True).strip() + args.version = 'pull-' + args.version + print(args.commit) + subprocess.check_call(['git', 'fetch']) + subprocess.check_call(['git', 'checkout', args.commit]) + os.chdir(workdir) + + if args.build: + build() + + if args.verify: + verify() + +if __name__ == '__main__': + main() diff --git a/contrib/gitian/gitian-linux.yml b/contrib/gitian/gitian-linux.yml new file mode 100644 index 000000000..473a7720d --- /dev/null +++ b/contrib/gitian/gitian-linux.yml @@ -0,0 +1,162 @@ +--- +name: "monero-linux-0.14" +enable_cache: true +suites: +- "bionic" +architectures: +- "amd64" +packages: +- "curl" +- "gperf" +- "gcc-7" +- "g++-7" +- "gcc" +- "g++" +- "gcc-7-aarch64-linux-gnu" +- "g++-7-aarch64-linux-gnu" +- "gcc-aarch64-linux-gnu" +- "g++-aarch64-linux-gnu" +- "binutils-aarch64-linux-gnu" +- "gcc-7-arm-linux-gnueabihf" +- "g++-7-arm-linux-gnueabihf" +- "gcc-arm-linux-gnueabihf" +- "g++-arm-linux-gnueabihf" +- "g++-7-multilib" +- "gcc-7-multilib" +- "binutils-arm-linux-gnueabihf" +- "binutils-gold" +- "git" +- "pkg-config" +- "build-essential" +- "autoconf" +- "libtool" +- "automake" +- "faketime" +- "bsdmainutils" +- "ca-certificates" +- "python" +- "cmake" +- "ccache" +- "protobuf-compiler" +- "libdbus-1-dev" +- "libharfbuzz-dev" +- "libprotobuf-dev" +- "python3-zmq" +remotes: +- "url": "https://github.com/monero-project/monero.git" + "dir": "monero" +files: [] +script: | + + WRAP_DIR=$HOME/wrapped + HOSTS="x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu i686-linux-gnu" + FAKETIME_HOST_PROGS="gcc g++" + FAKETIME_PROGS="date ar ranlib nm" + HOST_CFLAGS="-O2 -g" + HOST_CXXFLAGS="-O2 -g" + HOST_LDFLAGS=-static-libstdc++ + + export GZIP="-9n" + export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" + export TZ="UTC" + export BUILD_DIR=`pwd` + mkdir -p ${WRAP_DIR} + if test -n "$GBUILD_CACHE_ENABLED"; then + export SOURCES_PATH=${GBUILD_COMMON_CACHE} + export BASE_CACHE=${GBUILD_PACKAGE_CACHE} + mkdir -p ${BASE_CACHE} ${SOURCES_PATH} + fi + + function create_global_faketime_wrappers { + for prog in ${FAKETIME_PROGS}; do + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} + echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${prog} + chmod +x ${WRAP_DIR}/${prog} + done + } + + function create_per-host_faketime_wrappers { + for i in $HOSTS; do + for prog in ${FAKETIME_HOST_PROGS}; do + if which ${i}-${prog}-7 + then + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} + echo "REAL=\`which -a ${i}-${prog}-7 | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} + chmod +x ${WRAP_DIR}/${i}-${prog} + fi + done + done + } + + # Faketime for depends so intermediate results are comparable + export PATH_orig=${PATH} + create_global_faketime_wrappers "2000-01-01 12:00:00" + create_per-host_faketime_wrappers "2000-01-01 12:00:00" + export PATH=${WRAP_DIR}:${PATH} + + EXTRA_INCLUDES_BASE=$WRAP_DIR/extra_includes + mkdir -p $EXTRA_INCLUDES_BASE + + # x86 needs /usr/include/i386-linux-gnu/asm pointed to /usr/include/x86_64-linux-gnu/asm, + # but we can't write there. Instead, create a link here and force it to be included in the + # search paths by wrapping gcc/g++. + + mkdir -p $EXTRA_INCLUDES_BASE/i686-pc-linux-gnu + rm -f $WRAP_DIR/extra_includes/i686-pc-linux-gnu/asm + ln -s /usr/include/x86_64-linux-gnu/asm $EXTRA_INCLUDES_BASE/i686-pc-linux-gnu/asm + + for prog in gcc g++; do + rm -f ${WRAP_DIR}/${prog} + cat << EOF > ${WRAP_DIR}/${prog} + #!/usr/bin/env bash + REAL="`which -a ${prog}-7 | grep -v ${WRAP_DIR}/${prog} | head -1`" + for var in "\$@" + do + if [ "\$var" = "-m32" ]; then + export C_INCLUDE_PATH="$EXTRA_INCLUDES_BASE/i686-pc-linux-gnu" + export CPLUS_INCLUDE_PATH="$EXTRA_INCLUDES_BASE/i686-pc-linux-gnu" + break + fi + done + \$REAL \$@ + EOF + chmod +x ${WRAP_DIR}/${prog} + done + + cd monero + BASEPREFIX=`pwd`/contrib/depends + # Build dependencies for each host + for i in $HOSTS; do + EXTRA_INCLUDES="$EXTRA_INCLUDES_BASE/$i" + if [ -d "$EXTRA_INCLUDES" ]; then + export HOST_ID_SALT="$EXTRA_INCLUDES" + fi + make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" -j 4 V=1 + unset HOST_ID_SALT + done + + # Faketime for binaries + export PATH=${PATH_orig} + create_global_faketime_wrappers "${REFERENCE_DATETIME}" + create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" + export PATH=${WRAP_DIR}:${PATH} + + ORIGPATH="$PATH" + # Build in a new dir for each host + for i in ${HOSTS}; do + export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} + mkdir build && cd build + cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake + make + DISTNAME=monero-${i} + mv bin ${DISTNAME} + find ${DISTNAME}/ | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}.tar.gz + cd .. + rm -rf build + done diff --git a/contrib/gitian/gitian-osx.yml b/contrib/gitian/gitian-osx.yml new file mode 100644 index 000000000..a6fcff0da --- /dev/null +++ b/contrib/gitian/gitian-osx.yml @@ -0,0 +1,114 @@ +--- +name: "monero-osx-0.14" +enable_cache: true +suites: +- "bionic" +architectures: +- "amd64" +packages: +- "ca-certificates" +- "curl" +- "g++" +- "git" +- "pkg-config" +- "autoconf" +- "librsvg2-bin" +- "libtiff-tools" +- "libtool" +- "automake" +- "faketime" +- "bsdmainutils" +- "cmake" +- "imagemagick" +- "libcap-dev" +- "libz-dev" +- "libbz2-dev" +- "python" +- "python-dev" +- "python-setuptools" +- "fonts-tuffy" +remotes: +- "url": "https://github.com/monero-project/monero.git" + "dir": "monero" +files: +- "MacOSX10.11.sdk.tar.gz" +script: | + WRAP_DIR=$HOME/wrapped + HOSTS="x86_64-apple-darwin11" + FAKETIME_HOST_PROGS="" + FAKETIME_PROGS="ar ranlib date dmg genisoimage" + + export GZIP="-9n" + export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" + export TZ="UTC" + export BUILD_DIR=`pwd` + mkdir -p ${WRAP_DIR} + if test -n "$GBUILD_CACHE_ENABLED"; then + export SOURCES_PATH=${GBUILD_COMMON_CACHE} + export BASE_CACHE=${GBUILD_PACKAGE_CACHE} + mkdir -p ${BASE_CACHE} ${SOURCES_PATH} + fi + + export ZERO_AR_DATE=1 + + function create_global_faketime_wrappers { + for prog in ${FAKETIME_PROGS}; do + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} + echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${prog} + chmod +x ${WRAP_DIR}/${prog} + done + } + + function create_per-host_faketime_wrappers { + for i in $HOSTS; do + for prog in ${FAKETIME_HOST_PROGS}; do + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} + echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} + chmod +x ${WRAP_DIR}/${i}-${prog} + done + done + } + + # Faketime for depends so intermediate results are comparable + export PATH_orig=${PATH} + create_global_faketime_wrappers "2000-01-01 12:00:00" + create_per-host_faketime_wrappers "2000-01-01 12:00:00" + export PATH=${WRAP_DIR}:${PATH} + + cd monero + BASEPREFIX=`pwd`/contrib/depends + + mkdir -p ${BASEPREFIX}/SDKs + tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/MacOSX10.11.sdk.tar.gz + + # Build dependencies for each host + for i in $HOSTS; do + make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" + done + + # Faketime for binaries + export PATH=${PATH_orig} + create_global_faketime_wrappers "${REFERENCE_DATETIME}" + create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" + export PATH=${WRAP_DIR}:${PATH} + + ORIGPATH="$PATH" + # Build in a new dir for each host + for i in ${HOSTS}; do + export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} + mkdir build && cd build + cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake + make + DISTNAME=monero-${i} + mv bin ${DISTNAME} + find ${DISTNAME}/ | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}.tar.gz + cd .. + rm -rf build + done + diff --git a/contrib/gitian/gitian-win.yml b/contrib/gitian/gitian-win.yml new file mode 100644 index 000000000..fef5567f9 --- /dev/null +++ b/contrib/gitian/gitian-win.yml @@ -0,0 +1,134 @@ +--- +name: "monero-win-0.14" +enable_cache: true +suites: +- "bionic" +architectures: +- "amd64" +packages: +- "curl" +- "g++" +- "git" +- "pkg-config" +- "autoconf" +- "libtool" +- "automake" +- "faketime" +- "bsdmainutils" +- "mingw-w64" +- "g++-mingw-w64" +- "zip" +- "ca-certificates" +- "python" +- "rename" +- "cmake" +remotes: +- "url": "https://github.com/monero-project/monero.git" + "dir": "monero" +files: [] +script: | + WRAP_DIR=$HOME/wrapped + HOSTS="i686-w64-mingw32 x86_64-w64-mingw32" + FAKETIME_HOST_PROGS="ar ranlib nm windres strip objcopy" + FAKETIME_PROGS="date zip" + HOST_CFLAGS="-O2 -g" + HOST_CXXFLAGS="-O2 -g" + + export GZIP="-9n" + export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" + export TZ="UTC" + export BUILD_DIR=`pwd` + mkdir -p ${WRAP_DIR} + if test -n "$GBUILD_CACHE_ENABLED"; then + export SOURCES_PATH=${GBUILD_COMMON_CACHE} + export BASE_CACHE=${GBUILD_PACKAGE_CACHE} + mkdir -p ${BASE_CACHE} ${SOURCES_PATH} + fi + + function create_global_faketime_wrappers { + for prog in ${FAKETIME_PROGS}; do + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} + echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${prog} + chmod +x ${WRAP_DIR}/${prog} + done + } + + function create_per-host_faketime_wrappers { + for i in $HOSTS; do + for prog in ${FAKETIME_HOST_PROGS}; do + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} + echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} + chmod +x ${WRAP_DIR}/${i}-${prog} + done + done + } + + function create_per-host_linker_wrapper { + # This is only needed for trusty, as the mingw linker leaks a few bytes of + # heap, causing non-determinism. See discussion in https://github.com/bitcoin/bitcoin/pull/6900 + for i in $HOSTS; do + mkdir -p ${WRAP_DIR}/${i} + for prog in collect2; do + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}/${prog} + REAL=$(${i}-gcc -print-prog-name=${prog}) + echo "export MALLOC_PERTURB_=255" >> ${WRAP_DIR}/${i}/${prog} + echo "${REAL} \$@" >> $WRAP_DIR/${i}/${prog} + chmod +x ${WRAP_DIR}/${i}/${prog} + done + for prog in gcc g++; do + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} + echo "REAL=\`which -a ${i}-${prog}-posix | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} + echo "export COMPILER_PATH=${WRAP_DIR}/${i}" >> ${WRAP_DIR}/${i}-${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} + chmod +x ${WRAP_DIR}/${i}-${prog} + done + done + } + + # Faketime for depends so intermediate results are comparable + export PATH_orig=${PATH} + create_global_faketime_wrappers "2000-01-01 12:00:00" + create_per-host_faketime_wrappers "2000-01-01 12:00:00" + create_per-host_linker_wrapper "2000-01-01 12:00:00" + export PATH=${WRAP_DIR}:${PATH} + + cd monero + BASEPREFIX=`pwd`/contrib/depends + # Build dependencies for each host + for i in $HOSTS; do + EXTRA_INCLUDES="$EXTRA_INCLUDES_BASE/$i" + if [ -d "$EXTRA_INCLUDES" ]; then + export HOST_ID_SALT="$EXTRA_INCLUDES" + fi + make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" -j 4 V=1 + unset HOST_ID_SALT + done + + # Faketime for binaries + export PATH=${PATH_orig} + create_global_faketime_wrappers "${REFERENCE_DATETIME}" + create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" + export PATH=${WRAP_DIR}:${PATH} + + ORIGPATH="$PATH" + # Run cmake and make, for each create a new build/ directory, + # compile from there, archive, export and delete the archive again + for i in ${HOSTS}; do + export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} + mkdir build && cd build + cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake + make + DISTNAME=monero-${i} + mv bin ${DISTNAME} + find ${DISTNAME}/ | sort | zip -X@ ${OUTDIR}/${DISTNAME}.zip + cd .. && rm -rf build + done + diff --git a/contrib/gitian/symbol-check.py b/contrib/gitian/symbol-check.py new file mode 100755 index 000000000..6808e77da --- /dev/null +++ b/contrib/gitian/symbol-check.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014 Wladimir J. van der Laan +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' +A script to check that the (Linux) executables produced by gitian only contain +allowed gcc, glibc and libstdc++ version symbols. This makes sure they are +still compatible with the minimum supported Linux distribution versions. + +Example usage: + + find ../gitian-builder/build -type f -executable | xargs python contrib/devtools/symbol-check.py +''' +import subprocess +import re +import sys +import os + +# Debian 6.0.9 (Squeeze) has: +# +# - g++ version 4.4.5 (https://packages.debian.org/search?suite=default§ion=all&arch=any&searchon=names&keywords=g%2B%2B) +# - libc version 2.11.3 (https://packages.debian.org/search?suite=default§ion=all&arch=any&searchon=names&keywords=libc6) +# - libstdc++ version 4.4.5 (https://packages.debian.org/search?suite=default§ion=all&arch=any&searchon=names&keywords=libstdc%2B%2B6) +# +# Ubuntu 10.04.4 (Lucid Lynx) has: +# +# - g++ version 4.4.3 (http://packages.ubuntu.com/search?keywords=g%2B%2B&searchon=names&suite=lucid§ion=all) +# - libc version 2.11.1 (http://packages.ubuntu.com/search?keywords=libc6&searchon=names&suite=lucid§ion=all) +# - libstdc++ version 4.4.3 (http://packages.ubuntu.com/search?suite=lucid§ion=all&arch=any&keywords=libstdc%2B%2B&searchon=names) +# +# Taking the minimum of these as our target. +# +# According to GNU ABI document (http://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html) this corresponds to: +# GCC 4.4.0: GCC_4.4.0 +# GCC 4.4.2: GLIBCXX_3.4.13, CXXABI_1.3.3 +# (glibc) GLIBC_2_11 +# +MAX_VERSIONS = { +'GCC': (4,4,0), +'CXXABI': (1,3,3), +'GLIBCXX': (3,4,13), +'GLIBC': (2,11) +} +# See here for a description of _IO_stdin_used: +# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=634261#109 + +# Ignore symbols that are exported as part of every executable +IGNORE_EXPORTS = { +'_edata', '_end', '_init', '__bss_start', '_fini', '_IO_stdin_used', 'stdin', 'stdout', 'stderr' +} +READELF_CMD = os.getenv('READELF', '/usr/bin/readelf') +CPPFILT_CMD = os.getenv('CPPFILT', '/usr/bin/c++filt') +# Allowed NEEDED libraries +ALLOWED_LIBRARIES = { +# bitcoind and bitcoin-qt +'libgcc_s.so.1', # GCC base support +'libc.so.6', # C library +'libpthread.so.0', # threading +'libanl.so.1', # DNS resolve +'libm.so.6', # math library +'librt.so.1', # real-time (clock) +'ld-linux-x86-64.so.2', # 64-bit dynamic linker +'ld-linux.so.2', # 32-bit dynamic linker +# bitcoin-qt only +'libX11-xcb.so.1', # part of X11 +'libX11.so.6', # part of X11 +'libxcb.so.1', # part of X11 +'libfontconfig.so.1', # font support +'libfreetype.so.6', # font parsing +'libdl.so.2' # programming interface to dynamic linker +} + +class CPPFilt(object): + ''' + Demangle C++ symbol names. + + Use a pipe to the 'c++filt' command. + ''' + def __init__(self): + self.proc = subprocess.Popen(CPPFILT_CMD, stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) + + def __call__(self, mangled): + self.proc.stdin.write(mangled + '\n') + self.proc.stdin.flush() + return self.proc.stdout.readline().rstrip() + + def close(self): + self.proc.stdin.close() + self.proc.stdout.close() + self.proc.wait() + +def read_symbols(executable, imports=True): + ''' + Parse an ELF executable and return a list of (symbol,version) tuples + for dynamic, imported symbols. + ''' + p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) + (stdout, stderr) = p.communicate() + if p.returncode: + raise IOError('Could not read symbols for %s: %s' % (executable, stderr.strip())) + syms = [] + for line in stdout.splitlines(): + line = line.split() + if len(line)>7 and re.match('[0-9]+:$', line[0]): + (sym, _, version) = line[7].partition('@') + is_import = line[6] == 'UND' + if version.startswith('@'): + version = version[1:] + if is_import == imports: + syms.append((sym, version)) + return syms + +def check_version(max_versions, version): + if '_' in version: + (lib, _, ver) = version.rpartition('_') + else: + lib = version + ver = '0' + ver = tuple([int(x) for x in ver.split('.')]) + if not lib in max_versions: + return False + return ver <= max_versions[lib] + +def read_libraries(filename): + p = subprocess.Popen([READELF_CMD, '-d', '-W', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) + (stdout, stderr) = p.communicate() + if p.returncode: + raise IOError('Error opening file') + libraries = [] + for line in stdout.splitlines(): + tokens = line.split() + if len(tokens)>2 and tokens[1] == '(NEEDED)': + match = re.match('^Shared library: \[(.*)\]$', ' '.join(tokens[2:])) + if match: + libraries.append(match.group(1)) + else: + raise ValueError('Unparseable (NEEDED) specification') + return libraries + +if __name__ == '__main__': + cppfilt = CPPFilt() + retval = 0 + for filename in sys.argv[1:]: + # Check imported symbols + for sym,version in read_symbols(filename, True): + if version and not check_version(MAX_VERSIONS, version): + print('%s: symbol %s from unsupported version %s' % (filename, cppfilt(sym), version)) + retval = 1 + # Check exported symbols + for sym,version in read_symbols(filename, False): + if sym in IGNORE_EXPORTS: + continue + print('%s: export of symbol %s not allowed' % (filename, cppfilt(sym))) + retval = 1 + # Check dependency libraries + for library_name in read_libraries(filename): + if library_name not in ALLOWED_LIBRARIES: + print('%s: NEEDED library %s is not allowed' % (filename, library_name)) + retval = 1 + + sys.exit(retval) + + |