diff options
author | Riccardo Spagni <ric@spagni.net> | 2018-10-26 22:19:56 +0200 |
---|---|---|
committer | Riccardo Spagni <ric@spagni.net> | 2018-10-26 22:19:57 +0200 |
commit | a91b432591f32a5199a6ddfa59027ea5cee9f9eb (patch) | |
tree | 0ad11bef4b13d15bd5394c83dea2cf59f3b6fb35 /contrib | |
parent | Merge pull request #4513 (diff) | |
parent | Adapt Readme and script to monero gitian build signing (diff) | |
download | monero-a91b432591f32a5199a6ddfa59027ea5cee9f9eb.tar.xz |
Merge pull request #4526
8f96c718 Adapt Readme and script to monero gitian build signing (TheCharlatan)
9617fad0 Add OSX gitian descriptor (TheCharlatan)
d147d240 Add windows descriptor to gitian descriptors (TheCharlatan)
fed4e598 Change gitian.sigs repo from bitcoin-core to monero-project remote host (TheCharlatan)
f2127f9d Add checksums for download tools (TheCharlatan)
c2f17890 Add gitian build script (TheCharlatan)
6d0ca4e2 Prepare Depends Packages for Gitian Scripts (TheCharlatan)
Diffstat (limited to 'contrib')
-rw-r--r-- | contrib/depends/packages/libiconv.mk | 15 | ||||
-rw-r--r-- | contrib/depends/packages/sodium.mk | 2 | ||||
-rw-r--r-- | contrib/depends/patches/libiconv/fix-whitespace.patch | 13 | ||||
-rw-r--r-- | contrib/depends/patches/sodium/fix-whitespace.patch | 13 | ||||
-rw-r--r-- | contrib/gitian/README.md | 146 | ||||
-rwxr-xr-x | contrib/gitian/gitian-build.py | 192 | ||||
-rw-r--r-- | contrib/gitian/gitian-linux.yml | 162 | ||||
-rw-r--r-- | contrib/gitian/gitian-osx.yml | 114 | ||||
-rw-r--r-- | contrib/gitian/gitian-win.yml | 134 | ||||
-rwxr-xr-x | contrib/gitian/symbol-check.py | 163 |
10 files changed, 953 insertions, 1 deletions
diff --git a/contrib/depends/packages/libiconv.mk b/contrib/depends/packages/libiconv.mk index 87e30b208..dbcb28141 100644 --- a/contrib/depends/packages/libiconv.mk +++ b/contrib/depends/packages/libiconv.mk @@ -3,9 +3,22 @@ $(package)_version=1.15 $(package)_download_path=https://ftp.gnu.org/gnu/libiconv $(package)_file_name=libiconv-$($(package)_version).tar.gz $(package)_sha256_hash=ccf536620a45458d26ba83887a983b96827001e92a13847b45e4925cc8913178 +$(package)_patches=fix-whitespace.patch + +define $(package)_set_vars + $(package)_config_opts=--disable-nls + $(package)_config_opts=--enable-static + $(package)_config_opts=--disable-shared + $(package)_config_opts_linux=--with-pic +endef + +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub build-aux/ &&\ + patch -p1 < $($(package)_patch_dir)/fix-whitespace.patch +endef define $(package)_config_cmds - $($(package)_autoconf) --disable-nls --enable-static --disable-shared + $($(package)_autoconf) endef define $(package)_build_cmds diff --git a/contrib/depends/packages/sodium.mk b/contrib/depends/packages/sodium.mk index c38121bf7..35f444fd5 100644 --- a/contrib/depends/packages/sodium.mk +++ b/contrib/depends/packages/sodium.mk @@ -3,6 +3,7 @@ $(package)_version=1.0.15 $(package)_download_path=https://download.libsodium.org/libsodium/releases/ $(package)_file_name=libsodium-$($(package)_version).tar.gz $(package)_sha256_hash=fb6a9e879a2f674592e4328c5d9f79f082405ee4bb05cb6e679b90afe9e178f4 +$(package)_patches=fix-whitespace.patch define $(package)_set_vars $(package)_config_opts=--enable-static --disable-shared @@ -11,6 +12,7 @@ endef define $(package)_config_cmds ./autogen.sh &&\ + patch -p1 < $($(package)_patch_dir)/fix-whitespace.patch &&\ $($(package)_autoconf) $($(package)_config_opts) endef diff --git a/contrib/depends/patches/libiconv/fix-whitespace.patch b/contrib/depends/patches/libiconv/fix-whitespace.patch new file mode 100644 index 000000000..531364b45 --- /dev/null +++ b/contrib/depends/patches/libiconv/fix-whitespace.patch @@ -0,0 +1,13 @@ +diff --git a/preload/configure b/preload/configure +index aab5c77..e20b8f0 100755 +--- a/preload/configure ++++ b/preload/configure +@@ -588,7 +588,7 @@ MAKEFLAGS= + PACKAGE_NAME='libiconv' + PACKAGE_TARNAME='libiconv' + PACKAGE_VERSION='0' +-PACKAGE_STRING='libiconv 0' ++PACKAGE_STRING='libiconv0' + PACKAGE_BUGREPORT='' + PACKAGE_URL='' + diff --git a/contrib/depends/patches/sodium/fix-whitespace.patch b/contrib/depends/patches/sodium/fix-whitespace.patch new file mode 100644 index 000000000..c11838611 --- /dev/null +++ b/contrib/depends/patches/sodium/fix-whitespace.patch @@ -0,0 +1,13 @@ +diff --git a/configure b/configure +index b29f769..ca008ae 100755 +--- a/configure ++++ b/configure +@@ -591,7 +591,7 @@ MAKEFLAGS= + PACKAGE_NAME='libsodium' + PACKAGE_TARNAME='libsodium' + PACKAGE_VERSION='1.0.15' +-PACKAGE_STRING='libsodium 1.0.15' ++PACKAGE_STRING='libsodium' + PACKAGE_BUGREPORT='https://github.com/jedisct1/libsodium/issues' + PACKAGE_URL='https://github.com/jedisct1/libsodium' + diff --git a/contrib/gitian/README.md b/contrib/gitian/README.md new file mode 100644 index 000000000..4bd326f22 --- /dev/null +++ b/contrib/gitian/README.md @@ -0,0 +1,146 @@ +Gitian building +================ + +*Setup instructions for a Gitian build of Monero using a VM or physical system.* + +Gitian is the deterministic build process that is used to build the Monero CLI +executables. It provides a way to be reasonably sure that the +executables are really built from the git source. It also makes sure that +the same, tested dependencies are used and statically built into the executable. + +Multiple developers build the source code by following a specific descriptor +("recipe"), cryptographically sign the result, and upload the resulting signature. +These results are compared and only if they match, the build is accepted and provided +for download. + +More independent Gitian builders are needed, which is why this guide exists. +It is preferred you follow these steps yourself instead of using someone else's +VM image to avoid 'contaminating' the build. + +Table of Contents +------------------ + +Please note that these instructions have been forked from bitcoin's gitian build +instructions. Please also consult their documentation, when running into problems. +The signing is left as inherited from bitcoin at the moment. + +- [Preparing the Gitian builder host](#preparing-the-gitian-builder-host) +- [Getting and building the inputs](#getting-and-building-the-inputs) +- [Building Binaries](#building-bitcoin-core) +- [Signing externally](#signing-externally) +- [Uploading signatures](#uploading-signatures) + +Preparing the Gitian builder host +--------------------------------- + +The first step is to prepare the host environment that will be used to perform the Gitian builds. +This guide explains how to set up the environment, and how to start the builds. + +Gitian builds are for now executed on Ubuntu 18.04 "Bionic Beaver". A solution is being worked on to run +it in docker in the future. Please run Ubuntu in either a VM, or on your physical machine. +You need to be logged in as the `gitianuser` in order to build gitian builds. If this user does not exist yet on your system, +create it. + +Note that a version of `lxc-execute` higher or equal to 2.1.1 is required. +You can check the version with `lxc-execute --version`. + +First we need to set up dependencies. Type/paste the following in the terminal: + +```bash +sudo apt-get install git ruby apt-cacher-ng qemu-utils debootstrap lxc python-cheetah parted kpartx bridge-utils make ubuntu-archive-keyring curl firewalld +``` + +Then set up LXC and the rest with the following, which is a complex jumble of settings and workarounds: + +```bash +sudo -s +# the version of lxc-start in Debian needs to run as root, so make sure +# that the build script can execute it without providing a password +echo "%sudo ALL=NOPASSWD: /usr/bin/lxc-start" > /etc/sudoers.d/gitian-lxc +echo "%sudo ALL=NOPASSWD: /usr/bin/lxc-execute" >> /etc/sudoers.d/gitian-lxc +# make /etc/rc.local script that sets up bridge between guest and host +echo '#!/bin/sh -e' > /etc/rc.local +echo 'brctl addbr br0' >> /etc/rc.local +echo 'ip addr add 10.0.3.1/24 broadcast 10.0.3.255 dev br0' >> /etc/rc.local +echo 'ip link set br0 up' >> /etc/rc.local +echo 'firewall-cmd --zone=trusted --add-interface=br0' >> /etc/rc.local +echo 'exit 0' >> /etc/rc.local +chmod +x /etc/rc.local +# make sure that USE_LXC is always set when logging in as gitianuser, +# and configure LXC IP addresses +echo 'export USE_LXC=1' >> /home/gitianuser/.profile +echo 'export GITIAN_HOST_IP=10.0.3.1' >> /home/gitianuser/.profile +echo 'export LXC_GUEST_IP=10.0.3.5' >> /home/gitianuser/.profile +reboot +``` + +This setup is required to enable networking in the container. + + +Manual and Building +------------------- +The instructions below use the automated script [gitian-build.py](https://github.com/betcoin/bitcoin/blob/master/contrib/gitian-build.py) which only works in Ubuntu. +It calls all available descriptors. Help for the build steps taken can be accessed with `./gitian-build.py --help`. + +Initial Gitian Setup +-------------------- +The `gitian-build.py` script will checkout different release tags, so it's best to copy it: + +```bash +cp monero/contrib/gitian/gitian-build.py . +``` + +Setup the required environment, you only need to do this once: + +``` +./gitian-build.py --setup fluffypony 0.14.0 +``` + +Where `fluffypony` is your Github name and `0.14.0` is the version tag you want to build (without `v`). + +While gitian and this build script does provide a way for you to sign the build directly, it is recommended to sign in a seperate step. +This script is only there for convenience. Seperate steps for building can still be taken. +In order to sign gitian builds on your host machine, which has your PGP key, +fork the gitian.sigs repository and clone it on your host machine, +or pass the signed assert file back to your build machine. + +``` +git clone git@github.com:monero-project/gitian.sigs.git +git remote add fluffypony git@github.com:fluffypony/gitian.sigs.git +``` + +Build Binaries +----------------------------- +To build the most recent tag: + + `./gitian-build.py --detach-sign --no-commit -b fluffypony 0.14.0` + +To speed up the build, use `-j 5 -m 5000` as the first arguments, where `5` is the number of CPU's you allocated to the VM plus one, and 5000 is a little bit less than then the MB's of RAM you allocated. If there is memory corruption on your machine, try to tweak these values. + +If all went well, this produces a number of (uncommited) `.assert` files in the gitian.sigs repository. + +If you do detached, offline signing, you need to copy these uncommited changes to your host machine, where you can sign them. For example: + +``` +export NAME=fluffypony +export VERSION=0.14 +gpg --output $VERSION-linux/$NAME/monero-linux-$VERSION-build.assert.sig --detach-sign $VERSION-linux/$NAME/monero-linux-$VERSION-build.assert +gpg --output $VERSION-osx-unsigned/$NAME/monero-osx-$VERSION-build.assert.sig --detach-sign $VERSION-osx-unsigned/$NAME/monero-osx-$VERSION-build.assert +gpg --output $VERSION-win-unsigned/$NAME/monero-win-$VERSION-build.assert.sig --detach-sign $VERSION-win-unsigned/$NAME/monero-win-$VERSION-build.assert +``` + +Make a pull request (both the `.assert` and `.assert.sig` files) to the +[monero-project/gitian.sigs](https://github.com/monero-project/gitian.sigs/) repository: + +``` +git checkout -b 0.14.0 +git commit -S -a -m "Add $NAME 0.14.0" +git push --set-upstream $NAME 0.14.0 +``` + +```bash + gpg --detach-sign ${VERSION}-linux/${SIGNER}/monero-linux-*-build.assert + gpg --detach-sign ${VERSION}-win-unsigned/${SIGNER}/monero-win-*-build.assert + gpg --detach-sign ${VERSION}-osx-unsigned/${SIGNER}/monero-osx-*-build.assert +``` + diff --git a/contrib/gitian/gitian-build.py b/contrib/gitian/gitian-build.py new file mode 100755 index 000000000..99c64e9dd --- /dev/null +++ b/contrib/gitian/gitian-build.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 + +import argparse +import os +import subprocess +import sys + +def setup(): + global args, workdir + programs = ['ruby', 'git', 'apt-cacher-ng', 'make', 'wget'] + if args.kvm: + programs += ['python-vm-builder', 'qemu-kvm', 'qemu-utils'] + elif args.docker: + dockers = ['docker.io', 'docker-ce'] + for i in dockers: + return_code = subprocess.call(['sudo', 'apt-get', 'install', '-qq', i]) + if return_code == 0: + break + if return_code != 0: + print('Cannot find any way to install docker', file=sys.stderr) + exit(1) + else: + programs += ['lxc', 'debootstrap'] + subprocess.check_call(['sudo', 'apt-get', 'install', '-qq'] + programs) + if not os.path.isdir('gitian.sigs'): + subprocess.check_call(['git', 'clone', 'https://github.com/monero-project/gitian.sigs.git']) + if not os.path.isdir('gitian-builder'): + subprocess.check_call(['git', 'clone', 'https://github.com/devrandom/gitian-builder.git']) + if not os.path.isdir('monero'): + subprocess.check_call(['git', 'clone', 'https://github.com/monero-project/monero.git']) + os.chdir('gitian-builder') + subprocess.check_call(['git', 'checkout', '963322de8420c50502c4cc33d4d7c0d84437b576']) + make_image_prog = ['bin/make-base-vm', '--suite', 'bionic', '--arch', 'amd64'] + if args.docker: + make_image_prog += ['--docker'] + elif not args.kvm: + make_image_prog += ['--lxc'] + subprocess.check_call(make_image_prog) + os.chdir(workdir) + if args.is_bionic and not args.kvm and not args.docker: + subprocess.check_call(['sudo', 'sed', '-i', 's/lxcbr0/br0/', '/etc/default/lxc-net']) + print('Reboot is required') + exit(0) + +def build(): + global args, workdir + + os.makedirs('monero-binaries/' + args.version, exist_ok=True) + print('\nBuilding Dependencies\n') + os.chdir('gitian-builder') + os.makedirs('inputs', exist_ok=True) + + subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://downloads.sourceforge.net/project/osslsigncode/osslsigncode/osslsigncode-1.7.1.tar.gz']) + subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/cfields/osslsigncode-Backports-to-1.7.1.patch']) + subprocess.check_output(["echo 'a8c4e9cafba922f89de0df1f2152e7be286aba73f78505169bc351a7938dd911 inputs/osslsigncode-Backports-to-1.7.1.patch' | sha256sum -c"], shell=True) + subprocess.check_output(["echo 'f9a8cdb38b9c309326764ebc937cba1523a3a751a7ab05df3ecc99d18ae466c9 inputs/osslsigncode-1.7.1.tar.gz' | sha256sum -c"], shell=True) + subprocess.check_call(['make', '-C', '../monero/contrib/depends', 'download', 'SOURCES_PATH=' + os.getcwd() + '/cache/common']) + + if args.linux: + print('\nCompiling ' + args.version + ' Linux') + subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero='+args.commit, '--url', 'monero='+args.url, '../monero/contrib/gitian/gitian-linux.yml']) + subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-linux', '--destination', '../gitian.sigs/', '../monero/contrib/gitian/gitian-linux.yml']) + subprocess.check_call('mv build/out/monero-*.tar.gz ../monero-binaries/'+args.version, shell=True) + + if args.windows: + print('\nCompiling ' + args.version + ' Windows') + subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero='+args.commit, '--url', 'monero='+args.url, '../monero/contrib/gitian/gitian-win.yml']) + subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-win', '--destination', '../gitian.sigs/', '../monero/contrib/gitian/gitian-win.yml']) + subprocess.check_call('mv build/out/monero*.zip ../monero-binaries/'+args.version, shell=True) + + if args.macos: + print('\nCompiling ' + args.version + ' MacOS') + subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero='+args.commit, '--url', 'monero'+args.url, '../monero/contrib/gitian/gitian-osx.yml']) + subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx', '--destination', '../gitian.sigs/', '../monero/contrib/gitian/gitian-osx.yml']) + subprocess.check_call('mv build/out/monero*.tar.gz ../monero-binaries/'+args.version, shell=True) + + os.chdir(workdir) + + if args.commit_files: + print('\nCommitting '+args.version+' Unsigned Sigs\n') + os.chdir('gitian.sigs') + subprocess.check_call(['git', 'add', args.version+'-linux/'+args.signer]) + subprocess.check_call(['git', 'add', args.version+'-win/'+args.signer]) + subprocess.check_call(['git', 'add', args.version+'-osx/'+args.signer]) + subprocess.check_call(['git', 'commit', '-m', 'Add '+args.version+' unsigned sigs for '+args.signer]) + os.chdir(workdir) + +def verify(): + global args, workdir + os.chdir('gitian-builder') + + print('\nVerifying v'+args.version+' Linux\n') + subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-linux', '../monero/contrib/gitian/gitian-linux.yml']) + print('\nVerifying v'+args.version+' Windows\n') + subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win', '../monero/contrib/gitian/gitian-win.yml']) + print('\nVerifying v'+args.version+' MacOS\n') + subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx', '../monero/contrib/gitian/gitian-osx.yml']) + os.chdir(workdir) + +def main(): + global args, workdir + + parser = argparse.ArgumentParser(usage='%(prog)s [options] signer version') + parser.add_argument('-c', '--commit', action='store_true', dest='commit', help='Indicate that the version argument is for a commit or branch') + parser.add_argument('-p', '--pull', action='store_true', dest='pull', help='Indicate that the version argument is the number of a github repository pull request') + parser.add_argument('-u', '--url', dest='url', default='https://github.com/monero-project/monero', help='Specify the URL of the repository. Default is %(default)s') + parser.add_argument('-v', '--verify', action='store_true', dest='verify', help='Verify the Gitian build') + parser.add_argument('-b', '--build', action='store_true', dest='build', help='Do a Gitian build') + parser.add_argument('-B', '--buildsign', action='store_true', dest='buildsign', help='Build both signed and unsigned binaries') + parser.add_argument('-o', '--os', dest='os', default='lwm', help='Specify which Operating Systems the build is for. Default is %(default)s. l for Linux, w for Windows, m for MacOS') + parser.add_argument('-j', '--jobs', dest='jobs', default='2', help='Number of processes to use. Default %(default)s') + parser.add_argument('-m', '--memory', dest='memory', default='2000', help='Memory to allocate in MiB. Default %(default)s') + parser.add_argument('-k', '--kvm', action='store_true', dest='kvm', help='Use KVM instead of LXC') + parser.add_argument('-d', '--docker', action='store_true', dest='docker', help='Use Docker instead of LXC') + parser.add_argument('-S', '--setup', action='store_true', dest='setup', help='Set up the Gitian building environment. Uses LXC. If you want to use KVM, use the --kvm option. Only works on Debian-based systems (Ubuntu, Debian)') + parser.add_argument('-D', '--detach-sign', action='store_true', dest='detach_sign', help='Create the assert file for detached signing. Will not commit anything.') + parser.add_argument('-n', '--no-commit', action='store_false', dest='commit_files', help='Do not commit anything to git') + parser.add_argument('signer', help='GPG signer to sign each build assert file') + parser.add_argument('version', help='Version number, commit, or branch to build. If building a commit or branch, the -c option must be specified') + + args = parser.parse_args() + workdir = os.getcwd() + + args.linux = 'l' in args.os + args.windows = 'w' in args.os + args.macos = 'm' in args.os + + args.is_bionic = b'bionic' in subprocess.check_output(['lsb_release', '-cs']) + + if args.buildsign: + args.build=True + args.sign=True + + if args.kvm and args.docker: + raise Exception('Error: cannot have both kvm and docker') + + args.sign_prog = 'true' if args.detach_sign else 'gpg --detach-sign' + + # Set enviroment variable USE_LXC or USE_DOCKER, let gitian-builder know that we use lxc or docker + if args.docker: + os.environ['USE_DOCKER'] = '1' + elif not args.kvm: + os.environ['USE_LXC'] = '1' + if not 'GITIAN_HOST_IP' in os.environ.keys(): + os.environ['GITIAN_HOST_IP'] = '10.0.3.1' + if not 'LXC_GUEST_IP' in os.environ.keys(): + os.environ['LXC_GUEST_IP'] = '10.0.3.5' + + # Disable for MacOS if no SDK found + if args.macos and not os.path.isfile('gitian-builder/inputs/MacOSX10.11.sdk.tar.gz'): + print('Cannot build for MacOS, SDK does not exist. Will build for other OSes') + args.macos = False + + script_name = os.path.basename(sys.argv[0]) + # Signer and version shouldn't be empty + if args.signer == '': + print(script_name+': Missing signer.') + print('Try '+script_name+' --help for more information') + exit(1) + if args.version == '': + print(script_name+': Missing version.') + print('Try '+script_name+' --help for more information') + exit(1) + + # Add leading 'v' for tags + if args.commit and args.pull: + raise Exception('Cannot have both commit and pull') + args.commit = ('' if args.commit else) + args.version + + if args.setup: + setup() + + os.chdir('monero') + if args.pull: + subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge']) + os.chdir('../gitian-builder/inputs/monero') + subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge']) + args.commit = subprocess.check_output(['git', 'show', '-s', '--format=%H', 'FETCH_HEAD'], universal_newlines=True).strip() + args.version = 'pull-' + args.version + print(args.commit) + subprocess.check_call(['git', 'fetch']) + subprocess.check_call(['git', 'checkout', args.commit]) + os.chdir(workdir) + + if args.build: + build() + + if args.verify: + verify() + +if __name__ == '__main__': + main() diff --git a/contrib/gitian/gitian-linux.yml b/contrib/gitian/gitian-linux.yml new file mode 100644 index 000000000..473a7720d --- /dev/null +++ b/contrib/gitian/gitian-linux.yml @@ -0,0 +1,162 @@ +--- +name: "monero-linux-0.14" +enable_cache: true +suites: +- "bionic" +architectures: +- "amd64" +packages: +- "curl" +- "gperf" +- "gcc-7" +- "g++-7" +- "gcc" +- "g++" +- "gcc-7-aarch64-linux-gnu" +- "g++-7-aarch64-linux-gnu" +- "gcc-aarch64-linux-gnu" +- "g++-aarch64-linux-gnu" +- "binutils-aarch64-linux-gnu" +- "gcc-7-arm-linux-gnueabihf" +- "g++-7-arm-linux-gnueabihf" +- "gcc-arm-linux-gnueabihf" +- "g++-arm-linux-gnueabihf" +- "g++-7-multilib" +- "gcc-7-multilib" +- "binutils-arm-linux-gnueabihf" +- "binutils-gold" +- "git" +- "pkg-config" +- "build-essential" +- "autoconf" +- "libtool" +- "automake" +- "faketime" +- "bsdmainutils" +- "ca-certificates" +- "python" +- "cmake" +- "ccache" +- "protobuf-compiler" +- "libdbus-1-dev" +- "libharfbuzz-dev" +- "libprotobuf-dev" +- "python3-zmq" +remotes: +- "url": "https://github.com/monero-project/monero.git" + "dir": "monero" +files: [] +script: | + + WRAP_DIR=$HOME/wrapped + HOSTS="x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu i686-linux-gnu" + FAKETIME_HOST_PROGS="gcc g++" + FAKETIME_PROGS="date ar ranlib nm" + HOST_CFLAGS="-O2 -g" + HOST_CXXFLAGS="-O2 -g" + HOST_LDFLAGS=-static-libstdc++ + + export GZIP="-9n" + export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" + export TZ="UTC" + export BUILD_DIR=`pwd` + mkdir -p ${WRAP_DIR} + if test -n "$GBUILD_CACHE_ENABLED"; then + export SOURCES_PATH=${GBUILD_COMMON_CACHE} + export BASE_CACHE=${GBUILD_PACKAGE_CACHE} + mkdir -p ${BASE_CACHE} ${SOURCES_PATH} + fi + + function create_global_faketime_wrappers { + for prog in ${FAKETIME_PROGS}; do + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} + echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${prog} + chmod +x ${WRAP_DIR}/${prog} + done + } + + function create_per-host_faketime_wrappers { + for i in $HOSTS; do + for prog in ${FAKETIME_HOST_PROGS}; do + if which ${i}-${prog}-7 + then + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} + echo "REAL=\`which -a ${i}-${prog}-7 | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} + chmod +x ${WRAP_DIR}/${i}-${prog} + fi + done + done + } + + # Faketime for depends so intermediate results are comparable + export PATH_orig=${PATH} + create_global_faketime_wrappers "2000-01-01 12:00:00" + create_per-host_faketime_wrappers "2000-01-01 12:00:00" + export PATH=${WRAP_DIR}:${PATH} + + EXTRA_INCLUDES_BASE=$WRAP_DIR/extra_includes + mkdir -p $EXTRA_INCLUDES_BASE + + # x86 needs /usr/include/i386-linux-gnu/asm pointed to /usr/include/x86_64-linux-gnu/asm, + # but we can't write there. Instead, create a link here and force it to be included in the + # search paths by wrapping gcc/g++. + + mkdir -p $EXTRA_INCLUDES_BASE/i686-pc-linux-gnu + rm -f $WRAP_DIR/extra_includes/i686-pc-linux-gnu/asm + ln -s /usr/include/x86_64-linux-gnu/asm $EXTRA_INCLUDES_BASE/i686-pc-linux-gnu/asm + + for prog in gcc g++; do + rm -f ${WRAP_DIR}/${prog} + cat << EOF > ${WRAP_DIR}/${prog} + #!/usr/bin/env bash + REAL="`which -a ${prog}-7 | grep -v ${WRAP_DIR}/${prog} | head -1`" + for var in "\$@" + do + if [ "\$var" = "-m32" ]; then + export C_INCLUDE_PATH="$EXTRA_INCLUDES_BASE/i686-pc-linux-gnu" + export CPLUS_INCLUDE_PATH="$EXTRA_INCLUDES_BASE/i686-pc-linux-gnu" + break + fi + done + \$REAL \$@ + EOF + chmod +x ${WRAP_DIR}/${prog} + done + + cd monero + BASEPREFIX=`pwd`/contrib/depends + # Build dependencies for each host + for i in $HOSTS; do + EXTRA_INCLUDES="$EXTRA_INCLUDES_BASE/$i" + if [ -d "$EXTRA_INCLUDES" ]; then + export HOST_ID_SALT="$EXTRA_INCLUDES" + fi + make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" -j 4 V=1 + unset HOST_ID_SALT + done + + # Faketime for binaries + export PATH=${PATH_orig} + create_global_faketime_wrappers "${REFERENCE_DATETIME}" + create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" + export PATH=${WRAP_DIR}:${PATH} + + ORIGPATH="$PATH" + # Build in a new dir for each host + for i in ${HOSTS}; do + export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} + mkdir build && cd build + cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake + make + DISTNAME=monero-${i} + mv bin ${DISTNAME} + find ${DISTNAME}/ | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}.tar.gz + cd .. + rm -rf build + done diff --git a/contrib/gitian/gitian-osx.yml b/contrib/gitian/gitian-osx.yml new file mode 100644 index 000000000..a6fcff0da --- /dev/null +++ b/contrib/gitian/gitian-osx.yml @@ -0,0 +1,114 @@ +--- +name: "monero-osx-0.14" +enable_cache: true +suites: +- "bionic" +architectures: +- "amd64" +packages: +- "ca-certificates" +- "curl" +- "g++" +- "git" +- "pkg-config" +- "autoconf" +- "librsvg2-bin" +- "libtiff-tools" +- "libtool" +- "automake" +- "faketime" +- "bsdmainutils" +- "cmake" +- "imagemagick" +- "libcap-dev" +- "libz-dev" +- "libbz2-dev" +- "python" +- "python-dev" +- "python-setuptools" +- "fonts-tuffy" +remotes: +- "url": "https://github.com/monero-project/monero.git" + "dir": "monero" +files: +- "MacOSX10.11.sdk.tar.gz" +script: | + WRAP_DIR=$HOME/wrapped + HOSTS="x86_64-apple-darwin11" + FAKETIME_HOST_PROGS="" + FAKETIME_PROGS="ar ranlib date dmg genisoimage" + + export GZIP="-9n" + export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" + export TZ="UTC" + export BUILD_DIR=`pwd` + mkdir -p ${WRAP_DIR} + if test -n "$GBUILD_CACHE_ENABLED"; then + export SOURCES_PATH=${GBUILD_COMMON_CACHE} + export BASE_CACHE=${GBUILD_PACKAGE_CACHE} + mkdir -p ${BASE_CACHE} ${SOURCES_PATH} + fi + + export ZERO_AR_DATE=1 + + function create_global_faketime_wrappers { + for prog in ${FAKETIME_PROGS}; do + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} + echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${prog} + chmod +x ${WRAP_DIR}/${prog} + done + } + + function create_per-host_faketime_wrappers { + for i in $HOSTS; do + for prog in ${FAKETIME_HOST_PROGS}; do + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} + echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} + chmod +x ${WRAP_DIR}/${i}-${prog} + done + done + } + + # Faketime for depends so intermediate results are comparable + export PATH_orig=${PATH} + create_global_faketime_wrappers "2000-01-01 12:00:00" + create_per-host_faketime_wrappers "2000-01-01 12:00:00" + export PATH=${WRAP_DIR}:${PATH} + + cd monero + BASEPREFIX=`pwd`/contrib/depends + + mkdir -p ${BASEPREFIX}/SDKs + tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/MacOSX10.11.sdk.tar.gz + + # Build dependencies for each host + for i in $HOSTS; do + make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" + done + + # Faketime for binaries + export PATH=${PATH_orig} + create_global_faketime_wrappers "${REFERENCE_DATETIME}" + create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" + export PATH=${WRAP_DIR}:${PATH} + + ORIGPATH="$PATH" + # Build in a new dir for each host + for i in ${HOSTS}; do + export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} + mkdir build && cd build + cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake + make + DISTNAME=monero-${i} + mv bin ${DISTNAME} + find ${DISTNAME}/ | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}.tar.gz + cd .. + rm -rf build + done + diff --git a/contrib/gitian/gitian-win.yml b/contrib/gitian/gitian-win.yml new file mode 100644 index 000000000..fef5567f9 --- /dev/null +++ b/contrib/gitian/gitian-win.yml @@ -0,0 +1,134 @@ +--- +name: "monero-win-0.14" +enable_cache: true +suites: +- "bionic" +architectures: +- "amd64" +packages: +- "curl" +- "g++" +- "git" +- "pkg-config" +- "autoconf" +- "libtool" +- "automake" +- "faketime" +- "bsdmainutils" +- "mingw-w64" +- "g++-mingw-w64" +- "zip" +- "ca-certificates" +- "python" +- "rename" +- "cmake" +remotes: +- "url": "https://github.com/monero-project/monero.git" + "dir": "monero" +files: [] +script: | + WRAP_DIR=$HOME/wrapped + HOSTS="i686-w64-mingw32 x86_64-w64-mingw32" + FAKETIME_HOST_PROGS="ar ranlib nm windres strip objcopy" + FAKETIME_PROGS="date zip" + HOST_CFLAGS="-O2 -g" + HOST_CXXFLAGS="-O2 -g" + + export GZIP="-9n" + export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME"" + export TZ="UTC" + export BUILD_DIR=`pwd` + mkdir -p ${WRAP_DIR} + if test -n "$GBUILD_CACHE_ENABLED"; then + export SOURCES_PATH=${GBUILD_COMMON_CACHE} + export BASE_CACHE=${GBUILD_PACKAGE_CACHE} + mkdir -p ${BASE_CACHE} ${SOURCES_PATH} + fi + + function create_global_faketime_wrappers { + for prog in ${FAKETIME_PROGS}; do + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} + echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${prog} + chmod +x ${WRAP_DIR}/${prog} + done + } + + function create_per-host_faketime_wrappers { + for i in $HOSTS; do + for prog in ${FAKETIME_HOST_PROGS}; do + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} + echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} + chmod +x ${WRAP_DIR}/${i}-${prog} + done + done + } + + function create_per-host_linker_wrapper { + # This is only needed for trusty, as the mingw linker leaks a few bytes of + # heap, causing non-determinism. See discussion in https://github.com/bitcoin/bitcoin/pull/6900 + for i in $HOSTS; do + mkdir -p ${WRAP_DIR}/${i} + for prog in collect2; do + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}/${prog} + REAL=$(${i}-gcc -print-prog-name=${prog}) + echo "export MALLOC_PERTURB_=255" >> ${WRAP_DIR}/${i}/${prog} + echo "${REAL} \$@" >> $WRAP_DIR/${i}/${prog} + chmod +x ${WRAP_DIR}/${i}/${prog} + done + for prog in gcc g++; do + echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} + echo "REAL=\`which -a ${i}-${prog}-posix | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} + echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} + echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} + echo "export COMPILER_PATH=${WRAP_DIR}/${i}" >> ${WRAP_DIR}/${i}-${prog} + echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog} + chmod +x ${WRAP_DIR}/${i}-${prog} + done + done + } + + # Faketime for depends so intermediate results are comparable + export PATH_orig=${PATH} + create_global_faketime_wrappers "2000-01-01 12:00:00" + create_per-host_faketime_wrappers "2000-01-01 12:00:00" + create_per-host_linker_wrapper "2000-01-01 12:00:00" + export PATH=${WRAP_DIR}:${PATH} + + cd monero + BASEPREFIX=`pwd`/contrib/depends + # Build dependencies for each host + for i in $HOSTS; do + EXTRA_INCLUDES="$EXTRA_INCLUDES_BASE/$i" + if [ -d "$EXTRA_INCLUDES" ]; then + export HOST_ID_SALT="$EXTRA_INCLUDES" + fi + make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" -j 4 V=1 + unset HOST_ID_SALT + done + + # Faketime for binaries + export PATH=${PATH_orig} + create_global_faketime_wrappers "${REFERENCE_DATETIME}" + create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" + export PATH=${WRAP_DIR}:${PATH} + + ORIGPATH="$PATH" + # Run cmake and make, for each create a new build/ directory, + # compile from there, archive, export and delete the archive again + for i in ${HOSTS}; do + export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} + mkdir build && cd build + cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake + make + DISTNAME=monero-${i} + mv bin ${DISTNAME} + find ${DISTNAME}/ | sort | zip -X@ ${OUTDIR}/${DISTNAME}.zip + cd .. && rm -rf build + done + diff --git a/contrib/gitian/symbol-check.py b/contrib/gitian/symbol-check.py new file mode 100755 index 000000000..6808e77da --- /dev/null +++ b/contrib/gitian/symbol-check.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014 Wladimir J. van der Laan +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +''' +A script to check that the (Linux) executables produced by gitian only contain +allowed gcc, glibc and libstdc++ version symbols. This makes sure they are +still compatible with the minimum supported Linux distribution versions. + +Example usage: + + find ../gitian-builder/build -type f -executable | xargs python contrib/devtools/symbol-check.py +''' +import subprocess +import re +import sys +import os + +# Debian 6.0.9 (Squeeze) has: +# +# - g++ version 4.4.5 (https://packages.debian.org/search?suite=default§ion=all&arch=any&searchon=names&keywords=g%2B%2B) +# - libc version 2.11.3 (https://packages.debian.org/search?suite=default§ion=all&arch=any&searchon=names&keywords=libc6) +# - libstdc++ version 4.4.5 (https://packages.debian.org/search?suite=default§ion=all&arch=any&searchon=names&keywords=libstdc%2B%2B6) +# +# Ubuntu 10.04.4 (Lucid Lynx) has: +# +# - g++ version 4.4.3 (http://packages.ubuntu.com/search?keywords=g%2B%2B&searchon=names&suite=lucid§ion=all) +# - libc version 2.11.1 (http://packages.ubuntu.com/search?keywords=libc6&searchon=names&suite=lucid§ion=all) +# - libstdc++ version 4.4.3 (http://packages.ubuntu.com/search?suite=lucid§ion=all&arch=any&keywords=libstdc%2B%2B&searchon=names) +# +# Taking the minimum of these as our target. +# +# According to GNU ABI document (http://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html) this corresponds to: +# GCC 4.4.0: GCC_4.4.0 +# GCC 4.4.2: GLIBCXX_3.4.13, CXXABI_1.3.3 +# (glibc) GLIBC_2_11 +# +MAX_VERSIONS = { +'GCC': (4,4,0), +'CXXABI': (1,3,3), +'GLIBCXX': (3,4,13), +'GLIBC': (2,11) +} +# See here for a description of _IO_stdin_used: +# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=634261#109 + +# Ignore symbols that are exported as part of every executable +IGNORE_EXPORTS = { +'_edata', '_end', '_init', '__bss_start', '_fini', '_IO_stdin_used', 'stdin', 'stdout', 'stderr' +} +READELF_CMD = os.getenv('READELF', '/usr/bin/readelf') +CPPFILT_CMD = os.getenv('CPPFILT', '/usr/bin/c++filt') +# Allowed NEEDED libraries +ALLOWED_LIBRARIES = { +# bitcoind and bitcoin-qt +'libgcc_s.so.1', # GCC base support +'libc.so.6', # C library +'libpthread.so.0', # threading +'libanl.so.1', # DNS resolve +'libm.so.6', # math library +'librt.so.1', # real-time (clock) +'ld-linux-x86-64.so.2', # 64-bit dynamic linker +'ld-linux.so.2', # 32-bit dynamic linker +# bitcoin-qt only +'libX11-xcb.so.1', # part of X11 +'libX11.so.6', # part of X11 +'libxcb.so.1', # part of X11 +'libfontconfig.so.1', # font support +'libfreetype.so.6', # font parsing +'libdl.so.2' # programming interface to dynamic linker +} + +class CPPFilt(object): + ''' + Demangle C++ symbol names. + + Use a pipe to the 'c++filt' command. + ''' + def __init__(self): + self.proc = subprocess.Popen(CPPFILT_CMD, stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) + + def __call__(self, mangled): + self.proc.stdin.write(mangled + '\n') + self.proc.stdin.flush() + return self.proc.stdout.readline().rstrip() + + def close(self): + self.proc.stdin.close() + self.proc.stdout.close() + self.proc.wait() + +def read_symbols(executable, imports=True): + ''' + Parse an ELF executable and return a list of (symbol,version) tuples + for dynamic, imported symbols. + ''' + p = subprocess.Popen([READELF_CMD, '--dyn-syms', '-W', executable], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) + (stdout, stderr) = p.communicate() + if p.returncode: + raise IOError('Could not read symbols for %s: %s' % (executable, stderr.strip())) + syms = [] + for line in stdout.splitlines(): + line = line.split() + if len(line)>7 and re.match('[0-9]+:$', line[0]): + (sym, _, version) = line[7].partition('@') + is_import = line[6] == 'UND' + if version.startswith('@'): + version = version[1:] + if is_import == imports: + syms.append((sym, version)) + return syms + +def check_version(max_versions, version): + if '_' in version: + (lib, _, ver) = version.rpartition('_') + else: + lib = version + ver = '0' + ver = tuple([int(x) for x in ver.split('.')]) + if not lib in max_versions: + return False + return ver <= max_versions[lib] + +def read_libraries(filename): + p = subprocess.Popen([READELF_CMD, '-d', '-W', filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) + (stdout, stderr) = p.communicate() + if p.returncode: + raise IOError('Error opening file') + libraries = [] + for line in stdout.splitlines(): + tokens = line.split() + if len(tokens)>2 and tokens[1] == '(NEEDED)': + match = re.match('^Shared library: \[(.*)\]$', ' '.join(tokens[2:])) + if match: + libraries.append(match.group(1)) + else: + raise ValueError('Unparseable (NEEDED) specification') + return libraries + +if __name__ == '__main__': + cppfilt = CPPFilt() + retval = 0 + for filename in sys.argv[1:]: + # Check imported symbols + for sym,version in read_symbols(filename, True): + if version and not check_version(MAX_VERSIONS, version): + print('%s: symbol %s from unsupported version %s' % (filename, cppfilt(sym), version)) + retval = 1 + # Check exported symbols + for sym,version in read_symbols(filename, False): + if sym in IGNORE_EXPORTS: + continue + print('%s: export of symbol %s not allowed' % (filename, cppfilt(sym))) + retval = 1 + # Check dependency libraries + for library_name in read_libraries(filename): + if library_name not in ALLOWED_LIBRARIES: + print('%s: NEEDED library %s is not allowed' % (filename, library_name)) + retval = 1 + + sys.exit(retval) + + |