You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
477 lines
17 KiB
477 lines
17 KiB
#!/usr/bin/env bash |
|
export LC_ALL=C |
|
set -e -o pipefail |
|
|
|
# Source the common prelude, which: |
|
# 1. Checks if we're at the top directory of the Bitcoin Core repository |
|
# 2. Defines a few common functions and variables |
|
# |
|
# shellcheck source=libexec/prelude.bash |
|
source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" |
|
|
|
|
|
################### |
|
## SANITY CHECKS ## |
|
################### |
|
|
|
################ |
|
# Required non-builtin commands should be invocable |
|
################ |
|
|
|
check_tools cat mkdir make getent curl git guix |
|
|
|
################ |
|
# GUIX_BUILD_OPTIONS should be empty |
|
################ |
|
# |
|
# GUIX_BUILD_OPTIONS is an environment variable recognized by guix commands that |
|
# can perform builds. This seems like what we want instead of |
|
# ADDITIONAL_GUIX_COMMON_FLAGS, but the value of GUIX_BUILD_OPTIONS is actually |
|
# _appended_ to normal command-line options. Meaning that they will take |
|
# precedence over the command-specific ADDITIONAL_GUIX_<CMD>_FLAGS. |
|
# |
|
# This seems like a poor user experience. Thus we check for GUIX_BUILD_OPTIONS's |
|
# existence here and direct users of this script to use our (more flexible) |
|
# custom environment variables. |
|
if [ -n "$GUIX_BUILD_OPTIONS" ]; then |
|
cat << EOF |
|
Error: Environment variable GUIX_BUILD_OPTIONS is not empty: |
|
'$GUIX_BUILD_OPTIONS' |
|
|
|
Unfortunately this script is incompatible with GUIX_BUILD_OPTIONS, please unset |
|
GUIX_BUILD_OPTIONS and use ADDITIONAL_GUIX_COMMON_FLAGS to set build options |
|
across guix commands or ADDITIONAL_GUIX_<CMD>_FLAGS to set build options for a |
|
specific guix command. |
|
|
|
See contrib/guix/README.md for more details. |
|
EOF |
|
exit 1 |
|
fi |
|
|
|
################ |
|
# The git worktree should not be dirty |
|
################ |
|
|
|
if ! git diff-index --quiet HEAD -- && [ -z "$FORCE_DIRTY_WORKTREE" ]; then |
|
cat << EOF |
|
ERR: The current git worktree is dirty, which may lead to broken builds. |
|
|
|
Aborting... |
|
|
|
Hint: To make your git worktree clean, You may want to: |
|
1. Commit your changes, |
|
2. Stash your changes, or |
|
3. Set the 'FORCE_DIRTY_WORKTREE' environment variable if you insist on |
|
using a dirty worktree |
|
EOF |
|
exit 1 |
|
fi |
|
|
|
mkdir -p "$VERSION_BASE" |
|
|
|
################ |
|
# Build directories should not exist |
|
################ |
|
|
|
# Default to building for all supported HOSTs (overridable by environment) |
|
export HOSTS="${HOSTS:-x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu powerpc64-linux-gnu powerpc64le-linux-gnu |
|
x86_64-w64-mingw32 |
|
x86_64-apple-darwin18}" |
|
|
|
# Usage: distsrc_for_host HOST |
|
# |
|
# HOST: The current platform triple we're building for |
|
# |
|
distsrc_for_host() { |
|
echo "${DISTSRC_BASE}/distsrc-${VERSION}-${1}" |
|
} |
|
|
|
# Accumulate a list of build directories that already exist... |
|
hosts_distsrc_exists="" |
|
for host in $HOSTS; do |
|
if [ -e "$(distsrc_for_host "$host")" ]; then |
|
hosts_distsrc_exists+=" ${host}" |
|
fi |
|
done |
|
|
|
if [ -n "$hosts_distsrc_exists" ]; then |
|
# ...so that we can print them out nicely in an error message |
|
cat << EOF |
|
ERR: Build directories for this commit already exist for the following platform |
|
triples you're attempting to build, probably because of previous builds. |
|
Please remove, or otherwise deal with them prior to starting another build. |
|
|
|
Aborting... |
|
|
|
Hint: To blow everything away, you may want to use: |
|
|
|
$ ./contrib/guix/guix-clean |
|
|
|
Specifically, this will remove all files without an entry in the index, |
|
excluding the SDK directory, the depends download cache, the depends built |
|
packages cache, the garbage collector roots for Guix environments, and the |
|
output directory. |
|
EOF |
|
for host in $hosts_distsrc_exists; do |
|
echo " ${host} '$(distsrc_for_host "$host")'" |
|
done |
|
exit 1 |
|
else |
|
mkdir -p "$DISTSRC_BASE" |
|
fi |
|
|
|
################ |
|
# When building for darwin, the macOS SDK should exists |
|
################ |
|
|
|
for host in $HOSTS; do |
|
case "$host" in |
|
*darwin*) |
|
OSX_SDK="$(make -C "${PWD}/depends" --no-print-directory HOST="$host" print-OSX_SDK | sed 's@^[^=]\+=@@g')" |
|
if [ -e "$OSX_SDK" ]; then |
|
echo "Found macOS SDK at '${OSX_SDK}', using..." |
|
else |
|
echo "macOS SDK does not exist at '${OSX_SDK}', please place the extracted, untarred SDK there to perform darwin builds, exiting..." |
|
exit 1 |
|
fi |
|
;; |
|
esac |
|
done |
|
|
|
################ |
|
# VERSION_BASE should have enough space |
|
################ |
|
|
|
avail_KiB="$(df -Pk "$VERSION_BASE" | sed 1d | tr -s ' ' | cut -d' ' -f4)" |
|
total_required_KiB=0 |
|
for host in $HOSTS; do |
|
case "$host" in |
|
*darwin*) required_KiB=440000 ;; |
|
*mingw*) required_KiB=7600000 ;; |
|
*) required_KiB=6400000 ;; |
|
esac |
|
total_required_KiB=$((total_required_KiB+required_KiB)) |
|
done |
|
|
|
if (( total_required_KiB > avail_KiB )); then |
|
total_required_GiB=$((total_required_KiB / 1048576)) |
|
avail_GiB=$((avail_KiB / 1048576)) |
|
echo "Performing a Bitcoin Core Guix build for the selected HOSTS requires ${total_required_GiB} GiB, however, only ${avail_GiB} GiB is available. Please free up some disk space before performing the build." |
|
exit 1 |
|
fi |
|
|
|
################ |
|
# Check that we can connect to the guix-daemon |
|
################ |
|
|
|
cat << EOF |
|
Checking that we can connect to the guix-daemon... |
|
|
|
Hint: If this hangs, you may want to try turning your guix-daemon off and on |
|
again. |
|
|
|
EOF |
|
if ! guix gc --list-failures > /dev/null; then |
|
cat << EOF |
|
|
|
ERR: Failed to connect to the guix-daemon, please ensure that one is running and |
|
reachable. |
|
EOF |
|
exit 1 |
|
fi |
|
|
|
# Developer note: we could use `guix repl` for this check and run: |
|
# |
|
# (import (guix store)) (close-connection (open-connection)) |
|
# |
|
# However, the internal API is likely to change more than the CLI invocation |
|
|
|
################ |
|
# Services database must have basic entries |
|
################ |
|
|
|
if ! getent services http https ftp > /dev/null 2>&1; then |
|
cat << EOF |
|
ERR: Your system's C library can not find service database entries for at least |
|
one of the following services: http, https, ftp. |
|
|
|
Hint: Most likely, /etc/services does not exist yet (common for docker images |
|
and minimal distros), or you don't have permissions to access it. |
|
|
|
If /etc/services does not exist yet, you may want to install the |
|
appropriate package for your distro which provides it. |
|
|
|
On Debian/Ubuntu: netbase |
|
On Arch Linux: iana-etc |
|
|
|
For more information, see: getent(1), services(5) |
|
|
|
EOF |
|
|
|
fi |
|
|
|
######### |
|
# SETUP # |
|
######### |
|
|
|
# Determine the maximum number of jobs to run simultaneously (overridable by |
|
# environment) |
|
JOBS="${JOBS:-$(nproc)}" |
|
|
|
# Usage: host_to_commonname HOST |
|
# |
|
# HOST: The current platform triple we're building for |
|
# |
|
host_to_commonname() { |
|
case "$1" in |
|
*darwin*) echo osx ;; |
|
*mingw*) echo win ;; |
|
*linux*) echo linux ;; |
|
*) exit 1 ;; |
|
esac |
|
} |
|
|
|
# Determine the reference time used for determinism (overridable by environment) |
|
SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}" |
|
|
|
# Execute "$@" in a pinned, possibly older version of Guix, for reproducibility |
|
# across time. |
|
time-machine() { |
|
# shellcheck disable=SC2086 |
|
guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ |
|
--commit=aa34d4d28dfe25ba47d5800d05000fb7221788c0 \ |
|
--cores="$JOBS" \ |
|
--keep-failed \ |
|
--fallback \ |
|
${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ |
|
${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \ |
|
-- "$@" |
|
} |
|
|
|
|
|
# Precious directories are those which should not be cleaned between successive |
|
# guix builds |
|
depends_precious_dir_names='SOURCES_PATH BASE_CACHE SDK_PATH' |
|
precious_dir_names="${depends_precious_dir_names} OUTDIR_BASE PROFILES_BASE" |
|
|
|
# Usage: contains IFS-SEPARATED-LIST ITEM |
|
contains() { |
|
for i in ${1}; do |
|
if [ "$i" = "${2}" ]; then |
|
return 0 # Found! |
|
fi |
|
done |
|
return 1 |
|
} |
|
|
|
# If the user explicitly specified a precious directory, create it so we |
|
# can map it into the container |
|
for precious_dir_name in $precious_dir_names; do |
|
precious_dir_path="${!precious_dir_name}" |
|
if [ -n "$precious_dir_path" ]; then |
|
if [ ! -e "$precious_dir_path" ]; then |
|
mkdir -p "$precious_dir_path" |
|
elif [ -L "$precious_dir_path" ]; then |
|
echo "ERR: ${precious_dir_name} cannot be a symbolic link" |
|
exit 1 |
|
elif [ ! -d "$precious_dir_path" ]; then |
|
echo "ERR: ${precious_dir_name} must be a directory" |
|
exit 1 |
|
fi |
|
fi |
|
done |
|
|
|
mkdir -p "$VAR_BASE" |
|
|
|
# Record the _effective_ values of precious directories such that guix-clean can |
|
# avoid clobbering them if appropriate. |
|
# |
|
# shellcheck disable=SC2046,SC2086 |
|
{ |
|
# Get depends precious dir definitions from depends |
|
make -C "${PWD}/depends" \ |
|
--no-print-directory \ |
|
-- $(printf "print-%s\n" $depends_precious_dir_names) |
|
|
|
# Get remaining precious dir definitions from the environment |
|
for precious_dir_name in $precious_dir_names; do |
|
precious_dir_path="${!precious_dir_name}" |
|
if ! contains "$depends_precious_dir_names" "$precious_dir_name"; then |
|
echo "${precious_dir_name}=${precious_dir_path}" |
|
fi |
|
done |
|
} > "${VAR_BASE}/precious_dirs" |
|
|
|
# Make sure an output directory exists for our builds |
|
OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" |
|
mkdir -p "$OUTDIR_BASE" |
|
|
|
# Download the depends sources now as we won't have internet access in the build |
|
# container |
|
for host in $HOSTS; do |
|
make -C "${PWD}/depends" -j"$JOBS" download-"$(host_to_commonname "$host")" ${V:+V=1} ${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} |
|
done |
|
|
|
# Usage: outdir_for_host HOST SUFFIX |
|
# |
|
# HOST: The current platform triple we're building for |
|
# |
|
outdir_for_host() { |
|
echo "${OUTDIR_BASE}/${1}${2:+-${2}}" |
|
} |
|
|
|
# Usage: profiledir_for_host HOST SUFFIX |
|
# |
|
# HOST: The current platform triple we're building for |
|
# |
|
profiledir_for_host() { |
|
echo "${PROFILES_BASE}/${1}${2:+-${2}}" |
|
} |
|
|
|
|
|
######### |
|
# BUILD # |
|
######### |
|
|
|
# Function to be called when building for host ${1} and the user interrupts the |
|
# build |
|
int_trap() { |
|
cat << EOF |
|
** INT received while building ${1}, you may want to clean up the relevant |
|
work directories (e.g. distsrc-*) before rebuilding |
|
|
|
Hint: To blow everything away, you may want to use: |
|
|
|
$ ./contrib/guix/guix-clean |
|
|
|
Specifically, this will remove all files without an entry in the index, |
|
excluding the SDK directory, the depends download cache, the depends built |
|
packages cache, the garbage collector roots for Guix environments, and the |
|
output directory. |
|
EOF |
|
} |
|
|
|
# Deterministically build Bitcoin Core |
|
# shellcheck disable=SC2153 |
|
for host in $HOSTS; do |
|
|
|
# Display proper warning when the user interrupts the build |
|
trap 'int_trap ${host}' INT |
|
|
|
( |
|
# Required for 'contrib/guix/manifest.scm' to output the right manifest |
|
# for the particular $HOST we're building for |
|
export HOST="$host" |
|
|
|
# shellcheck disable=SC2030 |
|
cat << EOF |
|
INFO: Building ${VERSION:?not set} for platform triple ${HOST:?not set}: |
|
...using reference timestamp: ${SOURCE_DATE_EPOCH:?not set} |
|
...running at most ${JOBS:?not set} jobs |
|
...from worktree directory: '${PWD}' |
|
...bind-mounted in container to: '/bitcoin' |
|
...in build directory: '$(distsrc_for_host "$HOST")' |
|
...bind-mounted in container to: '$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")' |
|
...outputting in: '$(outdir_for_host "$HOST")' |
|
...bind-mounted in container to: '$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")' |
|
EOF |
|
|
|
# Run the build script 'contrib/guix/libexec/build.sh' in the build |
|
# container specified by 'contrib/guix/manifest.scm'. |
|
# |
|
# Explanation of `guix environment` flags: |
|
# |
|
# --container run command within an isolated container |
|
# |
|
# Running in an isolated container minimizes build-time differences |
|
# between machines and improves reproducibility |
|
# |
|
# --pure unset existing environment variables |
|
# |
|
# Same rationale as --container |
|
# |
|
# --no-cwd do not share current working directory with an |
|
# isolated container |
|
# |
|
# When --container is specified, the default behavior is to share |
|
# the current working directory with the isolated container at the |
|
# same exact path (e.g. mapping '/home/satoshi/bitcoin/' to |
|
# '/home/satoshi/bitcoin/'). This means that the $PWD inside the |
|
# container becomes a source of irreproducibility. --no-cwd disables |
|
# this behaviour. |
|
# |
|
# --share=SPEC for containers, share writable host file system |
|
# according to SPEC |
|
# |
|
# --share="$PWD"=/bitcoin |
|
# |
|
# maps our current working directory to /bitcoin |
|
# inside the isolated container, which we later cd |
|
# into. |
|
# |
|
# While we don't want to map our current working directory to the |
|
# same exact path (as this introduces irreproducibility), we do want |
|
# it to be at a _fixed_ path _somewhere_ inside the isolated |
|
# container so that we have something to build. '/bitcoin' was |
|
# chosen arbitrarily. |
|
# |
|
# ${SOURCES_PATH:+--share="$SOURCES_PATH"} |
|
# |
|
# make the downloaded depends sources path available |
|
# inside the isolated container |
|
# |
|
# The isolated container has no network access as it's in a |
|
# different network namespace from the main machine, so we have to |
|
# make the downloaded depends sources available to it. The sources |
|
# should have been downloaded prior to this invocation. |
|
# |
|
# --keep-failed keep build tree of failed builds |
|
# |
|
# When builds of the Guix environment itself (not Bitcoin Core) |
|
# fail, it is useful for the build tree to be kept for debugging |
|
# purposes. |
|
# |
|
# ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} |
|
# |
|
# fetch substitute from SUBSTITUTE_URLS if they are |
|
# authorized |
|
# |
|
# Depending on the user's security model, it may be desirable to use |
|
# substitutes (pre-built packages) from servers that the user trusts. |
|
# Please read the README.md in the same directory as this file for |
|
# more information. |
|
# |
|
# shellcheck disable=SC2086,SC2031 |
|
time-machine environment --manifest="${PWD}/contrib/guix/manifest.scm" \ |
|
--container \ |
|
--pure \ |
|
--no-cwd \ |
|
--share="$PWD"=/bitcoin \ |
|
--share="$DISTSRC_BASE"=/distsrc-base \ |
|
--share="$OUTDIR_BASE"=/outdir-base \ |
|
--expose="$(git rev-parse --git-common-dir)" \ |
|
${SOURCES_PATH:+--share="$SOURCES_PATH"} \ |
|
${BASE_CACHE:+--share="$BASE_CACHE"} \ |
|
${SDK_PATH:+--share="$SDK_PATH"} \ |
|
--cores="$JOBS" \ |
|
--keep-failed \ |
|
--fallback \ |
|
--link-profile \ |
|
--root="$(profiledir_for_host "${HOST}")" \ |
|
${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ |
|
${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ |
|
-- env HOST="$host" \ |
|
DISTNAME="$DISTNAME" \ |
|
JOBS="$JOBS" \ |
|
SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:?unable to determine value}" \ |
|
${V:+V=1} \ |
|
${SOURCES_PATH:+SOURCES_PATH="$SOURCES_PATH"} \ |
|
${BASE_CACHE:+BASE_CACHE="$BASE_CACHE"} \ |
|
${SDK_PATH:+SDK_PATH="$SDK_PATH"} \ |
|
DISTSRC="$(DISTSRC_BASE=/distsrc-base && distsrc_for_host "$HOST")" \ |
|
OUTDIR="$(OUTDIR_BASE=/outdir-base && outdir_for_host "$HOST")" \ |
|
DIST_ARCHIVE_BASE=/outdir-base/dist-archive \ |
|
bash -c "cd /bitcoin && bash contrib/guix/libexec/build.sh" |
|
) |
|
|
|
done
|
|
|