From 24c3a6db9bb0471e0f280b338b53b8bd55f52314 Mon Sep 17 00:00:00 2001 From: Ed Schouten Date: Sun, 14 Jan 2018 18:37:52 +0100 Subject: [PATCH 1/4] Add a Docker container for doing automated builds for CloudABI. Setting up a cross compilation toolchain for CloudABI is relatively easy. It's just a matter of installing a somewhat recent version of Clang (5.0 preferred) and installing the corresponding ${target}-cxx-runtime package, containing a set of core C/C++ libraries (libc, libc++, libunwind, etc). Eventually it would be nice if we could also run 'x.py test'. That, however still requires some more work. Both libtest and compiletest would need to be adjusted to deal with CloudABI's requirement of having all of an application's dependencies injected. Let's settle for just doing 'x.py dist' for now. --- .travis.yml | 2 + src/ci/docker/dist-cloudabi/Dockerfile | 30 +++++++++++ src/ci/docker/scripts/cloudabi-toolchain.sh | 59 +++++++++++++++++++++ 3 files changed, 91 insertions(+) create mode 100644 src/ci/docker/dist-cloudabi/Dockerfile create mode 100755 src/ci/docker/scripts/cloudabi-toolchain.sh diff --git a/.travis.yml b/.travis.yml index 6e242b74894..1642a399552 100644 --- a/.travis.yml +++ b/.travis.yml @@ -126,6 +126,8 @@ matrix: if: branch = auto - env: IMAGE=dist-armv7-linux DEPLOY=1 if: branch = auto + - env: IMAGE=dist-cloudabi DEPLOY=1 + if: branch = auto - env: IMAGE=dist-i586-gnu-i586-i686-musl DEPLOY=1 if: branch = auto - env: IMAGE=dist-i686-freebsd DEPLOY=1 diff --git a/src/ci/docker/dist-cloudabi/Dockerfile b/src/ci/docker/dist-cloudabi/Dockerfile new file mode 100644 index 00000000000..f1f6f0ff6ca --- /dev/null +++ b/src/ci/docker/dist-cloudabi/Dockerfile @@ -0,0 +1,30 @@ +FROM ubuntu:17.10 + +ENV TARGETS=aarch64-unknown-cloudabi +# FIXME(EdSchouten): Enable ARMv7 support once libc ≥0.2.37 has been merged. +# ENV TARGETS=armv7-unknown-cloudabi-eabihf +ENV TARGETS=$TARGETS,i686-unknown-cloudabi +ENV TARGETS=$TARGETS,x86_64-unknown-cloudabi + +COPY scripts/cloudabi-toolchain.sh /tmp/ +RUN /tmp/cloudabi-toolchain.sh + +COPY scripts/sccache.sh /scripts/ +RUN sh /scripts/sccache.sh + +# FIXME(EdSchouten): Remove this once cc ≥1.0.4 has been merged. It can +# automatically pick the right compiler path. +ENV \ + AR_aarch64_unknown_cloudabi=aarch64-unknown-cloudabi-ar \ + CC_aarch64_unknown_cloudabi=aarch64-unknown-cloudabi-clang \ + CXX_aarch64_unknown_cloudabi=aarch64-unknown-cloudabi-clang++ \ + AR_i686_unknown_cloudabi=i686-unknown-cloudabi-ar \ + CC_i686_unknown_cloudabi=i686-unknown-cloudabi-clang \ + CXX_i686_unknown_cloudabi=i686-unknown-cloudabi-clang++ \ + AR_x86_64_unknown_cloudabi=x86_64-unknown-cloudabi-ar \ + CC_x86_64_unknown_cloudabi=x86_64-unknown-cloudabi-clang \ + CXX_x86_64_unknown_cloudabi=x86_64-unknown-cloudabi-clang++ + +# FIXME(EdSchouten): Work towards being able to run 'x.py test'. +ENV RUST_CONFIGURE_ARGS --target=${TARGETS} --disable-jemalloc +ENV SCRIPT python2.7 /checkout/x.py dist --target ${TARGETS} diff --git a/src/ci/docker/scripts/cloudabi-toolchain.sh b/src/ci/docker/scripts/cloudabi-toolchain.sh new file mode 100755 index 00000000000..5d6ecb48d3c --- /dev/null +++ b/src/ci/docker/scripts/cloudabi-toolchain.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# Copyright 2018 The Rust Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution and at +# http://rust-lang.org/COPYRIGHT. +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +set -eux + +# Install prerequisites. +apt-get update +apt-get install -y --no-install-recommends \ + apt-transport-https \ + ca-certificates \ + clang-5.0 \ + cmake \ + curl \ + file \ + g++ \ + gdb \ + git \ + lld-5.0 \ + make \ + python \ + sudo \ + xz-utils + +# Set up a Clang-based cross compiler toolchain. +# Based on the steps described at https://nuxi.nl/cloudabi/debian/ +IFS=, +for target in ${TARGETS}; do + for tool in ar nm objdump ranlib size; do + ln -s ../lib/llvm-5.0/bin/llvm-${tool} /usr/bin/${target}-${tool} + done + ln -s ../lib/llvm-5.0/bin/clang /usr/bin/${target}-cc + ln -s ../lib/llvm-5.0/bin/clang /usr/bin/${target}-c++ + ln -s ../lib/llvm-5.0/bin/lld /usr/bin/${target}-ld + ln -s ../../${target} /usr/lib/llvm-5.0/${target} + + # FIXME(EdSchouten): Remove this once cc ≥1.0.4 has been merged. It + # can make use of ${target}-cc and ${target}-c++, without incorrectly + # assuming it's MSVC. + ln -s ../lib/llvm-5.0/bin/clang /usr/bin/${target}-clang + ln -s ../lib/llvm-5.0/bin/clang /usr/bin/${target}-clang++ +done + +# Install the C++ runtime libraries from CloudABI Ports. +echo deb https://nuxi.nl/distfiles/cloudabi-ports/debian/ cloudabi cloudabi > \ + /etc/apt/sources.list.d/cloudabi.list +curl 'https://pgp.mit.edu/pks/lookup?op=get&search=0x0DA51B8531344B15' | \ + apt-key add - +apt-get update +for target in ${TARGETS}; do + apt-get install -y $(echo ${target} | sed -e s/_/-/g)-cxx-runtime +done From 5801f95cbc2183285bc4423b3405f2767e448ef6 Mon Sep 17 00:00:00 2001 From: Ed Schouten Date: Sun, 14 Jan 2018 19:47:41 +0100 Subject: [PATCH 2/4] Move dist-cloudabi/ into disabled/. There is not enough capacity to do automated builds for CloudABI at this time. --- .travis.yml | 2 -- src/ci/docker/{ => disabled}/dist-cloudabi/Dockerfile | 0 2 files changed, 2 deletions(-) rename src/ci/docker/{ => disabled}/dist-cloudabi/Dockerfile (100%) diff --git a/.travis.yml b/.travis.yml index 1642a399552..6e242b74894 100644 --- a/.travis.yml +++ b/.travis.yml @@ -126,8 +126,6 @@ matrix: if: branch = auto - env: IMAGE=dist-armv7-linux DEPLOY=1 if: branch = auto - - env: IMAGE=dist-cloudabi DEPLOY=1 - if: branch = auto - env: IMAGE=dist-i586-gnu-i586-i686-musl DEPLOY=1 if: branch = auto - env: IMAGE=dist-i686-freebsd DEPLOY=1 diff --git a/src/ci/docker/dist-cloudabi/Dockerfile b/src/ci/docker/disabled/dist-cloudabi/Dockerfile similarity index 100% rename from src/ci/docker/dist-cloudabi/Dockerfile rename to src/ci/docker/disabled/dist-cloudabi/Dockerfile From ae4288f9ffd2fd431dba04a5906b5304e9106755 Mon Sep 17 00:00:00 2001 From: Ed Schouten Date: Tue, 16 Jan 2018 22:23:18 +0100 Subject: [PATCH 3/4] Integrate dist-cloudabi into dist-various-2. As discussed in #47427, let's not have a separate container for doing CloudABI builds. It's a lot faster if we integrate it into an existing container, so there's less duplication of what's being built. Upgrade the existing container to Ubuntu 17.10, which is required for CloudABI builds. The version of Clang shipped with 16.04 is not recent enough to support CloudABI properly. --- .../docker/disabled/dist-cloudabi/Dockerfile | 30 ----------------- src/ci/docker/dist-various-2/Dockerfile | 29 +++++++++++++++-- .../build-cloudabi-toolchain.sh} | 32 ++++++++----------- 3 files changed, 40 insertions(+), 51 deletions(-) delete mode 100644 src/ci/docker/disabled/dist-cloudabi/Dockerfile rename src/ci/docker/{scripts/cloudabi-toolchain.sh => dist-various-2/build-cloudabi-toolchain.sh} (59%) diff --git a/src/ci/docker/disabled/dist-cloudabi/Dockerfile b/src/ci/docker/disabled/dist-cloudabi/Dockerfile deleted file mode 100644 index f1f6f0ff6ca..00000000000 --- a/src/ci/docker/disabled/dist-cloudabi/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -FROM ubuntu:17.10 - -ENV TARGETS=aarch64-unknown-cloudabi -# FIXME(EdSchouten): Enable ARMv7 support once libc ≥0.2.37 has been merged. -# ENV TARGETS=armv7-unknown-cloudabi-eabihf -ENV TARGETS=$TARGETS,i686-unknown-cloudabi -ENV TARGETS=$TARGETS,x86_64-unknown-cloudabi - -COPY scripts/cloudabi-toolchain.sh /tmp/ -RUN /tmp/cloudabi-toolchain.sh - -COPY scripts/sccache.sh /scripts/ -RUN sh /scripts/sccache.sh - -# FIXME(EdSchouten): Remove this once cc ≥1.0.4 has been merged. It can -# automatically pick the right compiler path. -ENV \ - AR_aarch64_unknown_cloudabi=aarch64-unknown-cloudabi-ar \ - CC_aarch64_unknown_cloudabi=aarch64-unknown-cloudabi-clang \ - CXX_aarch64_unknown_cloudabi=aarch64-unknown-cloudabi-clang++ \ - AR_i686_unknown_cloudabi=i686-unknown-cloudabi-ar \ - CC_i686_unknown_cloudabi=i686-unknown-cloudabi-clang \ - CXX_i686_unknown_cloudabi=i686-unknown-cloudabi-clang++ \ - AR_x86_64_unknown_cloudabi=x86_64-unknown-cloudabi-ar \ - CC_x86_64_unknown_cloudabi=x86_64-unknown-cloudabi-clang \ - CXX_x86_64_unknown_cloudabi=x86_64-unknown-cloudabi-clang++ - -# FIXME(EdSchouten): Work towards being able to run 'x.py test'. -ENV RUST_CONFIGURE_ARGS --target=${TARGETS} --disable-jemalloc -ENV SCRIPT python2.7 /checkout/x.py dist --target ${TARGETS} diff --git a/src/ci/docker/dist-various-2/Dockerfile b/src/ci/docker/dist-various-2/Dockerfile index c7885db559a..5284a1c875c 100644 --- a/src/ci/docker/dist-various-2/Dockerfile +++ b/src/ci/docker/dist-various-2/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:16.04 +FROM ubuntu:17.10 COPY scripts/cross-apt-packages.sh /scripts/ RUN sh /scripts/cross-apt-packages.sh @@ -21,9 +21,14 @@ RUN apt-key adv --batch --yes --keyserver keyserver.ubuntu.com --recv-keys 74DA7 RUN add-apt-repository -y 'deb http://apt.dilos.org/dilos dilos2-testing main' WORKDIR /tmp -COPY dist-various-2/shared.sh dist-various-2/build-fuchsia-toolchain.sh /tmp/ -COPY dist-various-2/build-solaris-toolchain.sh /tmp/ +COPY dist-various-2/shared.sh /tmp/ +COPY dist-various-2/build-cloudabi-toolchain.sh /tmp/ +RUN /tmp/build-cloudabi-toolchain.sh aarch64-unknown-cloudabi +RUN /tmp/build-cloudabi-toolchain.sh i686-unknown-cloudabi +RUN /tmp/build-cloudabi-toolchain.sh x86_64-unknown-cloudabi +COPY dist-various-2/build-fuchsia-toolchain.sh /tmp/ RUN /tmp/build-fuchsia-toolchain.sh +COPY dist-various-2/build-solaris-toolchain.sh /tmp/ RUN /tmp/build-solaris-toolchain.sh x86_64 amd64 solaris-i386 RUN /tmp/build-solaris-toolchain.sh sparcv9 sparcv9 solaris-sparc @@ -44,12 +49,30 @@ ENV \ CC_x86_64_sun_solaris=x86_64-sun-solaris2.10-gcc \ CXX_x86_64_sun_solaris=x86_64-sun-solaris2.10-g++ +# FIXME(EdSchouten): Remove this once cc ≥1.0.4 has been merged. It can +# automatically pick the right compiler path. +ENV \ + AR_aarch64_unknown_cloudabi=aarch64-unknown-cloudabi-ar \ + CC_aarch64_unknown_cloudabi=aarch64-unknown-cloudabi-clang \ + CXX_aarch64_unknown_cloudabi=aarch64-unknown-cloudabi-clang++ \ + AR_i686_unknown_cloudabi=i686-unknown-cloudabi-ar \ + CC_i686_unknown_cloudabi=i686-unknown-cloudabi-clang \ + CXX_i686_unknown_cloudabi=i686-unknown-cloudabi-clang++ \ + AR_x86_64_unknown_cloudabi=x86_64-unknown-cloudabi-ar \ + CC_x86_64_unknown_cloudabi=x86_64-unknown-cloudabi-clang \ + CXX_x86_64_unknown_cloudabi=x86_64-unknown-cloudabi-clang++ + ENV TARGETS=x86_64-unknown-fuchsia ENV TARGETS=$TARGETS,aarch64-unknown-fuchsia ENV TARGETS=$TARGETS,sparcv9-sun-solaris ENV TARGETS=$TARGETS,wasm32-unknown-unknown ENV TARGETS=$TARGETS,x86_64-sun-solaris ENV TARGETS=$TARGETS,x86_64-unknown-linux-gnux32 +ENV TARGETS=$TARGETS,aarch64-unknown-cloudabi +# FIXME(EdSchouten): Enable ARMv7 support once libc ≥0.2.37 has been merged. +# ENV TARGETS=$TARGETS,armv7-unknown-cloudabi-eabihf +ENV TARGETS=$TARGETS,i686-unknown-cloudabi +ENV TARGETS=$TARGETS,x86_64-unknown-cloudabi ENV RUST_CONFIGURE_ARGS --target=$TARGETS --enable-extended ENV SCRIPT python2.7 ../x.py dist --target $TARGETS diff --git a/src/ci/docker/scripts/cloudabi-toolchain.sh b/src/ci/docker/dist-various-2/build-cloudabi-toolchain.sh similarity index 59% rename from src/ci/docker/scripts/cloudabi-toolchain.sh rename to src/ci/docker/dist-various-2/build-cloudabi-toolchain.sh index 5d6ecb48d3c..d64da436639 100755 --- a/src/ci/docker/scripts/cloudabi-toolchain.sh +++ b/src/ci/docker/dist-various-2/build-cloudabi-toolchain.sh @@ -31,22 +31,20 @@ apt-get install -y --no-install-recommends \ # Set up a Clang-based cross compiler toolchain. # Based on the steps described at https://nuxi.nl/cloudabi/debian/ -IFS=, -for target in ${TARGETS}; do - for tool in ar nm objdump ranlib size; do - ln -s ../lib/llvm-5.0/bin/llvm-${tool} /usr/bin/${target}-${tool} - done - ln -s ../lib/llvm-5.0/bin/clang /usr/bin/${target}-cc - ln -s ../lib/llvm-5.0/bin/clang /usr/bin/${target}-c++ - ln -s ../lib/llvm-5.0/bin/lld /usr/bin/${target}-ld - ln -s ../../${target} /usr/lib/llvm-5.0/${target} - - # FIXME(EdSchouten): Remove this once cc ≥1.0.4 has been merged. It - # can make use of ${target}-cc and ${target}-c++, without incorrectly - # assuming it's MSVC. - ln -s ../lib/llvm-5.0/bin/clang /usr/bin/${target}-clang - ln -s ../lib/llvm-5.0/bin/clang /usr/bin/${target}-clang++ +target=$1 +for tool in ar nm objdump ranlib size; do + ln -s ../lib/llvm-5.0/bin/llvm-${tool} /usr/bin/${target}-${tool} done +ln -s ../lib/llvm-5.0/bin/clang /usr/bin/${target}-cc +ln -s ../lib/llvm-5.0/bin/clang /usr/bin/${target}-c++ +ln -s ../lib/llvm-5.0/bin/lld /usr/bin/${target}-ld +ln -s ../../${target} /usr/lib/llvm-5.0/${target} + +# FIXME(EdSchouten): Remove this once cc ≥1.0.4 has been merged. It +# can make use of ${target}-cc and ${target}-c++, without incorrectly +# assuming it's MSVC. +ln -s ../lib/llvm-5.0/bin/clang /usr/bin/${target}-clang +ln -s ../lib/llvm-5.0/bin/clang /usr/bin/${target}-clang++ # Install the C++ runtime libraries from CloudABI Ports. echo deb https://nuxi.nl/distfiles/cloudabi-ports/debian/ cloudabi cloudabi > \ @@ -54,6 +52,4 @@ echo deb https://nuxi.nl/distfiles/cloudabi-ports/debian/ cloudabi cloudabi > \ curl 'https://pgp.mit.edu/pks/lookup?op=get&search=0x0DA51B8531344B15' | \ apt-key add - apt-get update -for target in ${TARGETS}; do - apt-get install -y $(echo ${target} | sed -e s/_/-/g)-cxx-runtime -done +apt-get install -y $(echo ${target} | sed -e s/_/-/g)-cxx-runtime From dcf0cd0ac00a91afcc24e4c073dd99befa188091 Mon Sep 17 00:00:00 2001 From: Ed Schouten Date: Tue, 16 Jan 2018 23:21:51 +0100 Subject: [PATCH 4/4] Only enable CloudABI builds for x86-64 for now. We'll turn on other architectures if it turns out we have enough capacity. --- src/ci/docker/dist-various-2/Dockerfile | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/ci/docker/dist-various-2/Dockerfile b/src/ci/docker/dist-various-2/Dockerfile index 5284a1c875c..d8f09bf47a4 100644 --- a/src/ci/docker/dist-various-2/Dockerfile +++ b/src/ci/docker/dist-various-2/Dockerfile @@ -23,8 +23,6 @@ RUN add-apt-repository -y 'deb http://apt.dilos.org/dilos dilos2-testing main' WORKDIR /tmp COPY dist-various-2/shared.sh /tmp/ COPY dist-various-2/build-cloudabi-toolchain.sh /tmp/ -RUN /tmp/build-cloudabi-toolchain.sh aarch64-unknown-cloudabi -RUN /tmp/build-cloudabi-toolchain.sh i686-unknown-cloudabi RUN /tmp/build-cloudabi-toolchain.sh x86_64-unknown-cloudabi COPY dist-various-2/build-fuchsia-toolchain.sh /tmp/ RUN /tmp/build-fuchsia-toolchain.sh @@ -52,12 +50,6 @@ ENV \ # FIXME(EdSchouten): Remove this once cc ≥1.0.4 has been merged. It can # automatically pick the right compiler path. ENV \ - AR_aarch64_unknown_cloudabi=aarch64-unknown-cloudabi-ar \ - CC_aarch64_unknown_cloudabi=aarch64-unknown-cloudabi-clang \ - CXX_aarch64_unknown_cloudabi=aarch64-unknown-cloudabi-clang++ \ - AR_i686_unknown_cloudabi=i686-unknown-cloudabi-ar \ - CC_i686_unknown_cloudabi=i686-unknown-cloudabi-clang \ - CXX_i686_unknown_cloudabi=i686-unknown-cloudabi-clang++ \ AR_x86_64_unknown_cloudabi=x86_64-unknown-cloudabi-ar \ CC_x86_64_unknown_cloudabi=x86_64-unknown-cloudabi-clang \ CXX_x86_64_unknown_cloudabi=x86_64-unknown-cloudabi-clang++ @@ -68,10 +60,6 @@ ENV TARGETS=$TARGETS,sparcv9-sun-solaris ENV TARGETS=$TARGETS,wasm32-unknown-unknown ENV TARGETS=$TARGETS,x86_64-sun-solaris ENV TARGETS=$TARGETS,x86_64-unknown-linux-gnux32 -ENV TARGETS=$TARGETS,aarch64-unknown-cloudabi -# FIXME(EdSchouten): Enable ARMv7 support once libc ≥0.2.37 has been merged. -# ENV TARGETS=$TARGETS,armv7-unknown-cloudabi-eabihf -ENV TARGETS=$TARGETS,i686-unknown-cloudabi ENV TARGETS=$TARGETS,x86_64-unknown-cloudabi ENV RUST_CONFIGURE_ARGS --target=$TARGETS --enable-extended