mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Auto merge of #38748 - alexcrichton:upload, r=brson

Browse Source 
travis: Start uploading artifacts on commits

This commit starts adding the infrastructure for uploading release artifacts
from AppVeyor/Travis on each commit. The idea is that eventually we'll upload a
full release to AppVeyor/Travis in accordance with plans [outlined earlier].

Right now this configures Travis/Appveyor to upload all tarballs in the `dist`
directory, and various images are updated to actually produce tarballs in these
directories. These are nowhere near ready to be actual release artifacts, but
this should allow us to play around with it and test it out. Once this commit
lands we should start seeing artifacts uploaded on each commit.

[outlined earlier]: https://internals.rust-lang.org/t/rust-ci-release-infrastructure-changes/4489
This commit is contained in:
bors 2017-01-13 02:56:20 +00:00
commit 927c55d86b
19 changed files with 147 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
.travis.yml
View File

@ -13,20 +13,20 @@ osx_image: xcode8.2
matrix:
include:
# Linux builders, all docker images
- env: IMAGE=arm-android
- env: IMAGE=cross
- env: IMAGE=dist-arm-unknown-linux-gnueabi
- env: IMAGE=dist-x86_64-unknown-freebsd
- env: IMAGE=i686-gnu
- env: IMAGE=arm-android DEPLOY=1
- env: IMAGE=cross DEPLOY=1
- env: IMAGE=dist-arm-unknown-linux-gnueabi DEPLOY=1
- env: IMAGE=dist-x86_64-unknown-freebsd DEPLOY=1
- env: IMAGE=i686-gnu DEPLOY=1
- env: IMAGE=i686-gnu-nopt
- env: IMAGE=x86_64-gnu
- env: IMAGE=x86_64-gnu DEPLOY=1
- env: IMAGE=x86_64-gnu-full-bootstrap
- env: IMAGE=x86_64-gnu-aux
- env: IMAGE=x86_64-gnu-debug
- env: IMAGE=x86_64-gnu-nopt
- env: IMAGE=x86_64-gnu-make
- env: IMAGE=x86_64-gnu-llvm-3.7 ALLOW_PR=1 RUST_BACKTRACE=1
- env: IMAGE=x86_64-musl
- env: IMAGE=x86_64-musl DEPLOY=1
- env: IMAGE=x86_64-gnu-distcheck
# OSX builders
@ -39,9 +39,10 @@ matrix:
curl -L https://api.pub.build.mozilla.org/tooltool/sha512/d0025b286468cc5ada83b23d3fafbc936b9f190eaa7d4a981715b18e8e3bf720a7bcee7bfe758cfdeb8268857f6098fd52dcdd8818232692a30ce91039936596 |
tar xJf - -C /usr/local/bin --strip-components=1
- env: >
RUST_CHECK_TARGET=check
SCRIPT="./x.py test && ./x.py dist"
RUST_CONFIGURE_ARGS=--build=i686-apple-darwin
SRC=.
DEPLOY=1
os: osx
install: *osx_install_sccache
- env: >
@ -51,9 +52,10 @@ matrix:
os: osx
install: *osx_install_sccache
- env: >
RUST_CHECK_TARGET=
RUST_CHECK_TARGET=dist
RUST_CONFIGURE_ARGS=--target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios
SRC=.
DEPLOY=1
os: osx
install: *osx_install_sccache
@ -91,3 +93,29 @@ notifications:
cache:
directories:
- $HOME/docker
before_deploy:
- mkdir -p deploy/$TRAVIS_COMMIT
- >
if [ "$TRAVIS_OS_NAME" == "osx" ]; then
cp build/dist/*.tar.gz deploy/$TRAVIS_COMMIT &&
find "deploy/$TRAVIS_COMMIT" -maxdepth 1 -type f -exec sh -c 'shasum -a 256 -b "{}" > "{}.sha256"' \;;
else
cp obj/build/dist/*.tar.gz deploy/$TRAVIS_COMMIT &&
find "deploy/$TRAVIS_COMMIT" -maxdepth 1 -type f -exec sh -c 'sha256sum -b "{}" > "{}.sha256"' \;;
fi
deploy:
- provider: s3
bucket: rust-lang-ci
skip_cleanup: true
local_dir: deploy
upload_dir: rustc-builds
acl: public_read
region: us-east-1
access_key_id: AKIAIPQVNYF2T3DTYIWQ
secret_access_key:
secure: "FBqDqOTeIPMu6v/WYPf4CFSlh9rLRZGKVtpLa5KkyuOhXRTrnEzBduEtS8/FMIxdQImvurhSvxWvqRybMOi4qoVfjMqqpHAI7uBbidbrvAcJoHNsx6BgUNVCIoH6a0UsAjTUtm6/YPIpzbHoLZXPL0GrHPMk6Mu04qVSmcYNWn4="
on:
branch: auto
condition: $DEPLOY = 1

38
appveyor.yml
View File

@ -8,10 +8,12 @@ environment:
# 32/64 bit MSVC
- MSYS_BITS: 64
RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc
RUST_CHECK_TARGET: check
SCRIPT: python x.py test && python x.py dist
DEPLOY: 1
- MSYS_BITS: 32
RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
RUST_CHECK_TARGET: check
SCRIPT: python x.py test && python x.py dist
DEPLOY: 1
# MSVC makefiles
- MSYS_BITS: 64
@ -50,10 +52,11 @@ environment:
# too long on appveyor and this is tested by rustbuild below.
- MSYS_BITS: 32
RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
RUST_CHECK_TARGET: check
SCRIPT: python x.py test && python x.py dist
MINGW_URL: https://s3.amazonaws.com/rust-lang-ci
MINGW_ARCHIVE: i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z
MINGW_DIR: mingw32
DEPLOY: 1
- MSYS_BITS: 32
RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --disable-rustbuild
@ -63,11 +66,12 @@ environment:
MINGW_DIR: mingw32
- MSYS_BITS: 64
RUST_CHECK_TARGET: check
SCRIPT: python x.py test && python x.py dist
RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
MINGW_URL: https://s3.amazonaws.com/rust-lang-ci
MINGW_ARCHIVE: x86_64-4.9.2-release-win32-seh-rt_v4-rev4.7z
MINGW_DIR: mingw64
DEPLOY: 1
clone_depth: 1
build: false
@ -123,6 +127,32 @@ branches:
only:
- auto
before_deploy:
- ps: |
New-Item -Path deploy -ItemType directory
Get-ChildItem -Path build\dist -Filter '*.tar.gz' | Move-Item -Destination deploy
Get-FileHash .\deploy\* | ForEach-Object {
[io.file]::WriteAllText($_.Path + ".sha256", $_.Hash.ToLower() + "`n")
}
Get-ChildItem -Path deploy | Foreach-Object {
Push-AppveyorArtifact $_.FullName -FileName ${env:APPVEYOR_REPO_COMMIT}/$_
}
deploy:
- provider: S3
skip_cleanup: true
access_key_id: AKIAIPQVNYF2T3DTYIWQ
secret_access_key:
secure: +11jsUNFTQ9dq5Ad1i2+PeUJaXluFJ0zIJAXESE1dFT3Kdjku4/eDdgyjgsB6GnV
bucket: rust-lang-ci
set_public: true
region: us-east-1
artifact: /.*\.(tar.gz|sha256)/
folder: rustc-builds
on:
branch: auto
DEPLOY: 1
# init:
# - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
# on_finish:

3
src/bootstrap/check.rs
View File

@ -457,6 +457,8 @@ fn krate_android(build: &Build,
let output = output(Command::new("adb").arg("shell").arg(&program));
println!("{}", output);
t!(fs::create_dir_all(build.out.join("tmp")));
build.run(Command::new("adb")
.arg("pull")
.arg(&log)
@ -516,6 +518,7 @@ pub fn android_copy_libs(build: &Build,
}
println!("Android copy libs to emulator ({})", target);
build.run(Command::new("adb").arg("wait-for-device"));
build.run(Command::new("adb").arg("remount"));
build.run(Command::new("adb").args(&["shell", "rm", "-r", ADB_TEST_DIR]));
build.run(Command::new("adb").args(&["shell", "mkdir", ADB_TEST_DIR]));

48
src/bootstrap/compile.rs
View File

@ -16,7 +16,6 @@
//! compiler. This module is also responsible for assembling the sysroot as it
//! goes along from the output of the previous stage.
use std::cmp;
use std::collections::HashMap;
use std::fs::{self, File};
use std::path::{Path, PathBuf};
@ -59,7 +58,7 @@ pub fn std(build: &Build, target: &str, compiler: &Compiler) {
}
build.run(&mut cargo);
update_mtime(&libstd_stamp(build, &compiler, target));
update_mtime(build, &libstd_stamp(build, &compiler, target));
}
/// Link all libstd rlibs/dylibs into the sysroot location.
@ -145,7 +144,7 @@ pub fn test(build: &Build, target: &str, compiler: &Compiler) {
cargo.arg("--manifest-path")
.arg(build.src.join("src/rustc/test_shim/Cargo.toml"));
build.run(&mut cargo);
update_mtime(&libtest_stamp(build, compiler, target));
update_mtime(build, &libtest_stamp(build, compiler, target));
}
/// Same as `std_link`, only for libtest
@ -390,26 +389,39 @@ pub fn tool(build: &Build, stage: u32, host: &str, tool: &str) {
}
/// Updates the mtime of a stamp file if necessary, only changing it if it's
/// older than some other file in the same directory.
/// older than some other library file in the same directory.
///
/// We don't know what file Cargo is going to output (because there's a hash in
/// the file name) but we know where it's going to put it. We use this helper to
/// detect changes to that output file by looking at the modification time for
/// all files in a directory and updating the stamp if any are newer.
fn update_mtime(path: &Path) {
let mut max = None;
if let Ok(entries) = path.parent().unwrap().join("deps").read_dir() {
for entry in entries.map(|e| t!(e)) {
if t!(entry.file_type()).is_file() {
let meta = t!(entry.metadata());
let time = FileTime::from_last_modification_time(&meta);
max = cmp::max(max, Some(time));
}
}
}
///
/// Note that we only consider Rust libraries as that's what we're interested in
/// propagating changes from. Files like executables are tracked elsewhere.
fn update_mtime(build: &Build, path: &Path) {
let entries = match path.parent().unwrap().join("deps").read_dir() {
Ok(entries) => entries,
Err(_) => return,
};
let files = entries.map(|e| t!(e)).filter(|e| t!(e.file_type()).is_file());
let files = files.filter(|e| {
let filename = e.file_name();
let filename = filename.to_str().unwrap();
filename.ends_with(".rlib") ||
filename.ends_with(".lib") ||
is_dylib(&filename)
});
let max = files.max_by_key(|entry| {
let meta = t!(entry.metadata());
FileTime::from_last_modification_time(&meta)
});
let max = match max {
Some(max) => max,
None => return,
};
if !max.is_none() && max <= Some(mtime(path)) {
return
if mtime(&max.path()) > mtime(path) {
build.verbose(&format!("updating {:?} as {:?} changed", path, max.path()));
t!(File::create(path));
}
t!(File::create(path));
}

11
src/bootstrap/dist.rs
View File

@ -357,14 +357,12 @@ pub fn analysis(build: &Build, compiler: &Compiler, target: &str) {
pub fn rust_src(build: &Build) {
println!("Dist src");
let plain_name = format!("rustc-{}-src", package_vers(build));
let name = format!("rust-src-{}", package_vers(build));
let image = tmpdir(build).join(format!("{}-image", name));
let _ = fs::remove_dir_all(&image);
let dst = image.join("lib/rustlib/src");
let dst_src = dst.join("rust");
let plain_dst_src = dst.join(&plain_name);
t!(fs::create_dir_all(&dst_src));
// This is the set of root paths which will become part of the source package
@ -444,7 +442,11 @@ pub fn rust_src(build: &Build) {
build.run(&mut cmd);
// Rename directory, so that root folder of tarball has the correct name
t!(fs::rename(&dst_src, &plain_dst_src));
let plain_name = format!("rustc-{}-src", package_vers(build));
let plain_dst_src = tmpdir(build).join(&plain_name);
let _ = fs::remove_dir_all(&plain_dst_src);
t!(fs::create_dir_all(&plain_dst_src));
cp_r(&dst_src, &plain_dst_src);
// Create the version file
write_file(&plain_dst_src.join("version"), build.version.as_bytes());
@ -453,10 +455,11 @@ pub fn rust_src(build: &Build) {
let mut cmd = Command::new("tar");
cmd.arg("-czf").arg(sanitize_sh(&rust_src_location(build)))
.arg(&plain_name)
.current_dir(&dst);
.current_dir(tmpdir(build));
build.run(&mut cmd);
t!(fs::remove_dir_all(&image));
t!(fs::remove_dir_all(&plain_dst_src));
}
fn install(src: &Path, dstdir: &Path, perms: u32) {

2
src/bootstrap/flags.rs
View File

@ -67,6 +67,7 @@ pub enum Subcommand {
},
Clean,
Dist {
paths: Vec<PathBuf>,
install: bool,
},
}
@ -249,6 +250,7 @@ pub fn parse(args: &[String]) -> Flags {
opts.optflag("", "install", "run installer as well");
m = parse(&opts);
Subcommand::Dist {
paths: remaining_as_path(&m),
install: m.opt_present("install"),
}
}

6
src/bootstrap/step.rs
View File

@ -615,7 +615,7 @@ fn crate_rule<'a, 'b>(build: &'a Build,
.default(true)
.only_host_build(true)
.run(move |s| dist::analysis(build, &s.compiler(), s.target));
rules.dist("install", "src")
rules.dist("install", "path/to/nowhere")
.dep(|s| s.name("default:dist"))
.run(move |s| install::install(build, s.stage, s.target));
@ -932,11 +932,11 @@ fn plan(&self) -> Vec<Step<'a>> {
Subcommand::Doc { ref paths } => (Kind::Doc, &paths[..]),
Subcommand::Test { ref paths, test_args: _ } => (Kind::Test, &paths[..]),
Subcommand::Bench { ref paths, test_args: _ } => (Kind::Bench, &paths[..]),
Subcommand::Dist { install } => {
Subcommand::Dist { ref paths, install } => {
if install {
return vec![self.sbuild.name("install")]
} else {
(Kind::Dist, &[][..])
(Kind::Dist, &paths[..])
}
}
Subcommand::Clean => panic!(),

7
src/ci/docker/arm-android/Dockerfile
View File

@ -48,4 +48,9 @@ ENV RUST_CONFIGURE_ARGS \
--armv7-linux-androideabi-ndk=/android/ndk-arm-9 \
--i686-linux-android-ndk=/android/ndk-x86-9 \
--aarch64-linux-android-ndk=/android/ndk-aarch64
ENV XPY_CHECK test --target arm-linux-androideabi
# Just a smoke test in dist to see if this works for now, we should expand this
# to all the targets above eventually.
ENV SCRIPT \
python2.7 ../x.py test --target arm-linux-androideabi && \
python2.7 ../x.py dist --target arm-linux-androideabi

1
src/ci/docker/arm-android/start-emulator.sh
View File

@ -13,5 +13,4 @@ set -ex
ANDROID_EMULATOR_FORCE_32BIT=true \
nohup nohup emulator @arm-18 -no-window -partition-size 2047 \
0<&- &>/dev/null &
adb wait-for-device
exec "$@"

7
src/ci/docker/cross/Dockerfile
View File

@ -56,7 +56,12 @@ ENV TARGETS=$TARGETS,wasm32-unknown-emscripten
ENV RUST_CONFIGURE_ARGS \
--target=$TARGETS \
--enable-rustbuild
ENV RUST_CHECK_TARGET ""
# Just a smoke test in dist to see if this works for now, we should expand this
# to all the targets above eventually.
ENV SCRIPT \
python2.7 ../x.py build && \
python2.7 ../x.py dist --target wasm32-unknown-emscripten
ENV AR_s390x_unknown_linux_gnu=s390x-linux-gnu-ar \
CC_s390x_unknown_linux_gnu=s390x-linux-gnu-gcc \

8
src/ci/docker/dist-arm-unknown-linux-gnueabi/Dockerfile
View File

@ -24,7 +24,7 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
ENTRYPOINT ["/usr/bin/dumb-init", "--"]
ENV RUST_CONFIGURE_ARGS --host=arm-unknown-linux-gnueabi
ENV XPY_RUN \
dist \
--host arm-unknown-linux-gnueabi \
--target arm-unknown-linux-gnueabi
ENV SCRIPT \
python2.7 ../x.py dist \
--host arm-unknown-linux-gnueabi \
--target arm-unknown-linux-gnueabi

8
src/ci/docker/dist-x86_64-unknown-freebsd/Dockerfile
View File

@ -32,7 +32,7 @@ ENV \
CXX_x86_64_unknown_freebsd=x86_64-unknown-freebsd10-g++
ENV RUST_CONFIGURE_ARGS --host=x86_64-unknown-freebsd
ENV XPY_RUN \
dist \
--host x86_64-unknown-freebsd \
--target x86_64-unknown-freebsd
ENV SCRIPT \
python2.7 ../x.py dist \
--host x86_64-unknown-freebsd \
--target x86_64-unknown-freebsd

2
src/ci/docker/i686-gnu/Dockerfile
View File

@ -23,4 +23,4 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
ENTRYPOINT ["/usr/bin/dumb-init", "--"]
ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu
ENV RUST_CHECK_TARGET check
ENV SCRIPT python2.7 ../x.py test && python2.7 ../x.py dist

2
src/ci/docker/x86_64-gnu-debug/Dockerfile
View File

@ -26,4 +26,4 @@ ENV RUST_CONFIGURE_ARGS \
--build=x86_64-unknown-linux-gnu \
--enable-debug \
--enable-optimize
ENV RUST_CHECK_TARGET ""
ENV SCRIPT python2.7 ../x.py build

2
src/ci/docker/x86_64-gnu-distcheck/Dockerfile
View File

@ -23,4 +23,4 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
ENTRYPOINT ["/usr/bin/dumb-init", "--"]
ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu
ENV XPY_RUN test distcheck
ENV SCRIPT python2.7 ../x.py test distcheck

2
src/ci/docker/x86_64-gnu-full-bootstrap/Dockerfile
View File

@ -25,4 +25,4 @@ ENTRYPOINT ["/usr/bin/dumb-init", "--"]
ENV RUST_CONFIGURE_ARGS \
--build=x86_64-unknown-linux-gnu \
--enable-full-bootstrap
ENV RUST_CHECK_TARGET ""
ENV SCRIPT python2.7 ../x.py build

2
src/ci/docker/x86_64-gnu/Dockerfile
View File

@ -23,4 +23,4 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
ENTRYPOINT ["/usr/bin/dumb-init", "--"]
ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu
ENV RUST_CHECK_TARGET check
ENV SCRIPT python2.7 ../x.py test && python2.7 ../x.py dist

5
src/ci/docker/x86_64-musl/Dockerfile
View File

@ -29,6 +29,7 @@ RUN curl -L https://api.pub.build.mozilla.org/tooltool/sha512/$SCCACHE_DIGEST |
ENV RUST_CONFIGURE_ARGS \
--target=x86_64-unknown-linux-musl \
--musl-root-x86_64=/musl-x86_64
ENV RUST_CHECK_TARGET check-stage2-T-x86_64-unknown-linux-musl-H-x86_64-unknown-linux-gnu
ENV PATH=$PATH:/musl-x86_64/bin
ENV XPY_CHECK test --target x86_64-unknown-linux-musl
ENV SCRIPT \
python2.7 ../x.py test --target x86_64-unknown-linux-musl && \
python2.7 ../x.py dist --target x86_64-unknown-linux-musl

10
src/ci/run.sh
View File

@ -43,14 +43,10 @@ else
ncpus=$(nproc)
fi
if [ ! -z "$XPY_RUN" ]; then
exec python2.7 $SRC/x.py $XPY_RUN
if [ ! -z "$SCRIPT" ]; then
sh -x -c "$SCRIPT"
else
make -j $ncpus tidy
make -j $ncpus
if [ ! -z "$XPY_CHECK" ]; then
exec python2.7 $SRC/x.py $XPY_CHECK
else
exec make $RUST_CHECK_TARGET -j $ncpus
fi
make $RUST_CHECK_TARGET -j $ncpus
fi