From 519f92f2aa7f2ec6833c57460283784a5077f73e Mon Sep 17 00:00:00 2001 From: kennytm Date: Mon, 4 Dec 2017 22:31:57 +0800 Subject: [PATCH] Upload the toolstate to the remote repository. --- .travis.yml | 13 +++ appveyor.yml | 6 +- src/ci/docker/run.sh | 1 + src/ci/docker/x86_64-gnu-tools/Dockerfile | 3 +- src/ci/docker/x86_64-gnu-tools/checktools.sh | 23 ++-- src/ci/docker/x86_64-gnu-tools/publish.py | 105 +++++++++++++++++++ src/ci/docker/x86_64-gnu-tools/repo.sh | 90 ++++++++++++++++ 7 files changed, 230 insertions(+), 11 deletions(-) create mode 100755 src/ci/docker/x86_64-gnu-tools/publish.py create mode 100644 src/ci/docker/x86_64-gnu-tools/repo.sh diff --git a/.travis.yml b/.travis.yml index 4458cd222f2..4a55aab5a41 100644 --- a/.travis.yml +++ b/.travis.yml @@ -179,6 +179,17 @@ matrix: - env: IMAGE=x86_64-gnu-incremental if: branch = auto + - stage: publish toolstate + if: branch = master AND type = push + before_install: [] + install: [] + cache: false + sudo: false + script: + MESSAGE_FILE=$(mktemp -t msg.XXXXXX); + . src/ci/docker/x86_64-gnu-tools/repo.sh; + commit_toolstate_change "$MESSAGE_FILE" "$TRAVIS_BUILD_DIR/src/ci/docker/x86_64-gnu-tools/publish.py" "$(git rev-parse HEAD)" "$(git log --format=%s -n1 HEAD)" "$MESSAGE_FILE" + env: global: - SCCACHE_BUCKET=rust-lang-ci-sccache2 @@ -186,6 +197,8 @@ env: - AWS_ACCESS_KEY_ID=AKIAJAMV3QAMMA6AXHFQ # AWS_SECRET_ACCESS_KEY=... - secure: "j96XxTVOSUf4s4r4htIxn/fvIa5DWbMgLqWl7r8z2QfgUwscmkMXAwXuFNc7s7bGTpV/+CgDiMFFM6BAFLGKutytIF6oA02s9b+usQYnM0th7YQ2AIgm9GtMTJCJp4AoyfFmh8F2faUICBZlfVLUJ34udHEe35vOklix+0k4WDo=" + # TOOLSTATE_REPO_ACCESS_TOKEN=... + - secure: "cFh8thThqEJLC98XKI5pfqflUzOlxsYPRW20AWRaYOOgYHPTiGWypTXiPbGSKaeAXTZoOA+DpQtEmefc0U6lt9dHc7a/MIaK6isFurjlnKYiLOeTruzyu1z7PWCeZ/jKXsU2RK/88DBtlNwfMdaMIeuKj14IVfpepPPL71ETbuk=" before_install: - zcat $HOME/docker/rust-ci.tar.gz | docker load || true diff --git a/appveyor.yml b/appveyor.yml index 94cfa3405ab..da788554049 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -5,6 +5,8 @@ environment: AWS_SECRET_ACCESS_KEY: secure: 7Y+JiquYedOAgnUU26uL0DPzrxmTtR+qIwG6rNKSuWDffqU3vVZxbGXim9QpTO80 SCCACHE_DIGEST: f808afabb4a4eb1d7112bcb3fa6be03b61e93412890c88e177c667eb37f46353d7ec294e559b16f9f4b5e894f2185fe7670a0df15fd064889ecbd80f0c34166c + TOOLSTATE_REPO_ACCESS_TOKEN: + secure: PTZiSxJMVUZ0VnMR5i13E4OagbXfglj7pcskDQiKufVrDm13mLoI0vDJAEM35+bY # By default schannel checks revocation of certificates unlike some other SSL # backends, but we've historically had problems on CI where a revocation @@ -27,8 +29,8 @@ environment: # MSVC tools tests - MSYS_BITS: 64 - SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py toolstates.json - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=toolstates.json --enable-test-miri + SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstates.json windows + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstates.json --enable-test-miri # 32/64-bit MinGW builds. # diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index a863e1a2d5d..f743c976f91 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -99,6 +99,7 @@ exec docker \ --env LOCAL_USER_ID=`id -u` \ --env TRAVIS \ --env TRAVIS_BRANCH \ + --env TOOLSTATE_REPO_ACCESS_TOKEN \ --volume "$HOME/.cargo:/cargo" \ --volume "$HOME/rustsrc:$HOME/rustsrc" \ --init \ diff --git a/src/ci/docker/x86_64-gnu-tools/Dockerfile b/src/ci/docker/x86_64-gnu-tools/Dockerfile index 23b77ab3d65..8975d419d20 100644 --- a/src/ci/docker/x86_64-gnu-tools/Dockerfile +++ b/src/ci/docker/x86_64-gnu-tools/Dockerfile @@ -18,9 +18,10 @@ COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh COPY x86_64-gnu-tools/checktools.sh /tmp/ +COPY x86_64-gnu-tools/repo.sh /tmp/ ENV RUST_CONFIGURE_ARGS \ --build=x86_64-unknown-linux-gnu \ --enable-test-miri \ --save-toolstates=/tmp/toolstates.json -ENV SCRIPT /tmp/checktools.sh ../x.py /tmp/toolstates.json +ENV SCRIPT /tmp/checktools.sh ../x.py /tmp/toolstates.json linux diff --git a/src/ci/docker/x86_64-gnu-tools/checktools.sh b/src/ci/docker/x86_64-gnu-tools/checktools.sh index bf39bc28a67..e06b6ab3d1b 100755 --- a/src/ci/docker/x86_64-gnu-tools/checktools.sh +++ b/src/ci/docker/x86_64-gnu-tools/checktools.sh @@ -13,7 +13,9 @@ set -eu X_PY="$1" -TOOLSTATE_FILE="$2" +TOOLSTATE_FILE="$(realpath $2)" +OS="$3" +COMMIT="$(git rev-parse HEAD)" touch "$TOOLSTATE_FILE" @@ -23,17 +25,22 @@ python2.7 "$X_PY" test --no-fail-fast \ src/tools/rustfmt \ src/tools/miri \ src/tools/clippy -TEST_RESULT=$? set -e -# FIXME: Upload this file to the repository. cat "$TOOLSTATE_FILE" -# FIXME: After we can properly inform dev-tool maintainers about failure, -# comment out the `exit 0` below. if [ "$RUST_RELEASE_CHANNEL" = nightly ]; then - # exit 0 - true + . "$(dirname $0)/repo.sh" + MESSAGE_FILE=$(mktemp -t msg.XXXXXX) + echo "($OS CI update)" > "$MESSAGE_FILE" + commit_toolstate_change "$MESSAGE_FILE" \ + sed -i "1 a\\ +$COMMIT\t$(cat "$TOOLSTATE_FILE") +" "history/$OS.tsv" + rm -f "$MESSAGE_FILE" + exit 0 fi -exit $TEST_RESULT +if grep -q 'Broken\|Compiling' "$TOOLSTATE_FILE"; then + exit 4 +fi diff --git a/src/ci/docker/x86_64-gnu-tools/publish.py b/src/ci/docker/x86_64-gnu-tools/publish.py new file mode 100755 index 00000000000..b90947e5a43 --- /dev/null +++ b/src/ci/docker/x86_64-gnu-tools/publish.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2017 The Rust Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution and at +# http://rust-lang.org/COPYRIGHT. +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import sys +import re +import json +import copy +import datetime +import collections + +# List of people to ping when the status of a tool changed. +MAINTAINERS = { + 'miri': '@oli-obk @RalfJung @eddyb', + 'clippy-driver': '@Manishearth @llogiq @mcarton @oli-obk', + 'rls': '@nrc', + 'rustfmt': '@nrc', +} + + +def read_current_status(current_commit, path): + '''Reads build status of `current_commit` from content of `history/*.tsv` + ''' + with open(path, 'rU') as f: + for line in f: + (commit, status) = line.split('\t', 1) + if commit == current_commit: + return json.loads(status) + return {} + + +def update_latest(current_commit, relevant_pr_number, current_datetime): + '''Updates `_data/latest.json` to match build result of the given commit. + ''' + with open('_data/latest.json', 'rb+') as f: + latest = json.load(f, object_pairs_hook=collections.OrderedDict) + + current_status = { + os: read_current_status(current_commit, 'history/' + os + '.tsv') + for os in ['windows', 'linux'] + } + + slug = 'rust-lang/rust' + message = '📣 Toolstate changed by {}!\n\nTested on commit {}@{}.\n\n' \ + .format(relevant_pr_number, slug, current_commit) + anything_changed = False + for status in latest: + tool = status['tool'] + changed = False + + for os, s in current_status.items(): + old = status[os] + new = s.get(tool, old) + status[os] = new + if new > old: + changed = True + message += '🎉 {} on {}: {} → {}.\n' \ + .format(tool, os, old, new) + elif new < old: + changed = True + message += '💔 {} on {}: {} → {} (cc {}).\n' \ + .format(tool, os, old, new, MAINTAINERS[tool]) + + if changed: + status['commit'] = current_commit + status['datetime'] = current_datetime + anything_changed = True + + if not anything_changed: + return '' + + f.seek(0) + f.truncate(0) + json.dump(latest, f, indent=4, separators=(',', ': ')) + return message + + +if __name__ == '__main__': + cur_commit = sys.argv[1] + cur_datetime = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') + cur_commit_msg = sys.argv[2] + save_message_to_path = sys.argv[3] + + relevant_pr_match = re.search('#[0-9]+', cur_commit_msg) + if relevant_pr_match: + relevant_pr_number = 'rust-lang/rust' + relevant_pr_match.group(0) + else: + relevant_pr_number = '' + + message = update_latest(cur_commit, relevant_pr_number, cur_datetime) + if message: + print(message) + with open(save_message_to_path, 'w') as f: + f.write(message) + else: + print('') diff --git a/src/ci/docker/x86_64-gnu-tools/repo.sh b/src/ci/docker/x86_64-gnu-tools/repo.sh new file mode 100644 index 00000000000..c10afef753e --- /dev/null +++ b/src/ci/docker/x86_64-gnu-tools/repo.sh @@ -0,0 +1,90 @@ +#!/bin/sh + +# Copyright 2017 The Rust Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution and at +# http://rust-lang.org/COPYRIGHT. +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +# This file provides the function `commit_toolstate_change` for pushing a change +# to the `rust-toolstate` repository. +# +# The function relies on a GitHub bot user, which should have a Personal access +# token defined in the environment variable $TOOLSTATE_REPO_ACCESS_TOKEN. If for +# some reason you need to change the token, please update `.travis.yml` and +# `appveyor.yml`: +# +# 1. Generate a new Personal access token: +# +# * Login to the bot account, and go to Settings -> Developer settings -> +# Personal access tokens +# * Click "Generate new token" +# * Enable the "public_repo" permission, then click "Generate token" +# * Copy the generated token (should be a 40-digit hexadecimal number). +# Save it somewhere secure, as the token would be gone once you leave +# the page. +# +# 2. Encrypt the token for Travis CI +# +# * Install the `travis` tool locally (`gem install travis`). +# * Encrypt the token: +# ``` +# travis -r rust-lang/rust encrypt \ +# TOOLSTATE_REPO_ACCESS_TOKEN=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +# ``` +# * Copy output to replace the existing one in `.travis.yml`. +# * Details of this step can be found in +# +# +# 3. Encrypt the token for AppVeyor +# +# * Login to AppVeyor using your main account, and login as the rust-lang +# organization. +# * Open the ["Encrypt data" tool](https://ci.appveyor.com/tools/encrypt) +# * Paste the 40-digit token into the "Value to encrypt" box, then click +# "Encrypt" +# * Copy the output to replace the existing one in `appveyor.yml`. +# * Details of this step can be found in +# +# +# 4. Replace the email address below if the bot account identity is changed +# +# * See +# if a private email by GitHub is wanted. + +commit_toolstate_change() { + OLDFLAGS="$-" + set -eu + + git config --global user.email '34210020+rust-toolstate-update@users.noreply.github.com' + git config --global user.name 'Rust Toolstate Update' + git config --global credential.helper store + printf 'https://%s:x-oauth-basic@github.com\n' "$TOOLSTATE_REPO_ACCESS_TOKEN" \ + > "$HOME/.git-credentials" + git clone --depth=1 https://github.com/rust-lang-nursery/rust-toolstate.git + + cd rust-toolstate + FAILURE=1 + MESSAGE_FILE="$1" + shift + for RETRY_COUNT in 1 2 3 4 5; do + "$@" + # `git commit` failing means nothing to commit. + FAILURE=0 + git commit -a -F "$MESSAGE_FILE" || break + # On failure randomly sleep for 0 to 3 seconds as a crude way to introduce jittering. + git push origin master && break || sleep $(LC_ALL=C tr -cd 0-3 < /dev/urandom | head -c 1) + FAILURE=1 + git fetch origin master + git reset --hard origin/master + done + cd .. + + set +eu + set "-$OLDFLAGS" + return $FAILURE +}