Auto merge of #54638 - christianpoveda:master, r=kennytm

Add checking for tool distribution in Tier 1

This fixes https://github.com/rust-lang/rust/issues/54483
r? @kennytm

.travis.yml

@@ -94,6 +94,7 @@ matrix:
         MACOSX_DEPLOYMENT_TARGET=10.7
         NO_LLVM_ASSERTIONS=1
         NO_DEBUG_ASSERTIONS=1
+        DIST_REQUIRE_ALL_TOOLS=1
         CI_JOB_NAME=dist-i686-apple
       os: osx
       osx_image: xcode9.3-moar
@@ -108,6 +109,7 @@ matrix:
         MACOSX_DEPLOYMENT_TARGET=10.7
         NO_LLVM_ASSERTIONS=1
         NO_DEBUG_ASSERTIONS=1
+        DIST_REQUIRE_ALL_TOOLS=1
         CI_JOB_NAME=dist-x86_64-apple
       os: osx
       osx_image: xcode9.3-moar

appveyor.yml

@@ -85,6 +85,7 @@ environment:
       --enable-full-tools
       --enable-profiler
     SCRIPT: python x.py dist
+    DIST_REQUIRE_ALL_TOOLS: 1
     DEPLOY: 1
     CI_JOB_NAME: dist-x86_64-msvc
   - RUST_CONFIGURE_ARGS: >
@@ -93,6 +94,7 @@ environment:
       --enable-full-tools
       --enable-profiler
     SCRIPT: python x.py dist
+    DIST_REQUIRE_ALL_TOOLS: 1
     DEPLOY: 1
     CI_JOB_NAME: dist-i686-msvc
   - MSYS_BITS: 32
@@ -101,6 +103,7 @@ environment:
     MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror
     MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z
     MINGW_DIR: mingw32
+    DIST_REQUIRE_ALL_TOOLS: 1
     DEPLOY: 1
     CI_JOB_NAME: dist-i686-mingw
   - MSYS_BITS: 64
@@ -109,6 +112,7 @@ environment:
     MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror
     MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z
     MINGW_DIR: mingw64
+    DIST_REQUIRE_ALL_TOOLS: 1
     DEPLOY: 1
     CI_JOB_NAME: dist-x86_64-mingw
 

config.toml.example

@@ -481,3 +481,7 @@
 # as the one built on Windows will contain backslashes in paths causing problems
 # on linux
 #src-tarball = true
+#
+
+# Whether to allow failures when building tools
+#missing-tools = false

src/bootstrap/config.rs

@@ -134,6 +134,7 @@ pub struct Config {
     pub test_miri: bool,
     pub save_toolstates: Option<PathBuf>,
     pub print_step_timings: bool,
+    pub missing_tools: bool,
 
     // Fallback musl-root for all targets
     pub musl_root: Option<PathBuf>,
@@ -271,6 +272,7 @@ struct Dist {
     gpg_password_file: Option<String>,
     upload_addr: Option<String>,
     src_tarball: Option<bool>,
+    missing_tools: Option<bool>,
 }
 
 #[derive(Deserialize)]
@@ -375,6 +377,7 @@ impl Config {
         config.rust_codegen_backends = vec![INTERNER.intern_str("llvm")];
         config.rust_codegen_backends_dir = "codegen-backends".to_owned();
         config.deny_warnings = true;
+        config.missing_tools = false;
 
         // set by bootstrap.py
         config.build = INTERNER.intern_str(&env::var("BUILD").expect("'BUILD' to be set"));
@@ -613,6 +616,7 @@ impl Config {
             config.dist_gpg_password_file = t.gpg_password_file.clone().map(PathBuf::from);
             config.dist_upload_addr = t.upload_addr.clone();
             set(&mut config.rust_dist_src, t.src_tarball);
+            set(&mut config.missing_tools, t.missing_tools);
         }
 
         // Now that we've reached the end of our configuration, infer the

src/bootstrap/configure.py

@@ -69,6 +69,7 @@ o("profiler", "build.profiler", "build the profiler runtime")
 o("emscripten", None, "compile the emscripten backend as well as LLVM")
 o("full-tools", None, "enable all tools")
 o("lldb", "rust.lldb", "build lldb")
+o("missing-tools", "dist.missing-tools", "allow failures when building tools")
 
 # Optimization and debugging options. These may be overridden by the release
 # channel, etc.

src/bootstrap/dist.rs

@@ -67,6 +67,14 @@ fn rust_installer(builder: &Builder) -> Command {
     builder.tool_cmd(Tool::RustInstaller)
 }
 
+fn missing_tool(tool_name: &str, skip: bool) {
+    if skip {
+        println!("Unable to build {}, skipping dist", tool_name)
+    } else {
+        panic!("Unable to build {}", tool_name)
+    }
+}
+
 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 pub struct Docs {
     pub stage: u32,
@@ -1166,7 +1174,7 @@ impl Step for Rls {
         let rls = builder.ensure(tool::Rls {
             compiler: builder.compiler(stage, builder.config.build),
             target, extra_features: Vec::new()
-        }).or_else(|| { println!("Unable to build RLS, skipping dist"); None })?;
+        }).or_else(|| { missing_tool("RLS", builder.build.config.missing_tools); None })?;
 
         builder.install(&rls, &image.join("bin"), 0o755);
         let doc = image.join("share/doc/rls");
@@ -1245,11 +1253,11 @@ impl Step for Clippy {
         let clippy = builder.ensure(tool::Clippy {
             compiler: builder.compiler(stage, builder.config.build),
             target, extra_features: Vec::new()
-        }).or_else(|| { println!("Unable to build clippy, skipping dist"); None })?;
+        }).or_else(|| { missing_tool("clippy", builder.build.config.missing_tools); None })?;
         let cargoclippy = builder.ensure(tool::CargoClippy {
             compiler: builder.compiler(stage, builder.config.build),
             target, extra_features: Vec::new()
-        }).or_else(|| { println!("Unable to build cargo clippy, skipping dist"); None })?;
+        }).or_else(|| { missing_tool("cargo clippy", builder.build.config.missing_tools); None })?;
 
         builder.install(&clippy, &image.join("bin"), 0o755);
         builder.install(&cargoclippy, &image.join("bin"), 0o755);
@@ -1324,11 +1332,11 @@ impl Step for Rustfmt {
         let rustfmt = builder.ensure(tool::Rustfmt {
             compiler: builder.compiler(stage, builder.config.build),
             target, extra_features: Vec::new()
-        }).or_else(|| { println!("Unable to build Rustfmt, skipping dist"); None })?;
+        }).or_else(|| { missing_tool("Rustfmt", builder.build.config.missing_tools); None })?;
         let cargofmt = builder.ensure(tool::Cargofmt {
             compiler: builder.compiler(stage, builder.config.build),
             target, extra_features: Vec::new()
-        }).or_else(|| { println!("Unable to build Cargofmt, skipping dist"); None })?;
+        }).or_else(|| { missing_tool("Cargofmt", builder.build.config.missing_tools); None })?;
 
         builder.install(&rustfmt, &image.join("bin"), 0o755);
         builder.install(&cargofmt, &image.join("bin"), 0o755);

src/ci/docker/dist-i686-linux/Dockerfile

@@ -110,3 +110,5 @@ ENV CFLAGS -mstackrealign
 # When we build cargo in this container, we don't want it to use the system
 # libcurl, instead it should compile its own.
 ENV LIBCURL_NO_PKG_CONFIG 1
+
+ENV DIST_REQUIRE_ALL_TOOLS 1

src/ci/docker/dist-x86_64-linux/Dockerfile

@@ -106,3 +106,5 @@ ENV DIST_SRC 1
 # When we build cargo in this container, we don't want it to use the system
 # libcurl, instead it should compile its own.
 ENV LIBCURL_NO_PKG_CONFIG 1
+
+ENV DIST_REQUIRE_ALL_TOOLS 1

src/ci/run.sh

@@ -76,6 +76,10 @@ else
   fi
 fi
 
+if [ "$RUST_RELEASE_CHANNEL" = "nightly" ] || [ "$DIST_REQUIRE_ALL_TOOLS" = "" ]; then
+    RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-missing-tools"
+fi
+
 # We've had problems in the past of shell scripts leaking fds into the sccache
 # server (#48192) which causes Cargo to erroneously think that a build script
 # hasn't finished yet. Try to solve that problem by starting a very long-lived