Auto merge of #81489 - nikic:x86-64-dist-update, r=Mark-Simulacrum

Update Python and Clang on x86 dist images

LLVM 12 no longer builds with Python 2, so install Python 3 in
preparation for the upgrade (#81451).

However, Clang 10 does not build with Python 3, so we need update
to Clang 11 as well, which supports both.

Unfortunately, doing so results in errors while linking the
libLLVM.so into other binaries:
> __morestack: invalid needed version 2

This is fixed by using LLD instead. Possibly this is due to a binutils
linker bug, but updating to the latest binutils version does not fix
it.

r? `@Mark-Simulacrum`
cc `@cuviper`

src/bootstrap/native.rs

@@ -201,7 +201,7 @@ fn run(self, builder: &Builder<'_>) -> PathBuf {
         if builder.config.llvm_thin_lto {
             cfg.define("LLVM_ENABLE_LTO", "Thin");
             if !target.contains("apple") {
-                cfg.define("LLVM_ENABLE_LLD", "ON");
+                cfg.define("LLVM_USE_LINKER", "lld");
             }
         }
 
@@ -556,6 +556,9 @@ fn run(self, builder: &Builder<'_>) -> PathBuf {
         t!(fs::create_dir_all(&out_dir));
 
         let mut cfg = cmake::Config::new(builder.src.join("src/llvm-project/lld"));
+        if let Some(ref linker) = builder.config.llvm_use_linker {
+            cfg.define("LLVM_USE_LINKER", linker);
+        }
         configure_cmake(builder, target, &mut cfg, true);
 
         // This is an awful, awful hack. Discovered when we migrated to using

src/ci/docker/host-x86_64/dist-i686-linux/Dockerfile

@@ -68,11 +68,11 @@ RUN ./build-binutils.sh
 COPY host-x86_64/dist-x86_64-linux/build-gcc.sh /tmp/
 RUN ./build-gcc.sh && apt-get remove -y gcc g++
 
-# Debian 6 has Python 2.6 by default, but LLVM needs 2.7+
+# Debian 6 has Python 2.6 by default, but LLVM >= 12 needs Python 3
 COPY host-x86_64/dist-x86_64-linux/build-python.sh /tmp/
 RUN ./build-python.sh
 
-# LLVM needs cmake 3.4.3 or higher, and is planning to raise to 3.13.4.
+# LLVM needs cmake 3.13.4 or higher
 COPY host-x86_64/dist-x86_64-linux/build-cmake.sh /tmp/
 RUN ./build-cmake.sh
 
@@ -94,8 +94,10 @@ ENV RUST_CONFIGURE_ARGS \
       --set target.i686-unknown-linux-gnu.linker=clang \
       --build=i686-unknown-linux-gnu \
       --set llvm.ninja=false \
+      --set llvm.use-linker=lld \
+      --set rust.use-lld=true \
       --set rust.jemalloc
-ENV SCRIPT python2.7 ../x.py dist --build $HOSTS --host $HOSTS --target $HOSTS
+ENV SCRIPT python3 ../x.py dist --build $HOSTS --host $HOSTS --target $HOSTS
 ENV CARGO_TARGET_I686_UNKNOWN_LINUX_GNU_LINKER=clang
 
 # This was added when we switched from gcc to clang. It's not clear why this is

src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile

@@ -68,11 +68,11 @@ RUN ./build-binutils.sh
 COPY host-x86_64/dist-x86_64-linux/build-gcc.sh /tmp/
 RUN ./build-gcc.sh && apt-get remove -y gcc g++
 
-# Debian 6 has Python 2.6 by default, but LLVM needs 2.7+
+# Debian 6 has Python 2.6 by default, but LLVM >= 12 needs Python 3
 COPY host-x86_64/dist-x86_64-linux/build-python.sh /tmp/
 RUN ./build-python.sh
 
-# LLVM needs cmake 3.4.3 or higher, and is planning to raise to 3.13.4.
+# LLVM needs cmake 3.13.4 or higher
 COPY host-x86_64/dist-x86_64-linux/build-cmake.sh /tmp/
 RUN ./build-cmake.sh
 
@@ -99,8 +99,10 @@ ENV RUST_CONFIGURE_ARGS \
       --set target.x86_64-unknown-linux-gnu.ranlib=/rustroot/bin/llvm-ranlib \
       --set llvm.thin-lto=true \
       --set llvm.ninja=false \
+      --set llvm.use-linker=lld \
+      --set rust.use-lld=true \
       --set rust.jemalloc
-ENV SCRIPT ../src/ci/pgo.sh python2.7 ../x.py dist \
+ENV SCRIPT ../src/ci/pgo.sh python3 ../x.py dist \
     --host $HOSTS --target $HOSTS \
     --include-default-paths \
     src/tools/build-manifest

src/ci/docker/host-x86_64/dist-x86_64-linux/build-clang.sh

@@ -4,7 +4,7 @@ set -ex
 
 source shared.sh
 
-LLVM=llvmorg-10.0.0
+LLVM=llvmorg-11.0.1
 
 mkdir llvm-project
 cd llvm-project

src/ci/docker/host-x86_64/dist-x86_64-linux/build-python.sh

@@ -3,7 +3,7 @@
 set -ex
 source shared.sh
 
-curl https://www.python.org/ftp/python/2.7.12/Python-2.7.12.tgz | \
+curl https://www.python.org/ftp/python/3.9.1/Python-3.9.1.tgz | \
   tar xzf -
 
 mkdir python-build
@@ -12,10 +12,10 @@ cd python-build
 # Gotta do some hackery to tell python about our custom OpenSSL build, but other
 # than that fairly normal.
 CFLAGS='-I /rustroot/include' LDFLAGS='-L /rustroot/lib -L /rustroot/lib64' \
-    hide_output ../Python-2.7.12/configure --prefix=/rustroot
+    hide_output ../Python-3.9.1/configure --prefix=/rustroot
 hide_output make -j10
 hide_output make install
 
 cd ..
 rm -rf python-build
-rm -rf Python-2.7.12
+rm -rf Python-3.9.1

src/ci/pgo.sh

@@ -4,7 +4,7 @@ set -euxo pipefail
 
 rm -rf /tmp/rustc-pgo
 
-python2.7 ../x.py build --target=$PGO_HOST --host=$PGO_HOST \
+python3 ../x.py build --target=$PGO_HOST --host=$PGO_HOST \
     --stage 2 library/std --rust-profile-generate=/tmp/rustc-pgo
 
 ./build/$PGO_HOST/stage2/bin/rustc --edition=2018 \