Enable downloading prebuilt LLVM in test builders

See comment added for details on the test builder restriction. This is primarily
intended for macOS CI, but is likely to be a slight win on other builders too.

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7eb41fec36aba9615db2496eb479ad96106abbd0..11b115cb2ff245bbd11209790c5e4f91b0ca8621 100644 (file)
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -391,24 +391,28 @@ jobs:
             env:
               RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu --set llvm.allow-old-toolchain"
               SCRIPT: make ci-mingw-subset-1
+              NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
             os: windows-latest-xl
           - name: i686-mingw-2
             env:
               RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu --set llvm.allow-old-toolchain"
               SCRIPT: make ci-mingw-subset-2
+              NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
             os: windows-latest-xl
           - name: x86_64-mingw-1
             env:
               SCRIPT: make ci-mingw-subset-1
               RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-profiler --set llvm.allow-old-toolchain"
+              NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
             os: windows-latest-xl
           - name: x86_64-mingw-2
             env:
               SCRIPT: make ci-mingw-subset-2
               RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-profiler --set llvm.allow-old-toolchain"
+              NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
             os: windows-latest-xl
           - name: dist-x86_64-msvc
@@ -433,6 +437,7 @@ jobs:
           - name: dist-i686-mingw
             env:
               RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu --enable-full-tools --enable-profiler --set llvm.allow-old-toolchain"
+              NO_DOWNLOAD_CI_LLVM: 1
               SCRIPT: python x.py dist
               CUSTOM_MINGW: 1
               DIST_REQUIRE_ALL_TOOLS: 1
@@ -441,6 +446,7 @@ jobs:
             env:
               SCRIPT: python x.py dist
               RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler --set llvm.allow-old-toolchain"
+              NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
               DIST_REQUIRE_ALL_TOOLS: 1
             os: windows-latest-xl

diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs
index 56a0c1b2d9158d6f83f1ce2b63b0f495eaba6e95..13e3049f2c81ff001a9bfbbce5efb075c676772c 100644 (file)
--- a/src/bootstrap/config.rs
+++ b/src/bootstrap/config.rs
@@ -990,7 +990,7 @@ pub fn parse(args: &[String]) -> Config {
             config.llvm_from_ci = match llvm.download_ci_llvm {
                 Some(StringOrBool::String(s)) => {
                     assert!(s == "if-available", "unknown option `{}` for download-ci-llvm", s);
-                    crate::native::is_ci_llvm_available(&config)
+                    crate::native::is_ci_llvm_available(&config, llvm_assertions.unwrap_or(false))
                 }
                 Some(StringOrBool::Bool(b)) => b,
                 None => false,

diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs
index 6a33b264015f3f0c1969c5f9620bc03c8c684b70..7ecf74d3068d0a54e1b1278449af6bf33ceddd2b 100644 (file)
--- a/src/bootstrap/native.rs
+++ b/src/bootstrap/native.rs
@@ -148,7 +148,7 @@ pub(crate) fn detect_llvm_sha(config: &crate::config::Config) -> String {
 /// This checks both the build triple platform to confirm we're usable at all,
 /// and then verifies if the current HEAD matches the detected LLVM SHA head,
 /// in which case LLVM is indicated as not available.
-pub(crate) fn is_ci_llvm_available(config: &crate::config::Config) -> bool {
+pub(crate) fn is_ci_llvm_available(config: &crate::config::Config, asserts: bool) -> bool {
     // This is currently all tier 1 targets and tier 2 targets with host tools
     // (since others may not have CI artifacts)
     // https://doc.rust-lang.org/rustc/platform-support.html#tier-1
@@ -187,6 +187,12 @@ pub(crate) fn is_ci_llvm_available(config: &crate::config::Config) -> bool {
         return false;
     }
 
+    let triple = &*config.build.triple;
+    if (triple == "aarch64-unknown-linux-gnu" || triple.contains("i686")) && asserts {
+        // No alt builder for aarch64-unknown-linux-gnu today.
+        return false;
+    }
+
     if crate::util::CiEnv::is_ci() {
         let llvm_sha = detect_llvm_sha(config);
         let head_sha = output(config.git().arg("rev-parse").arg("HEAD"));

diff --git a/src/ci/docker/host-x86_64/arm-android/Dockerfile b/src/ci/docker/host-x86_64/arm-android/Dockerfile
index a2335687ab33568266850d9437942656b0ae0f3d..d0d367b39b493850cfce53e8095f2f05f7127905 100644 (file)
--- a/src/ci/docker/host-x86_64/arm-android/Dockerfile
+++ b/src/ci/docker/host-x86_64/arm-android/Dockerfile
@@ -29,6 +29,10 @@ ENV PATH=$PATH:/android/sdk/platform-tools
 
 ENV TARGETS=arm-linux-androideabi
 
+# We are intentionally allowing an old toolchain on this builder (and that's
+# incompatible with LLVM downloads today).
+ENV NO_DOWNLOAD_CI_LLVM 1
+
 ENV RUST_CONFIGURE_ARGS --arm-linux-androideabi-ndk=/android/ndk/arm-14 \
     --set llvm.allow-old-toolchain
 

diff --git a/src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile b/src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile
index 1f8d0a64ea48e0ecc380ea6411c21eea80ed36b2..71ea13071bd1f1345d0841f3208fa92f131b77a5 100644 (file)
--- a/src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile
+++ b/src/ci/docker/host-x86_64/i686-gnu-nopt/Dockerfile
@@ -27,6 +27,10 @@ RUN mkdir -p /config
 RUN echo "[rust]" > /config/nopt-std-config.toml
 RUN echo "optimize = false" >> /config/nopt-std-config.toml
 
+# We are intentionally allowing an old toolchain on this builder (and that's
+# incompatible with LLVM downloads today).
+ENV NO_DOWNLOAD_CI_LLVM 1
+
 ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu --disable-optimize-tests \
     --set llvm.allow-old-toolchain
 ENV SCRIPT python3 ../x.py test --stage 0 --config /config/nopt-std-config.toml library/std \

diff --git a/src/ci/docker/host-x86_64/i686-gnu/Dockerfile b/src/ci/docker/host-x86_64/i686-gnu/Dockerfile
index 7bca0398dea7c6679bcf8e6286b2b5c116392340..ebeab3dbdb79913c87a55877c553e80e15dca62a 100644 (file)
--- a/src/ci/docker/host-x86_64/i686-gnu/Dockerfile
+++ b/src/ci/docker/host-x86_64/i686-gnu/Dockerfile
@@ -23,6 +23,9 @@ RUN sh /scripts/sccache.sh
 COPY scripts/cmake.sh /scripts/
 RUN /scripts/cmake.sh
 
+# We are intentionally allowing an old toolchain on this builder (and that's
+# incompatible with LLVM downloads today).
+ENV NO_DOWNLOAD_CI_LLVM 1
 ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu \
     --set llvm.allow-old-toolchain
 # Exclude some tests that are unlikely to be platform specific, to speed up

diff --git a/src/ci/docker/host-x86_64/test-various/Dockerfile b/src/ci/docker/host-x86_64/test-various/Dockerfile
index 4d554a2852a0910c490a623b7afdda4a8cf381b0..b75e2f085cd3b73f2f66ea7323bea08fc8edb6eb 100644 (file)
--- a/src/ci/docker/host-x86_64/test-various/Dockerfile
+++ b/src/ci/docker/host-x86_64/test-various/Dockerfile
@@ -30,6 +30,10 @@ WORKDIR /
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 
+# We are disabling CI LLVM since this builder needs to build LLD, which is
+# currently unsupported when downloading pre-built LLVM.
+ENV NO_DOWNLOAD_CI_LLVM 1
+
 ENV RUST_CONFIGURE_ARGS \
   --musl-root-x86_64=/usr/local/x86_64-linux-musl \
   --set build.nodejs=/node-v15.14.0-linux-x64/bin/node \

diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile
index bfc6975c19dc9b174e0b6deaf3ecb4beeb4ec474..321b3d6ace486d84f5cc9bbe0cd6df052a96e7e0 100644 (file)
--- a/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile
@@ -26,6 +26,10 @@ RUN sh /scripts/sccache.sh
 COPY scripts/cmake.sh /scripts/
 RUN /scripts/cmake.sh
 
+# We are intentionally allowing an old toolchain on this builder (and that's
+# incompatible with LLVM downloads today).
+ENV NO_DOWNLOAD_CI_LLVM 1
+
 ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu \
     --set llvm.allow-old-toolchain
 ENV RUST_CHECK_TARGET check-aux

diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile
index 13d440423b20b4ba244b2b7cffe321707f28ccf6..739045248fea205d7c0ef0ec80fffc6f71fc6af3 100644 (file)
--- a/src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile
@@ -31,6 +31,9 @@ RUN sh /scripts/sccache.sh
 ENV RUSTBUILD_FORCE_CLANG_BASED_TESTS 1
 ENV RUN_CHECK_WITH_PARALLEL_QUERIES 1
 
+# llvm.use-linker conflicts with downloading CI LLVM
+ENV NO_DOWNLOAD_CI_LLVM 1
+
 ENV RUST_CONFIGURE_ARGS \
       --build=x86_64-unknown-linux-gnu \
       --enable-debug \

diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile
index 4bb4042cd7e2ffa75a0986c2a7dd3a12dc560c35..be4def85f14c4735c41b63da42f00a317273d438 100644 (file)
--- a/src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-distcheck/Dockerfile
@@ -22,6 +22,10 @@ RUN sh /scripts/sccache.sh
 COPY scripts/cmake.sh /scripts/
 RUN /scripts/cmake.sh
 
+# We are intentionally allowing an old toolchain on this builder (and that's
+# incompatible with LLVM downloads today).
+ENV NO_DOWNLOAD_CI_LLVM 1
+
 ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu --set rust.ignore-git=false \
     --set llvm.allow-old-toolchain
 ENV SCRIPT python3 ../x.py --stage 2 test distcheck

diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile
index 21891c1244d7c9a75d2300c93756a32df7bf9d14..23f2215c2d93cfbe85e3e06028f00793c96af05d 100644 (file)
--- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13-stage1/Dockerfile
@@ -26,7 +26,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 
-# using llvm-link-shared due to libffi issues -- see #34486
+# We are disabling CI LLVM since this builder is intentionally using a host
+# LLVM, rather than the typical src/llvm-project LLVM.
+ENV NO_DOWNLOAD_CI_LLVM 1
+
+# Using llvm-link-shared due to libffi issues -- see #34486
 ENV RUST_CONFIGURE_ARGS \
     --build=x86_64-unknown-linux-gnu \
     --llvm-root=/usr/lib/llvm-13 \

diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13/Dockerfile
index 5f3af2f47caec7205dcf777a14bba9dcc566a2e4..8f6831bc54e63e6b693c031ab6d7cda8f701d7e8 100644 (file)
--- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13/Dockerfile
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-13/Dockerfile
@@ -37,7 +37,11 @@ RUN apt-get update && \
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 
-# using llvm-link-shared due to libffi issues -- see #34486
+# We are disabling CI LLVM since this builder is intentionally using a host
+# LLVM, rather than the typical src/llvm-project LLVM.
+ENV NO_DOWNLOAD_CI_LLVM 1
+
+# Using llvm-link-shared due to libffi issues -- see #34486
 ENV RUST_CONFIGURE_ARGS \
       --build=x86_64-unknown-linux-gnu \
       --llvm-root=/usr/lib/llvm-13 \

diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile
index f442a477a7b1e121a00f8da94ae63b40677cd90d..e2ec4f275001b53d24fececc91607eaa810e1a75 100644 (file)
--- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile
@@ -80,6 +80,10 @@ COPY host-x86_64/x86_64-gnu-tools/browser-ui-test.version /tmp/
 # the local version of the package is different than the one used by the CI.
 RUN npm install -g browser-ui-test@$(head -n 1 /tmp/browser-ui-test.version) --unsafe-perm=true
 
+# We are intentionally allowing an old toolchain on this builder (and that's
+# incompatible with LLVM downloads today).
+ENV NO_DOWNLOAD_CI_LLVM 1
+
 ENV RUST_CONFIGURE_ARGS \
   --set llvm.allow-old-toolchain \
   --build=x86_64-unknown-linux-gnu \

diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml
index 91204eb0b05725e7d89f8a74f220dd086c8f98b3..a6bdd9f3186b1fc8e3f9eea3a21945e48381d9ee 100644 (file)
--- a/src/ci/github-actions/ci.yml
+++ b/src/ci/github-actions/ci.yml
@@ -600,6 +600,9 @@ jobs:
                 --build=i686-pc-windows-gnu
                 --set llvm.allow-old-toolchain
               SCRIPT: make ci-mingw-subset-1
+              # We are intentionally allowing an old toolchain on this builder (and that's
+              # incompatible with LLVM downloads today).
+              NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
             <<: *job-windows-xl
 
@@ -609,6 +612,9 @@ jobs:
                 --build=i686-pc-windows-gnu
                 --set llvm.allow-old-toolchain
               SCRIPT: make ci-mingw-subset-2
+              # We are intentionally allowing an old toolchain on this builder (and that's
+              # incompatible with LLVM downloads today).
+              NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
             <<: *job-windows-xl
 
@@ -619,6 +625,9 @@ jobs:
                 --build=x86_64-pc-windows-gnu
                 --enable-profiler
                 --set llvm.allow-old-toolchain
+              # We are intentionally allowing an old toolchain on this builder (and that's
+              # incompatible with LLVM downloads today).
+              NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
             <<: *job-windows-xl
 
@@ -629,6 +638,9 @@ jobs:
                 --build=x86_64-pc-windows-gnu
                 --enable-profiler
                 --set llvm.allow-old-toolchain
+              # We are intentionally allowing an old toolchain on this builder (and that's
+              # incompatible with LLVM downloads today).
+              NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
             <<: *job-windows-xl
 
@@ -678,6 +690,9 @@ jobs:
                 --enable-full-tools
                 --enable-profiler
                 --set llvm.allow-old-toolchain
+              # We are intentionally allowing an old toolchain on this builder (and that's
+              # incompatible with LLVM downloads today).
+              NO_DOWNLOAD_CI_LLVM: 1
               SCRIPT: python x.py dist
               CUSTOM_MINGW: 1
               DIST_REQUIRE_ALL_TOOLS: 1
@@ -691,6 +706,9 @@ jobs:
                 --enable-full-tools
                 --enable-profiler
                 --set llvm.allow-old-toolchain
+              # We are intentionally allowing an old toolchain on this builder (and that's
+              # incompatible with LLVM downloads today).
+              NO_DOWNLOAD_CI_LLVM: 1
               CUSTOM_MINGW: 1
               DIST_REQUIRE_ALL_TOOLS: 1
             <<: *job-windows-xl

diff --git a/src/ci/run.sh b/src/ci/run.sh
index 6545475d996419ef4d607aa44e7453546cdada85..091ccf58e2b81cdebd075dac134fda52357e88c7 100755 (executable)
--- a/src/ci/run.sh
+++ b/src/ci/run.sh
@@ -102,6 +102,18 @@ else
   fi
 
   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir"
+
+  # We enable this for non-dist builders, since those aren't trying to produce
+  # fresh binaries. We currently don't entirely support distributing a fresh
+  # copy of the compiler (including llvm tools, etc.) if we haven't actually
+  # built LLVM, since not everything necessary is copied into the
+  # local-usage-only LLVM artifacts. If that changes, this could maybe be made
+  # true for all builds. In practice it's probably a good idea to keep building
+  # LLVM continuously on at least some builders to ensure it works, though.
+  # (And PGO is its own can of worms).
+  if [ "$NO_DOWNLOAD_CI_LLVM" = "" ]; then
+    RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set llvm.download-ci-llvm=if-available"
+  fi
 fi
 
 if [ "$RUST_RELEASE_CHANNEL" = "nightly" ] || [ "$DIST_REQUIRE_ALL_TOOLS" = "" ]; then

diff --git a/src/ci/scripts/install-clang.sh b/src/ci/scripts/install-clang.sh
index 0bc8a0389a8d4c0e6fc153fdbd26423d2b5f759c..02b72625d6eb578bb1c889646ea65dbd2f754a89 100755 (executable)
--- a/src/ci/scripts/install-clang.sh
+++ b/src/ci/scripts/install-clang.sh
@@ -61,6 +61,10 @@ elif isWindows && [[ ${CUSTOM_MINGW-0} -ne 1 ]]; then
     7z x -oclang-rust/ "LLVM-${LLVM_VERSION}-win64.exe"
     ciCommandSetEnv RUST_CONFIGURE_ARGS \
         "${RUST_CONFIGURE_ARGS} --set llvm.clang-cl=$(pwd)/clang-rust/bin/clang-cl.exe"
+
+    # Disable downloading CI LLVM on this builder;
+    # setting up clang-cl just above conflicts with the default if-available option.
+    ciCommandSetEnv NO_DOWNLOAD_CI_LLVM 1
 fi
 
 if isWindows; then