Rollup merge of #63485 - pietroalbini:new-mirror-bucket, r=alexcrichton

ci: move mirrors to their standalone bucket

Currently mirrors are stored in the rust-lang-ci2 S3 bucket along with
CI toolchains. This is problematic for multiple reasons:

- CI IAM credentials are allowed to both edit and delete those files.
  A malicious user gaining access to those credentials would be able to
  change our mirrored dependencies, possibly compromising the compiler.

- Contents of the rust-lang-ci2 bucket are disposable except for the
  mirrors' content. When we implement backups for S3 buckets we'd have
  to replicate just that part of the bucket, complicating the backup
  logic and increasing the chance of mistakes. A standalone bucket will
  be way easier to backup.

This commit switches our CI to use the new rust-lang-ci-mirrors bucket.

r? @alexcrichton

diff --git a/src/ci/azure-pipelines/auto.yml b/src/ci/azure-pipelines/auto.yml
index 687856cca6b62cc2b860afb59964d8c0d8d95e00..06fa3bd9f434710290d0a75848d17060f8e7af6b 100644 (file)
--- a/src/ci/azure-pipelines/auto.yml
+++ b/src/ci/azure-pipelines/auto.yml
@@ -273,7 +273,7 @@ jobs:
         MSYS_BITS: 32
         RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
         SCRIPT: make ci-subset-1
-        MINGW_URL: https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror
+        MINGW_URL: https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc
         MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z
         MINGW_DIR: mingw32
         # FIXME(#59637)
@@ -283,14 +283,14 @@ jobs:
         MSYS_BITS: 32
         RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
         SCRIPT: make ci-subset-2
-        MINGW_URL: https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror
+        MINGW_URL: https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc
         MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z
         MINGW_DIR: mingw32
       x86_64-mingw-1:
         MSYS_BITS: 64
         SCRIPT: make ci-subset-1
         RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
-        MINGW_URL: https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror
+        MINGW_URL: https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc
         MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z
         MINGW_DIR: mingw64
         # FIXME(#59637)
@@ -300,7 +300,7 @@ jobs:
         MSYS_BITS: 64
         SCRIPT: make ci-subset-2
         RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
-        MINGW_URL: https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror
+        MINGW_URL: https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc
         MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z
         MINGW_DIR: mingw64
 
@@ -327,7 +327,7 @@ jobs:
         MSYS_BITS: 32
         RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools --enable-profiler
         SCRIPT: python x.py dist
-        MINGW_URL: https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror
+        MINGW_URL: https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc
         MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z
         MINGW_DIR: mingw32
         DIST_REQUIRE_ALL_TOOLS: 1
@@ -336,7 +336,7 @@ jobs:
         MSYS_BITS: 64
         SCRIPT: python x.py dist
         RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler
-        MINGW_URL: https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror
+        MINGW_URL: https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc
         MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z
         MINGW_DIR: mingw64
         DIST_REQUIRE_ALL_TOOLS: 1

diff --git a/src/ci/azure-pipelines/steps/install-clang.yml b/src/ci/azure-pipelines/steps/install-clang.yml
index 45ec767e0b875469c800631dea493be81760689e..14daf81b430755cda020c104ec9da8a8cc138f2c 100644 (file)
--- a/src/ci/azure-pipelines/steps/install-clang.yml
+++ b/src/ci/azure-pipelines/steps/install-clang.yml
@@ -36,7 +36,7 @@ steps:
     set -e
     mkdir -p citools
     cd citools
-    curl -f https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/LLVM-7.0.0-win64.tar.gz | tar xzf -
+    curl -f https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/LLVM-7.0.0-win64.tar.gz | tar xzf -
     echo "##vso[task.setvariable variable=RUST_CONFIGURE_ARGS]$RUST_CONFIGURE_ARGS --set llvm.clang-cl=`pwd`/clang-rust/bin/clang-cl.exe"
   condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['MINGW_URL'],''))
   displayName: Install clang (Windows)

diff --git a/src/ci/azure-pipelines/steps/install-sccache.yml b/src/ci/azure-pipelines/steps/install-sccache.yml
index 427e50f571f764197cfca484c0e93fd6da670808..d4679c1c6733e50807df5f8c61dec675a15e031c 100644 (file)
--- a/src/ci/azure-pipelines/steps/install-sccache.yml
+++ b/src/ci/azure-pipelines/steps/install-sccache.yml
@@ -2,14 +2,14 @@ steps:
 
 - bash: |
     set -e
-    curl -fo /usr/local/bin/sccache https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/2018-04-02-sccache-x86_64-apple-darwin
+    curl -fo /usr/local/bin/sccache https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2018-04-02-sccache-x86_64-apple-darwin
     chmod +x /usr/local/bin/sccache
   displayName: Install sccache (OSX)
   condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin'))
 
 - script: |
     md sccache
-    powershell -Command "$ProgressPreference = 'SilentlyContinue'; iwr -outf sccache\sccache.exe https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/2018-04-26-sccache-x86_64-pc-windows-msvc"
+    powershell -Command "$ProgressPreference = 'SilentlyContinue'; iwr -outf sccache\sccache.exe https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2018-04-26-sccache-x86_64-pc-windows-msvc"
     echo ##vso[task.prependpath]%CD%\sccache
   displayName: Install sccache (Windows)
   condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))

diff --git a/src/ci/azure-pipelines/steps/install-windows-build-deps.yml b/src/ci/azure-pipelines/steps/install-windows-build-deps.yml
index c42c2311b493ff05cea5a751206fb36bea676fc5..9aaeb4b79d63410ff36f91d64185ca4f6c6e5c75 100644 (file)
--- a/src/ci/azure-pipelines/steps/install-windows-build-deps.yml
+++ b/src/ci/azure-pipelines/steps/install-windows-build-deps.yml
@@ -4,7 +4,7 @@ steps:
 # https://github.com/wixtoolset/wix3 originally
 - bash: |
     set -e
-    curl -O https://rust-lang-ci2.s3-us-west-1.amazonaws.com/rust-ci-mirror/wix311-binaries.zip
+    curl -O https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/wix311-binaries.zip
     echo "##vso[task.setvariable variable=WIX]`pwd`/wix"
     mkdir -p wix/bin
     cd wix/bin
@@ -18,7 +18,7 @@ steps:
 # one is MSI installers and one is EXE, but they're not used so frequently at
 # this point anyway so perhaps it's a wash!
 - script: |
-    powershell -Command "$ProgressPreference = 'SilentlyContinue'; iwr -outf is-install.exe https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/2017-08-22-is.exe"
+    powershell -Command "$ProgressPreference = 'SilentlyContinue'; iwr -outf is-install.exe https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2017-08-22-is.exe"
     is-install.exe /VERYSILENT /SUPPRESSMSGBOXES /NORESTART /SP-
     echo ##vso[task.prependpath]C:\Program Files (x86)\Inno Setup 5
   displayName: Install InnoSetup
@@ -109,7 +109,7 @@ steps:
 # Note that this is originally from the github releases patch of Ninja
 - script: |
     md ninja
-    powershell -Command "$ProgressPreference = 'SilentlyContinue'; iwr -outf 2017-03-15-ninja-win.zip https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/2017-03-15-ninja-win.zip"
+    powershell -Command "$ProgressPreference = 'SilentlyContinue'; iwr -outf 2017-03-15-ninja-win.zip https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2017-03-15-ninja-win.zip"
     7z x -oninja 2017-03-15-ninja-win.zip
     del 2017-03-15-ninja-win.zip
     set RUST_CONFIGURE_ARGS=%RUST_CONFIGURE_ARGS% --enable-ninja

diff --git a/src/ci/docker/armhf-gnu/Dockerfile b/src/ci/docker/armhf-gnu/Dockerfile
index 235920833f83980cf89acdbe1e6fcc1969cfebd0..9493b33698708b08449d55d4b30ee3b54151c355 100644 (file)
--- a/src/ci/docker/armhf-gnu/Dockerfile
+++ b/src/ci/docker/armhf-gnu/Dockerfile
@@ -72,7 +72,7 @@ RUN arm-linux-gnueabihf-gcc addentropy.c -o rootfs/addentropy -static
 
 # TODO: What is this?!
 # Source of the file: https://github.com/vfdev-5/qemu-rpi2-vexpress/raw/master/vexpress-v2p-ca15-tc1.dtb
-RUN curl -O https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/vexpress-v2p-ca15-tc1.dtb
+RUN curl -O https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/vexpress-v2p-ca15-tc1.dtb
 
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh

diff --git a/src/ci/docker/dist-various-1/install-mips-musl.sh b/src/ci/docker/dist-various-1/install-mips-musl.sh
index 60a96e3b8e9524e5d951249ae58649ae4b23cea6..29cfb5d96083ef6eabbe39e400d2a939bc60590c 100755 (executable)
--- a/src/ci/docker/dist-various-1/install-mips-musl.sh
+++ b/src/ci/docker/dist-various-1/install-mips-musl.sh
@@ -5,7 +5,7 @@ mkdir /usr/local/mips-linux-musl
 # originally from
 # https://downloads.openwrt.org/snapshots/trunk/ar71xx/generic/
 # OpenWrt-Toolchain-ar71xx-generic_gcc-5.3.0_musl-1.1.16.Linux-x86_64.tar.bz2
-URL="https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror"
+URL="https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc"
 FILE="OpenWrt-Toolchain-ar71xx-generic_gcc-5.3.0_musl-1.1.16.Linux-x86_64.tar.bz2"
 curl -L "$URL/$FILE" | tar xjf - -C /usr/local/mips-linux-musl --strip-components=2
 

diff --git a/src/ci/docker/dist-various-2/build-wasi-toolchain.sh b/src/ci/docker/dist-various-2/build-wasi-toolchain.sh
index 7bf8946c4f136a4d3b7b09206bd3b3404701232c..f04ee78157167104e53c37390677ae6dfa0af131 100755 (executable)
--- a/src/ci/docker/dist-various-2/build-wasi-toolchain.sh
+++ b/src/ci/docker/dist-various-2/build-wasi-toolchain.sh
@@ -5,7 +5,7 @@
 set -ex
 
 # Originally from https://releases.llvm.org/8.0.0/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz
-curl https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/clang%2Bllvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz | \
+curl https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/clang%2Bllvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz | \
   tar xJf -
 export PATH=`pwd`/clang+llvm-8.0.0-x86_64-linux-gnu-ubuntu-14.04/bin:$PATH
 

diff --git a/src/ci/docker/dist-x86_64-linux/build-openssl.sh b/src/ci/docker/dist-x86_64-linux/build-openssl.sh
index 13dae6169053a09a3eedd892180c11da48d8aba5..be8a6c93945e96f7d8c7323b4a917df15e3c8910 100755 (executable)
--- a/src/ci/docker/dist-x86_64-linux/build-openssl.sh
+++ b/src/ci/docker/dist-x86_64-linux/build-openssl.sh
@@ -4,7 +4,7 @@ set -ex
 source shared.sh
 
 VERSION=1.0.2k
-URL=https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/openssl-$VERSION.tar.gz
+URL=https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/openssl-$VERSION.tar.gz
 
 curl $URL | tar xzf -
 

diff --git a/src/ci/docker/dist-x86_64-netbsd/build-netbsd-toolchain.sh b/src/ci/docker/dist-x86_64-netbsd/build-netbsd-toolchain.sh
index 2e9b9dcc2344ed2d1c38f30c53bf18f5e348381e..797f674b954f2e54c272cb6b4b2a9bb3f512cc25 100755 (executable)
--- a/src/ci/docker/dist-x86_64-netbsd/build-netbsd-toolchain.sh
+++ b/src/ci/docker/dist-x86_64-netbsd/build-netbsd-toolchain.sh
@@ -25,7 +25,7 @@ cd netbsd
 
 mkdir -p /x-tools/x86_64-unknown-netbsd/sysroot
 
-URL=https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror
+URL=https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc
 
 # Originally from ftp://ftp.netbsd.org/pub/NetBSD/NetBSD-$BSD/source/sets/*.tgz
 curl $URL/2018-03-01-netbsd-src.tgz | tar xzf -

diff --git a/src/ci/docker/scripts/android-sdk-manager.py b/src/ci/docker/scripts/android-sdk-manager.py
index 7c9a8b82e928266aa914a07093c8352ca4e3698d..c9e2961f6eb156ca8de88dd6ee51aff71bc8ff8f 100755 (executable)
--- a/src/ci/docker/scripts/android-sdk-manager.py
+++ b/src/ci/docker/scripts/android-sdk-manager.py
@@ -23,8 +23,9 @@ REPOSITORIES = [
 HOST_OS = "linux"
 
 # Mirroring options
-MIRROR_BUCKET = "rust-lang-ci2"
-MIRROR_BASE_DIR = "rust-ci-mirror/android/"
+MIRROR_BUCKET = "rust-lang-ci-mirrors"
+MIRROR_BUCKET_REGION = "us-west-1"
+MIRROR_BASE_DIR = "rustc/android/"
 
 import argparse
 import hashlib
@@ -144,7 +145,8 @@ def cli_install(args):
     lockfile = Lockfile(args.lockfile)
     for package in lockfile.packages.values():
         # Download the file from the mirror into a temp file
-        url = "https://" + MIRROR_BUCKET + ".s3.amazonaws.com/" + MIRROR_BASE_DIR
+        url = "https://" + MIRROR_BUCKET + ".s3-" + MIRROR_BUCKET_REGION + \
+              ".amazonaws.com/" + MIRROR_BASE_DIR
         downloaded = package.download(url)
         # Extract the file in a temporary directory
         extract_dir = tempfile.mkdtemp()

diff --git a/src/ci/docker/scripts/freebsd-toolchain.sh b/src/ci/docker/scripts/freebsd-toolchain.sh
index 8cef69d9c26bb5b34c4f726d541ffe5985cb9a5f..70155e770a960281eb90668a7485b8a249889938 100755 (executable)
--- a/src/ci/docker/scripts/freebsd-toolchain.sh
+++ b/src/ci/docker/scripts/freebsd-toolchain.sh
@@ -59,7 +59,7 @@ done
 
 # Originally downloaded from:
 # https://download.freebsd.org/ftp/releases/${freebsd_arch}/${freebsd_version}-RELEASE/base.txz
-URL=https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/2019-04-04-freebsd-${freebsd_arch}-${freebsd_version}-RELEASE-base.txz
+URL=https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2019-04-04-freebsd-${freebsd_arch}-${freebsd_version}-RELEASE-base.txz
 curl "$URL" | tar xJf - -C "$sysroot" --wildcards "${files_to_extract[@]}"
 
 # Fix up absolute symlinks from the system image.  This can be removed

diff --git a/src/ci/docker/scripts/sccache.sh b/src/ci/docker/scripts/sccache.sh
index 194de3c339f8c4ef22ef574214237e1013ac1a2b..efeb0ed0d72d0f9461f17a4f701039e2f4dee1d7 100644 (file)
--- a/src/ci/docker/scripts/sccache.sh
+++ b/src/ci/docker/scripts/sccache.sh
@@ -1,6 +1,6 @@
 set -ex
 
 curl -fo /usr/local/bin/sccache \
-  https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/2018-04-02-sccache-x86_64-unknown-linux-musl
+  https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2018-04-02-sccache-x86_64-unknown-linux-musl
 
 chmod +x /usr/local/bin/sccache

diff --git a/src/ci/install-awscli.sh b/src/ci/install-awscli.sh
index d491b9fbcdcf81a6ae67ac5496fe05b5f8635fb2..69c8d2e3099ab9bdf70e8dc8894495fe011b6d68 100755 (executable)
--- a/src/ci/install-awscli.sh
+++ b/src/ci/install-awscli.sh
@@ -16,7 +16,7 @@
 set -euo pipefail
 IFS=$'\n\t'
 
-MIRROR="https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/2019-07-27-awscli.tar"
+MIRROR="https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2019-07-27-awscli.tar"
 DEPS_DIR="/tmp/awscli-deps"
 
 pip="pip"