5 export MSYS_NO_PATHCONV=1
7 script=`cd $(dirname $0) && pwd`/`basename $0`
21 echo "expected single argument for the image name"
30 script_dir="`dirname $script`"
31 docker_dir="${script_dir}/host-$(uname -m)"
32 ci_dir="`dirname $script_dir`"
33 src_dir="`dirname $ci_dir`"
34 root_dir="`dirname $src_dir`"
37 dist=$objdir/build/dist
39 source "$ci_dir/shared.sh"
41 CACHE_DOMAIN="${CACHE_DOMAIN:-ci-caches.rust-lang.org}"
43 if [ -f "$docker_dir/$image/Dockerfile" ]; then
44 if [ "$CI" != "" ]; then
45 hash_key=/tmp/.docker-hash-key.txt
47 echo $image >> $hash_key
49 cat "$docker_dir/$image/Dockerfile" >> $hash_key
50 # Look for all source files involves in the COPY command
51 copied_files=/tmp/.docker-copied-files.txt
53 for i in $(sed -n -e '/^COPY --from=/! s/^COPY \(.*\) .*$/\1/p' \
54 "$docker_dir/$image/Dockerfile"); do
56 find "$script_dir/$i" -type f >> $copied_files
58 # Sort the file names and cat the content into the hash key
59 sort $copied_files | xargs cat >> $hash_key
61 # Include the architecture in the hash key, since our Linux CI does not
62 # only run in x86_64 machines.
65 docker --version >> $hash_key
66 cksum=$(sha512sum $hash_key | \
69 url="https://$CACHE_DOMAIN/docker/$cksum"
71 echo "Attempting to download $url"
72 rm -f /tmp/rustci_docker_cache
74 retry curl --max-time 600 -y 30 -Y 10 --connect-timeout 30 -f -L -C - \
75 -o /tmp/rustci_docker_cache "$url"
76 echo "Loading images into docker"
77 # docker load sometimes hangs in the CI, so time out after 10 minutes with TERM,
78 # KILL after 12 minutes
79 loaded_images=$(/usr/bin/timeout -k 720 600 docker load -i /tmp/rustci_docker_cache \
80 | sed 's/.* sha/sha/')
82 echo "Downloaded containers:\n$loaded_images"
85 dockerfile="$docker_dir/$image/Dockerfile"
86 if [ -x /usr/bin/cygpath ]; then
87 context="`cygpath -w $script_dir`"
88 dockerfile="`cygpath -w $dockerfile`"
99 if [ "$CI" != "" ]; then
100 s3url="s3://$SCCACHE_BUCKET/docker/$cksum"
101 upload="aws s3 cp - $s3url"
102 digest=$(docker inspect rust-ci --format '{{.Id}}')
103 echo "Built container $digest"
104 if ! grep -q "$digest" <(echo "$loaded_images"); then
105 echo "Uploading finished image to $url"
107 docker history -q rust-ci | \
109 xargs docker save | \
114 echo "Looks like docker image is the same as before, not uploading"
116 # Record the container image for reuse, e.g. by rustup.rs builds
117 info="$dist/image-$image.txt"
120 echo "$digest" >>"$info"
122 elif [ -f "$docker_dir/disabled/$image/Dockerfile" ]; then
124 echo Cannot run disabled images on CI!
127 # Transform changes the context of disabled Dockerfiles to match the enabled ones
128 tar --transform 's#disabled/#./#' -C $script_dir -c . | docker \
132 -f "host-$(uname -m)/$image/Dockerfile" \
135 echo Invalid image: $image
137 # Check whether the image exists for other architectures
138 for arch_dir in "${script_dir}"/host-*; do
139 # Avoid checking non-directories and the current host architecture directory
140 if ! [[ -d "${arch_dir}" ]]; then
143 if [[ "${arch_dir}" = "${docker_dir}" ]]; then
147 arch_name="$(basename "${arch_dir}" | sed 's/^host-//')"
148 if [[ -f "${arch_dir}/${image}/Dockerfile" ]]; then
149 echo "Note: the image exists for the ${arch_name} host architecture"
150 elif [[ -f "${arch_dir}/disabled/${image}/Dockerfile" ]]; then
151 echo "Note: the disabled image exists for the ${arch_name} host architecture"
155 echo "Note: the current host architecture is $(uname -m)"
161 mkdir -p $HOME/.cargo
163 mkdir -p $objdir/cores
164 mkdir -p /tmp/toolstate
167 if [ "$SCCACHE_BUCKET" != "" ]; then
168 args="$args --env SCCACHE_BUCKET"
169 args="$args --env SCCACHE_REGION"
170 args="$args --env AWS_ACCESS_KEY_ID"
171 args="$args --env AWS_SECRET_ACCESS_KEY"
173 mkdir -p $HOME/.cache/sccache
174 args="$args --env SCCACHE_DIR=/sccache --volume $HOME/.cache/sccache:/sccache"
177 # Run containers as privileged as it should give them access to some more
178 # syscalls such as ptrace and whatnot. In the upgrade to LLVM 5.0 it was
179 # discovered that the leak sanitizer apparently needs these syscalls nowadays so
180 # we'll need `--privileged` for at least the `x86_64-gnu` builder, so this just
181 # goes ahead and sets it for all builders.
182 args="$args --privileged"
184 # Things get a little weird if this script is already running in a docker
185 # container. If we're already in a docker container then we assume it's set up
186 # to do docker-in-docker where we have access to a working `docker` command.
188 # If this is the case (we check via the presence of `/.dockerenv`)
189 # then we can't actually use the `--volume` argument. Typically we use
190 # `--volume` to efficiently share the build and source directory between this
191 # script and the container we're about to spawn. If we're inside docker already
192 # though the `--volume` argument maps the *host's* folder to the container we're
193 # about to spawn, when in fact we want the folder in this container itself. To
194 # work around this we use a recipe cribbed from
195 # https://circleci.com/docs/2.0/building-docker-images/#mounting-folders to
196 # create a temporary container with a volume. We then copy the entire source
197 # directory into this container, and then use that copy in the container we're
198 # about to spawn. Finally after the build finishes we re-extract the object
201 # Note that none of this is necessary if we're *not* in a docker-in-docker
202 # scenario. If this script is run on a bare metal host then we share a bunch of
203 # data directories to share as much data as possible. Note that we also use
204 # `LOCAL_USER_ID` (recognized in `src/ci/run.sh`) to ensure that files are all
205 # read/written as the same user as the bare-metal user.
206 if [ -f /.dockerenv ]; then
207 docker create -v /checkout --name checkout alpine:3.4 /bin/true
208 docker cp . checkout:/checkout
209 args="$args --volumes-from checkout"
211 args="$args --volume $root_dir:/checkout:ro"
212 args="$args --volume $objdir:/checkout/obj"
213 args="$args --volume $HOME/.cargo:/cargo"
214 args="$args --volume $HOME/rustsrc:$HOME/rustsrc"
215 args="$args --volume /tmp/toolstate:/tmp/toolstate"
216 args="$args --env LOCAL_USER_ID=`id -u`"
225 command="/checkout/src/ci/run.sh"
228 if [ "$CI" != "" ]; then
229 # Get some needed information for $BASE_COMMIT
231 # This command gets the last merge commit which we'll use as base to list
232 # deleted files since then.
233 BASE_COMMIT="$(git log --author=bors@rust-lang.org -n 2 --pretty=format:%H | tail -n 1)"
240 --workdir /checkout/obj \
241 --env SRC=/checkout \
243 --env CARGO_HOME=/cargo \
248 --env BUILD_SOURCEBRANCHNAME \
249 --env GITHUB_ACTIONS \
251 --env TOOLSTATE_REPO_ACCESS_TOKEN \
252 --env TOOLSTATE_REPO \
253 --env TOOLSTATE_PUBLISH \
254 --env RUST_CI_OVERRIDE_RELEASE_CHANNEL \
255 --env CI_JOB_NAME="${CI_JOB_NAME-$IMAGE}" \
256 --env BASE_COMMIT="$BASE_COMMIT" \
262 if [ -f /.dockerenv ]; then
264 docker cp checkout:/checkout/obj $objdir