Retry when downloading the Docker cache.

Prevent spuriously needing to rebuild the docker image when the network
was down.

Also, adjusted the retry function to insert a sleep between retries,
because retrying immediately will often just hit the same issue.

diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh
index c470ae7eb30306cdaa081c06e3c5d79e4d3b90c3..3465e386cd925c90c188a7cd27cd009c368c8beb 100755 (executable)
--- a/src/ci/docker/run.sh
+++ b/src/ci/docker/run.sh
@@ -36,8 +36,10 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then
       s3url="s3://$SCCACHE_BUCKET/docker/$cksum"
       url="https://s3-us-west-1.amazonaws.com/$SCCACHE_BUCKET/docker/$cksum"
       echo "Attempting to download $s3url"
+      rm -f /tmp/rustci_docker_cache
       set +e
-      loaded_images=$(curl $url | docker load | sed 's/.* sha/sha/')
+      retry curl -f -L -C - -o /tmp/rustci_docker_cache "$url"
+      loaded_images=$(docker load -i /tmp/rustci_docker_cache | sed 's/.* sha/sha/')
       set -e
       echo "Downloaded containers:\n$loaded_images"
     fi

diff --git a/src/ci/shared.sh b/src/ci/shared.sh
index 4a08683e3ee86511f505d16a88f00b39d3a2fe5d..bb6945f0fd6bbb5b542778b466c532e8d3682c23 100644 (file)
--- a/src/ci/shared.sh
+++ b/src/ci/shared.sh
@@ -21,11 +21,12 @@ function retry {
   while true; do
     "$@" && break || {
       if [[ $n -lt $max ]]; then
+        sleep $n  # don't retry immediately
         ((n++))
         echo "Command failed. Attempt $n/$max:"
       else
         echo "The command has failed after $n attempts."
-        exit 1
+        return 1
       fi
     }
   done