src/ci/github-actions/ci.yml

   1 ######################################################
   2 #   WARNING! Action needed when changing this file   #
   3 ######################################################
   4
   5 # Due to GitHub Actions limitations, we can't use YAML Anchors directly in the
   6 # CI configuration stored on the repository. To work around that this file is
   7 # expanded by a tool in the repository, and the expansion is committed as well.
   8 #
   9 # After you make any change to the file you'll need to run this command:
  10 #
  11 #   ./x.py run src/tools/expand-yaml-anchors
  12 #
  13 # ...and commit the file it updated in addition to this one. If you forget this
  14 # step CI will fail.
  15
  16 ---
  17
  18 ###############################
  19 #   YAML Anchors Definition   #
  20 ###############################
  21
  22 # This key contains most of the YAML anchors that will be used later in the
  23 # document. YAML anchors allows us to greatly reduce duplication inside the CI
  24 # configuration by reusing parts of the configuration.
  25 #
  26 # YAML anchors work by defining an anchor with `&anchor-name` and reusing its
  27 # content in another place with `*anchor-name`. The special `<<` map key merges
  28 # the content of the map with the content of the anchor (or list of anchors).
  29 #
  30 # The expand-yaml-anchors tool will automatically remove this block from the
  31 # output YAML file.
  32 x--expand-yaml-anchors--remove:
  33
  34   - &shared-ci-variables
  35     CI_JOB_NAME: ${{ matrix.name }}
  36
  37   - &public-variables
  38     SCCACHE_BUCKET: rust-lang-gha-caches
  39     TOOLSTATE_REPO: https://github.com/pietroalbini/rust-toolstate
  40     CACHE_DOMAIN: ci-caches-gha.rust-lang.org
  41
  42   - &prod-variables
  43     SCCACHE_BUCKET: rust-lang-gha-caches
  44     DEPLOY_BUCKET: rust-lang-gha
  45     TOOLSTATE_REPO: https://github.com/pietroalbini/rust-toolstate
  46     TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/pietroalbini/rust-toolstate/issues
  47     TOOLSTATE_PUBLISH: 1
  48     # AWS_SECRET_ACCESS_KEYs are stored in GitHub's secrets storage, named
  49     # AWS_SECRET_ACCESS_KEY_<keyid>. Including the key id in the name allows to
  50     # rotate them in a single branch while keeping the old key in another
  51     # branch, which wouldn't be possible if the key was named with the kind
  52     # (caches, artifacts...).
  53     CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZOMUQATD5
  54     ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZH5AYXDVF
  55     CACHE_DOMAIN: ci-caches-gha.rust-lang.org
  56
  57   - &base-job
  58     env: {}
  59
  60   - &job-linux-xl
  61     os: ubuntu-latest-xl
  62     <<: *base-job
  63
  64   - &job-macos-xl
  65     os: macos-latest  # We don't have an XL builder for this
  66     <<: *base-job
  67
  68   - &job-windows-xl
  69     os: windows-latest-xl
  70     <<: *base-job
  71
  72   - &step
  73     if: success() && !env.SKIP_JOB
  74
  75   - &base-ci-job
  76     timeout-minutes: 600
  77     runs-on: "${{ matrix.os }}"
  78     env: *shared-ci-variables
  79     steps:
  80       - name: disable git crlf conversion
  81         run: git config --global core.autocrlf false
  82         shell: bash
  83
  84       - name: checkout the source code
  85         uses: actions/checkout@v1
  86         with:
  87           fetch-depth: 2
  88
  89       - name: configure GitHub Actions to kill the build when outdated
  90         uses: rust-lang/simpleinfra/github-actions/cancel-outdated-builds@master
  91         with:
  92           github_token: "${{ secrets.github_token }}"
  93         if: success() && !env.SKIP_JOB && github.ref != 'refs/heads/try'
  94         <<: *step
  95
  96       - name: add extra environment variables
  97         run: src/ci/scripts/setup-environment.sh
  98         env:
  99           # Since it's not possible to merge `${{ matrix.env }}` with the other
 100           # variables in `job.<name>.env`, the variables defined in the matrix
 101           # are passed to the `setup-environment.sh` script encoded in JSON,
 102           # which then uses log commands to actually set them.
 103           EXTRA_VARIABLES: ${{ toJson(matrix.env) }}
 104         <<: *step
 105
 106       - name: decide whether to skip this job
 107         run: src/ci/scripts/should-skip-this.sh
 108         <<: *step
 109
 110       - name: collect CPU statistics
 111         run: src/ci/scripts/collect-cpu-stats.sh
 112         <<: *step
 113
 114       - name: show the current environment
 115         run: src/ci/scripts/dump-environment.sh
 116         <<: *step
 117
 118       - name: install awscli
 119         run: src/ci/scripts/install-awscli.sh
 120         <<: *step
 121
 122       - name: install sccache
 123         run: src/ci/scripts/install-sccache.sh
 124         <<: *step
 125
 126       - name: install clang
 127         run: src/ci/scripts/install-clang.sh
 128         <<: *step
 129
 130       - name: install WIX
 131         run: src/ci/scripts/install-wix.sh
 132         <<: *step
 133
 134       - name: install InnoSetup
 135         run: src/ci/scripts/install-innosetup.sh
 136         <<: *step
 137
 138       - name: ensure the build happens on a partition with enough space
 139         run: src/ci/scripts/symlink-build-dir.sh
 140         <<: *step
 141
 142       - name: disable git crlf conversion
 143         run: src/ci/scripts/disable-git-crlf-conversion.sh
 144         <<: *step
 145
 146       - name: install MSYS2
 147         run: src/ci/scripts/install-msys2.sh
 148         <<: *step
 149
 150       - name: install MinGW
 151         run: src/ci/scripts/install-mingw.sh
 152         <<: *step
 153
 154       - name: install ninja
 155         run: src/ci/scripts/install-ninja.sh
 156         <<: *step
 157
 158       - name: enable ipv6 on Docker
 159         run: src/ci/scripts/enable-docker-ipv6.sh
 160         <<: *step
 161
 162       # Disable automatic line ending conversion (again). On Windows, when we're
 163       # installing dependencies, something switches the git configuration directory or
 164       # re-enables autocrlf. We've not tracked down the exact cause -- and there may
 165       # be multiple -- but this should ensure submodules are checked out with the
 166       # appropriate line endings.
 167       - name: disable git crlf conversion
 168         run: src/ci/scripts/disable-git-crlf-conversion.sh
 169         <<: *step
 170
 171       - name: checkout submodules
 172         run: src/ci/scripts/checkout-submodules.sh
 173         <<: *step
 174
 175       - name: ensure line endings are correct
 176         run: src/ci/scripts/verify-line-endings.sh
 177         <<: *step
 178
 179       - name: run the build
 180         run: src/ci/scripts/run-build-from-ci.sh
 181         env:
 182           AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }}
 183           AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}
 184           TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
 185         <<: *step
 186
 187       - name: upload artifacts to S3
 188         run: src/ci/scripts/upload-artifacts.sh
 189         env:
 190           AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}
 191           AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}
 192         # Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy
 193         # builders *should* have the AWS credentials available. Still, explicitly
 194         # adding the condition is helpful as this way CI will not silently skip
 195         # deploying artifacts from a dist builder if the variables are misconfigured,
 196         # erroring about invalid credentials instead.
 197         if: success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')
 198         <<: *step
 199
 200   # These snippets are used by the try-success, try-failure, auto-success and auto-failure jobs.
 201   # Check out their documentation for more information on why they're needed.
 202
 203   - &base-outcome-job
 204     name: bors build finished
 205     runs-on: ubuntu-latest
 206
 207   - &base-success-job
 208     steps:
 209       - name: mark the job as a success
 210         run: exit 0
 211         shell: bash
 212     <<: *base-outcome-job
 213
 214   - &base-failure-job
 215     steps:
 216       - name: mark the job as a failure
 217         run: exit 1
 218         shell: bash
 219     <<: *base-outcome-job
 220
 221 ###########################
 222 #   Builders definition   #
 223 ###########################
 224
 225 name: CI
 226 on:
 227   push:
 228     branches:
 229       - auto
 230       - try
 231       - master
 232   pull_request:
 233     branches:
 234       - "**"
 235
 236 defaults:
 237   run:
 238     # While on Linux and macOS builders it just forwards the arguments to the
 239     # system bash, this wrapper allows switching from the host's bash.exe to
 240     # the one we install along with MSYS2 mid-build on Windows.
 241     #
 242     # Once the step to install MSYS2 is executed, the CI_OVERRIDE_SHELL
 243     # environment variable is set pointing to our MSYS2's bash.exe. From that
 244     # moment the host's bash.exe will not be called anymore.
 245     #
 246     # This is needed because we can't launch our own bash.exe from the host
 247     # bash.exe, as that would load two different cygwin1.dll in memory, causing
 248     # "cygwin heap mismatch" errors.
 249     shell: python src/ci/exec-with-shell.py {0}
 250
 251 jobs:
 252   pr:
 253     <<: *base-ci-job
 254     name: PR
 255     env:
 256       <<: [*shared-ci-variables, *public-variables]
 257     if: github.event_name == 'pull_request'
 258     strategy:
 259       matrix:
 260         include:
 261           - name: mingw-check
 262             <<: *job-linux-xl
 263
 264           - name: x86_64-gnu-llvm-8
 265             <<: *job-linux-xl
 266
 267           - name: x86_64-gnu-tools
 268             env:
 269               CI_ONLY_WHEN_SUBMODULES_CHANGED: 1
 270             <<: *job-linux-xl
 271
 272   try:
 273     <<: *base-ci-job
 274     name: try
 275     env:
 276       <<: [*shared-ci-variables, *prod-variables]
 277     if: github.event_name == 'push' && github.ref == 'refs/heads/try' && github.repository == 'rust-lang-ci/rust'
 278     strategy:
 279       matrix:
 280         include:
 281           - name: dist-x86_64-linux
 282             <<: *job-linux-xl
 283
 284   auto:
 285     <<: *base-ci-job
 286     name: auto
 287     env:
 288       <<: [*shared-ci-variables, *prod-variables]
 289     if: github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'
 290     strategy:
 291       matrix:
 292         include:
 293           #############################
 294           #   Linux/Docker builders   #
 295           #############################
 296
 297           - name: arm-android
 298             <<: *job-linux-xl
 299
 300           - name: armhf-gnu
 301             <<: *job-linux-xl
 302
 303           - name: dist-aarch64-linux
 304             <<: *job-linux-xl
 305
 306           - name: dist-android
 307             <<: *job-linux-xl
 308
 309           - name: dist-arm-linux
 310             <<: *job-linux-xl
 311
 312           - name: dist-armhf-linux
 313             <<: *job-linux-xl
 314
 315           - name: dist-armv7-linux
 316             <<: *job-linux-xl
 317
 318           - name: dist-i586-gnu-i586-i686-musl
 319             <<: *job-linux-xl
 320
 321           - name: dist-i686-freebsd
 322             <<: *job-linux-xl
 323
 324           - name: dist-i686-linux
 325             <<: *job-linux-xl
 326
 327           - name: dist-mips-linux
 328             <<: *job-linux-xl
 329
 330           - name: dist-mips64-linux
 331             <<: *job-linux-xl
 332
 333           - name: dist-mips64el-linux
 334             <<: *job-linux-xl
 335
 336           - name: dist-mipsel-linux
 337             <<: *job-linux-xl
 338
 339           - name: dist-powerpc-linux
 340             <<: *job-linux-xl
 341
 342           - name: dist-powerpc64-linux
 343             <<: *job-linux-xl
 344
 345           - name: dist-powerpc64le-linux
 346             <<: *job-linux-xl
 347
 348           - name: dist-s390x-linux
 349             <<: *job-linux-xl
 350
 351           - name: dist-various-1
 352             <<: *job-linux-xl
 353
 354           - name: dist-various-2
 355             <<: *job-linux-xl
 356
 357           - name: dist-x86_64-freebsd
 358             <<: *job-linux-xl
 359
 360           - name: dist-x86_64-linux
 361             <<: *job-linux-xl
 362
 363           - name: dist-x86_64-linux-alt
 364             env:
 365               IMAGE: dist-x86_64-linux
 366             <<: *job-linux-xl
 367
 368           - name: dist-x86_64-musl
 369             <<: *job-linux-xl
 370
 371           - name: dist-x86_64-netbsd
 372             <<: *job-linux-xl
 373
 374           - name: i686-gnu
 375             <<: *job-linux-xl
 376
 377           - name: i686-gnu-nopt
 378             <<: *job-linux-xl
 379
 380           - name: mingw-check
 381             <<: *job-linux-xl
 382
 383           - name: test-various
 384             <<: *job-linux-xl
 385
 386           - name: wasm32
 387             <<: *job-linux-xl
 388
 389           - name: x86_64-gnu
 390             <<: *job-linux-xl
 391
 392           - name: x86_64-gnu-aux
 393             <<: *job-linux-xl
 394
 395           - name: x86_64-gnu-debug
 396             <<: *job-linux-xl
 397
 398           - name: x86_64-gnu-distcheck
 399             <<: *job-linux-xl
 400
 401           - name: x86_64-gnu-full-bootstrap
 402             <<: *job-linux-xl
 403
 404           - name: x86_64-gnu-llvm-8
 405             env:
 406               RUST_BACKTRACE: 1
 407             <<: *job-linux-xl
 408
 409           - name: x86_64-gnu-nopt
 410             <<: *job-linux-xl
 411
 412           - name: x86_64-gnu-tools
 413             env:
 414               DEPLOY_TOOLSTATES_JSON: toolstates-linux.json
 415             <<: *job-linux-xl
 416
 417           ####################
 418           #  macOS Builders  #
 419           ####################
 420
 421           - name: dist-x86_64-apple
 422             env:
 423               SCRIPT: ./x.py dist
 424               RUST_CONFIGURE_ARGS: --target=aarch64-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc
 425               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
 426               MACOSX_DEPLOYMENT_TARGET: 10.7
 427               NO_LLVM_ASSERTIONS: 1
 428               NO_DEBUG_ASSERTIONS: 1
 429               DIST_REQUIRE_ALL_TOOLS: 1
 430             <<: *job-macos-xl
 431
 432           - name: dist-x86_64-apple-alt
 433             env:
 434               SCRIPT: ./x.py dist
 435               RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --set rust.jemalloc
 436               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
 437               MACOSX_DEPLOYMENT_TARGET: 10.7
 438               NO_LLVM_ASSERTIONS: 1
 439               NO_DEBUG_ASSERTIONS: 1
 440             <<: *job-macos-xl
 441
 442           - name: x86_64-apple
 443             env:
 444               SCRIPT: ./x.py test
 445               RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
 446               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
 447               MACOSX_DEPLOYMENT_TARGET: 10.8
 448               MACOSX_STD_DEPLOYMENT_TARGET: 10.7
 449               NO_LLVM_ASSERTIONS: 1
 450               NO_DEBUG_ASSERTIONS: 1
 451             <<: *job-macos-xl
 452
 453           ######################
 454           #  Windows Builders  #
 455           ######################
 456
 457           - name: x86_64-msvc-1
 458             env:
 459               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
 460               SCRIPT: make ci-subset-1
 461               # FIXME(#59637)
 462               NO_DEBUG_ASSERTIONS: 1
 463               NO_LLVM_ASSERTIONS: 1
 464             <<: *job-windows-xl
 465
 466           - name: x86_64-msvc-2
 467             env:
 468               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
 469               SCRIPT: make ci-subset-2
 470             <<: *job-windows-xl
 471
 472           - name: i686-msvc-1
 473             env:
 474               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
 475               SCRIPT: make ci-subset-1
 476               # FIXME(#59637)
 477               NO_DEBUG_ASSERTIONS: 1
 478               NO_LLVM_ASSERTIONS: 1
 479             <<: *job-windows-xl
 480
 481           - name: i686-msvc-2
 482             env:
 483               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
 484               SCRIPT: make ci-subset-2
 485               # FIXME(#59637)
 486               NO_DEBUG_ASSERTIONS: 1
 487               NO_LLVM_ASSERTIONS: 1
 488             <<: *job-windows-xl
 489
 490           - name: x86_64-msvc-cargo
 491             env:
 492               SCRIPT: python x.py test src/tools/cargotest src/tools/cargo
 493               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-lld
 494               VCVARS_BAT: vcvars64.bat
 495               # FIXME(#59637)
 496               NO_DEBUG_ASSERTIONS: 1
 497               NO_LLVM_ASSERTIONS: 1
 498             <<: *job-windows-xl
 499
 500           - name: x86_64-msvc-tools
 501             env:
 502               SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows
 503               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstate/toolstates.json
 504             <<: *job-windows-xl
 505
 506           # 32/64-bit MinGW builds.
 507           #
 508           # We are using MinGW with posix threads since LLVM does not compile with
 509           # the win32 threads version due to missing support for C++'s std::thread.
 510           #
 511           # Instead of relying on the MinGW version installed on appveryor we download
 512           # and install one ourselves so we won't be surprised by changes to appveyor's
 513           # build image.
 514           #
 515           # Finally, note that the downloads below are all in the `rust-lang-ci` S3
 516           # bucket, but they cleraly didn't originate there! The downloads originally
 517           # came from the mingw-w64 SourceForge download site. Unfortunately
 518           # SourceForge is notoriously flaky, so we mirror it on our own infrastructure.
 519
 520           - name: i686-mingw-1
 521             env:
 522               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
 523               SCRIPT: make ci-mingw-subset-1
 524               CUSTOM_MINGW: 1
 525               # FIXME(#59637)
 526               NO_DEBUG_ASSERTIONS: 1
 527               NO_LLVM_ASSERTIONS: 1
 528             <<: *job-windows-xl
 529
 530           - name: i686-mingw-2
 531             env:
 532               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
 533               SCRIPT: make ci-mingw-subset-2
 534               CUSTOM_MINGW: 1
 535             <<: *job-windows-xl
 536
 537           - name: x86_64-mingw-1
 538             env:
 539               SCRIPT: make ci-mingw-subset-1
 540               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
 541               CUSTOM_MINGW: 1
 542               # FIXME(#59637)
 543               NO_DEBUG_ASSERTIONS: 1
 544               NO_LLVM_ASSERTIONS: 1
 545             <<: *job-windows-xl
 546
 547           - name: x86_64-mingw-2
 548             env:
 549               SCRIPT: make ci-mingw-subset-2
 550               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
 551               CUSTOM_MINGW: 1
 552             <<: *job-windows-xl
 553
 554           - name: dist-x86_64-msvc
 555             env:
 556               RUST_CONFIGURE_ARGS: >-
 557                 --build=x86_64-pc-windows-msvc
 558                 --target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc
 559                 --enable-full-tools
 560                 --enable-profiler
 561               SCRIPT: python x.py dist
 562               DIST_REQUIRE_ALL_TOOLS: 1
 563             <<: *job-windows-xl
 564
 565           - name: dist-i686-msvc
 566             env:
 567               RUST_CONFIGURE_ARGS: >-
 568                 --build=i686-pc-windows-msvc
 569                 --target=i586-pc-windows-msvc
 570                 --enable-full-tools
 571                 --enable-profiler
 572               SCRIPT: python x.py dist
 573               DIST_REQUIRE_ALL_TOOLS: 1
 574             <<: *job-windows-xl
 575
 576           - name: dist-i686-mingw
 577             env:
 578               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools --enable-profiler
 579               SCRIPT: python x.py dist
 580               CUSTOM_MINGW: 1
 581               DIST_REQUIRE_ALL_TOOLS: 1
 582             <<: *job-windows-xl
 583
 584           - name: dist-x86_64-mingw
 585             env:
 586               SCRIPT: python x.py dist
 587               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler
 588               CUSTOM_MINGW: 1
 589               DIST_REQUIRE_ALL_TOOLS: 1
 590             <<: *job-windows-xl
 591
 592           - name: dist-x86_64-msvc-alt
 593             env:
 594               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler
 595               SCRIPT: python x.py dist
 596             <<: *job-windows-xl
 597
 598   master:
 599     name: master
 600     runs-on: ubuntu-latest
 601     env:
 602       <<: [*prod-variables]
 603     if: github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'rust-lang-ci/rust'
 604     steps:
 605       - name: checkout the source code
 606         uses: actions/checkout@v1
 607         with:
 608           fetch-depth: 2
 609
 610       - name: publish toolstate
 611         run: src/ci/publish_toolstate.sh
 612         env:
 613           TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
 614         <<: *step
 615
 616   # These jobs don't actually test anything, but they're used to tell bors the
 617   # build completed, as there is no practical way to detect when a workflow is
 618   # successful listening to webhooks only.
 619   try-success:
 620     needs: [try]
 621     if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/try' && github.repository == 'rust-lang-ci/rust'"
 622     <<: *base-success-job
 623   try-failure:
 624     needs: [try]
 625     if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/try' && github.repository == 'rust-lang-ci/rust'"
 626     <<: *base-failure-job
 627   auto-success:
 628     needs: [auto]
 629     if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
 630     <<: *base-success-job
 631   auto-failure:
 632     needs: [auto]
 633     if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
 634     <<: *base-failure-job