src/ci/github-actions/ci.yml

   1 ######################################################
   2 #   WARNING! Action needed when changing this file   #
   3 ######################################################
   4
   5 # Due to GitHub Actions limitations, we can't use YAML Anchors directly in the
   6 # CI configuration stored on the repository. To work around that this file is
   7 # expanded by a tool in the repository, and the expansion is committed as well.
   8 #
   9 # After you make any change to the file you'll need to run this command:
  10 #
  11 #   ./x.py run src/tools/expand-yaml-anchors
  12 #
  13 # ...and commit the file it updated in addition to this one. If you forget this
  14 # step CI will fail.
  15
  16 ---
  17
  18 ###############################
  19 #   YAML Anchors Definition   #
  20 ###############################
  21
  22 # This key contains most of the YAML anchors that will be used later in the
  23 # document. YAML anchors allows us to greatly reduce duplication inside the CI
  24 # configuration by reusing parts of the configuration.
  25 #
  26 # YAML anchors work by defining an anchor with `&anchor-name` and reusing its
  27 # content in another place with `*anchor-name`. The special `<<` map key merges
  28 # the content of the map with the content of the anchor (or list of anchors).
  29 #
  30 # The expand-yaml-anchors tool will automatically remove this block from the
  31 # output YAML file.
  32 x--expand-yaml-anchors--remove:
  33
  34   - &shared-ci-variables
  35     CI_JOB_NAME: ${{ matrix.name }}
  36
  37   - &public-variables
  38     SCCACHE_BUCKET: rust-lang-gha-caches
  39     TOOLSTATE_REPO: https://github.com/pietroalbini/rust-toolstate
  40     CACHE_DOMAIN: ci-caches-gha.rust-lang.org
  41
  42   - &prod-variables
  43     SCCACHE_BUCKET: rust-lang-gha-caches
  44     DEPLOY_BUCKET: rust-lang-gha
  45     TOOLSTATE_REPO: https://github.com/pietroalbini/rust-toolstate
  46     TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/pietroalbini/rust-toolstate/issues
  47     TOOLSTATE_PUBLISH: 1
  48     # AWS_SECRET_ACCESS_KEYs are stored in GitHub's secrets storage, named
  49     # AWS_SECRET_ACCESS_KEY_<keyid>. Including the key id in the name allows to
  50     # rotate them in a single branch while keeping the old key in another
  51     # branch, which wouldn't be possible if the key was named with the kind
  52     # (caches, artifacts...).
  53     CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZOMUQATD5
  54     ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZH5AYXDVF
  55     CACHE_DOMAIN: ci-caches-gha.rust-lang.org
  56
  57   - &base-job
  58     env: {}
  59
  60   - &job-linux-xl
  61     os: ubuntu-latest-xl
  62     <<: *base-job
  63
  64   - &job-macos-xl
  65     os: macos-latest  # We don't have an XL builder for this
  66     <<: *base-job
  67
  68   - &job-windows-xl
  69     os: windows-latest-xl
  70     <<: *base-job
  71
  72   - &step
  73     if: success() && !env.SKIP_JOB
  74
  75   - &base-ci-job
  76     timeout-minutes: 600
  77     runs-on: "${{ matrix.os }}"
  78     env: *shared-ci-variables
  79     steps:
  80       - name: disable git crlf conversion
  81         run: git config --global core.autocrlf false
  82
  83       - name: checkout the source code
  84         uses: actions/checkout@v1
  85         with:
  86           fetch-depth: 2
  87
  88       - name: configure GitHub Actions to kill the build when outdated
  89         uses: rust-lang/simpleinfra/github-actions/cancel-outdated-builds@master
  90         with:
  91           github_token: "${{ secrets.github_token }}"
  92         if: success() && !env.SKIP_JOB && github.ref != 'refs/heads/try'
  93         <<: *step
  94
  95       - name: add extra environment variables
  96         run: src/ci/scripts/setup-environment.sh
  97         env:
  98           # Since it's not possible to merge `${{ matrix.env }}` with the other
  99           # variables in `job.<name>.env`, the variables defined in the matrix
 100           # are passed to the `setup-environment.sh` script encoded in JSON,
 101           # which then uses log commands to actually set them.
 102           EXTRA_VARIABLES: ${{ toJson(matrix.env) }}
 103         <<: *step
 104
 105       - name: decide whether to skip this job
 106         run: src/ci/scripts/should-skip-this.sh
 107         <<: *step
 108
 109       - name: collect CPU statistics
 110         run: src/ci/scripts/collect-cpu-stats.sh
 111         <<: *step
 112
 113       - name: show the current environment
 114         run: src/ci/scripts/dump-environment.sh
 115         <<: *step
 116
 117       - name: install awscli
 118         run: src/ci/scripts/install-awscli.sh
 119         <<: *step
 120
 121       - name: install sccache
 122         run: src/ci/scripts/install-sccache.sh
 123         <<: *step
 124
 125       - name: install clang
 126         run: src/ci/scripts/install-clang.sh
 127         <<: *step
 128
 129       - name: install WIX
 130         run: src/ci/scripts/install-wix.sh
 131         <<: *step
 132
 133       - name: ensure the build happens on a partition with enough space
 134         run: src/ci/scripts/symlink-build-dir.sh
 135         <<: *step
 136
 137       - name: disable git crlf conversion
 138         run: src/ci/scripts/disable-git-crlf-conversion.sh
 139         <<: *step
 140
 141       - name: install MSYS2
 142         run: src/ci/scripts/install-msys2.sh
 143         <<: *step
 144
 145       - name: install MinGW
 146         run: src/ci/scripts/install-mingw.sh
 147         <<: *step
 148
 149       - name: install ninja
 150         run: src/ci/scripts/install-ninja.sh
 151         <<: *step
 152
 153       - name: enable ipv6 on Docker
 154         run: src/ci/scripts/enable-docker-ipv6.sh
 155         <<: *step
 156
 157       # Disable automatic line ending conversion (again). On Windows, when we're
 158       # installing dependencies, something switches the git configuration directory or
 159       # re-enables autocrlf. We've not tracked down the exact cause -- and there may
 160       # be multiple -- but this should ensure submodules are checked out with the
 161       # appropriate line endings.
 162       - name: disable git crlf conversion
 163         run: src/ci/scripts/disable-git-crlf-conversion.sh
 164         <<: *step
 165
 166       - name: checkout submodules
 167         run: src/ci/scripts/checkout-submodules.sh
 168         <<: *step
 169
 170       - name: ensure line endings are correct
 171         run: src/ci/scripts/verify-line-endings.sh
 172         <<: *step
 173
 174       - name: run the build
 175         run: src/ci/scripts/run-build-from-ci.sh
 176         env:
 177           AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }}
 178           AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}
 179           TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
 180         <<: *step
 181
 182       - name: upload artifacts to S3
 183         run: src/ci/scripts/upload-artifacts.sh
 184         env:
 185           AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}
 186           AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}
 187         # Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy
 188         # builders *should* have the AWS credentials available. Still, explicitly
 189         # adding the condition is helpful as this way CI will not silently skip
 190         # deploying artifacts from a dist builder if the variables are misconfigured,
 191         # erroring about invalid credentials instead.
 192         if: success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')
 193         <<: *step
 194
 195   # These snippets are used by the try-success, try-failure, auto-success and auto-failure jobs.
 196   # Check out their documentation for more information on why they're needed.
 197
 198   - &base-outcome-job
 199     name: bors build finished
 200     runs-on: ubuntu-latest
 201
 202   - &base-success-job
 203     steps:
 204       - name: mark the job as a success
 205         run: exit 0
 206         shell: bash
 207     <<: *base-outcome-job
 208
 209   - &base-failure-job
 210     steps:
 211       - name: mark the job as a failure
 212         run: exit 1
 213         shell: bash
 214     <<: *base-outcome-job
 215
 216 ###########################
 217 #   Builders definition   #
 218 ###########################
 219
 220 name: CI
 221 on:
 222   push:
 223     branches:
 224       - auto
 225       - try
 226       - master
 227   pull_request:
 228     branches:
 229       - "**"
 230
 231 defaults:
 232   run:
 233     # On Linux, macOS, and Windows, use the system-provided bash as the default
 234     # shell. (This should only make a difference on Windows, where the default
 235     # shell is PowerShell.)
 236     shell: bash
 237
 238 jobs:
 239   pr:
 240     <<: *base-ci-job
 241     name: PR
 242     env:
 243       <<: [*shared-ci-variables, *public-variables]
 244     if: github.event_name == 'pull_request'
 245     strategy:
 246       matrix:
 247         include:
 248           - name: mingw-check
 249             <<: *job-linux-xl
 250
 251           - name: x86_64-gnu-llvm-8
 252             <<: *job-linux-xl
 253
 254           - name: x86_64-gnu-tools
 255             env:
 256               CI_ONLY_WHEN_SUBMODULES_CHANGED: 1
 257             <<: *job-linux-xl
 258
 259   try:
 260     <<: *base-ci-job
 261     name: try
 262     env:
 263       <<: [*shared-ci-variables, *prod-variables]
 264     if: github.event_name == 'push' && github.ref == 'refs/heads/try' && github.repository == 'rust-lang-ci/rust'
 265     strategy:
 266       matrix:
 267         include:
 268           - name: dist-x86_64-linux
 269             <<: *job-linux-xl
 270
 271   auto:
 272     <<: *base-ci-job
 273     name: auto
 274     env:
 275       <<: [*shared-ci-variables, *prod-variables]
 276     if: github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'
 277     strategy:
 278       matrix:
 279         include:
 280           #############################
 281           #   Linux/Docker builders   #
 282           #############################
 283
 284           - name: arm-android
 285             <<: *job-linux-xl
 286
 287           - name: armhf-gnu
 288             <<: *job-linux-xl
 289
 290           - name: dist-aarch64-linux
 291             <<: *job-linux-xl
 292
 293           - name: dist-android
 294             <<: *job-linux-xl
 295
 296           - name: dist-arm-linux
 297             <<: *job-linux-xl
 298
 299           - name: dist-armhf-linux
 300             <<: *job-linux-xl
 301
 302           - name: dist-armv7-linux
 303             <<: *job-linux-xl
 304
 305           - name: dist-i586-gnu-i586-i686-musl
 306             <<: *job-linux-xl
 307
 308           - name: dist-i686-freebsd
 309             <<: *job-linux-xl
 310
 311           - name: dist-i686-linux
 312             <<: *job-linux-xl
 313
 314           - name: dist-mips-linux
 315             <<: *job-linux-xl
 316
 317           - name: dist-mips64-linux
 318             <<: *job-linux-xl
 319
 320           - name: dist-mips64el-linux
 321             <<: *job-linux-xl
 322
 323           - name: dist-mipsel-linux
 324             <<: *job-linux-xl
 325
 326           - name: dist-powerpc-linux
 327             <<: *job-linux-xl
 328
 329           - name: dist-powerpc64-linux
 330             <<: *job-linux-xl
 331
 332           - name: dist-powerpc64le-linux
 333             <<: *job-linux-xl
 334
 335           - name: dist-riscv64-linux
 336             <<: *job-linux-xl
 337
 338           - name: dist-s390x-linux
 339             <<: *job-linux-xl
 340
 341           - name: dist-various-1
 342             <<: *job-linux-xl
 343
 344           - name: dist-various-2
 345             <<: *job-linux-xl
 346
 347           - name: dist-x86_64-freebsd
 348             <<: *job-linux-xl
 349
 350           - name: dist-x86_64-illumos
 351             <<: *job-linux-xl
 352
 353           - name: dist-x86_64-linux
 354             <<: *job-linux-xl
 355
 356           - name: dist-x86_64-linux-alt
 357             env:
 358               IMAGE: dist-x86_64-linux
 359             <<: *job-linux-xl
 360
 361           - name: dist-x86_64-musl
 362             <<: *job-linux-xl
 363
 364           - name: dist-x86_64-netbsd
 365             <<: *job-linux-xl
 366
 367           - name: i686-gnu
 368             <<: *job-linux-xl
 369
 370           - name: i686-gnu-nopt
 371             <<: *job-linux-xl
 372
 373           - name: mingw-check
 374             <<: *job-linux-xl
 375
 376           - name: test-various
 377             <<: *job-linux-xl
 378
 379           - name: wasm32
 380             <<: *job-linux-xl
 381
 382           - name: x86_64-gnu
 383             <<: *job-linux-xl
 384
 385           - name: x86_64-gnu-aux
 386             <<: *job-linux-xl
 387
 388           - name: x86_64-gnu-debug
 389             <<: *job-linux-xl
 390
 391           - name: x86_64-gnu-distcheck
 392             <<: *job-linux-xl
 393
 394           - name: x86_64-gnu-full-bootstrap
 395             <<: *job-linux-xl
 396
 397           - name: x86_64-gnu-llvm-8
 398             env:
 399               RUST_BACKTRACE: 1
 400             <<: *job-linux-xl
 401
 402           - name: x86_64-gnu-nopt
 403             <<: *job-linux-xl
 404
 405           - name: x86_64-gnu-tools
 406             env:
 407               DEPLOY_TOOLSTATES_JSON: toolstates-linux.json
 408             <<: *job-linux-xl
 409
 410           ######################
 411           #  Windows Builders  #
 412           ######################
 413
 414           - name: x86_64-msvc-1
 415             env:
 416               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
 417               SCRIPT: make ci-subset-1
 418               # FIXME(#59637)
 419               NO_DEBUG_ASSERTIONS: 1
 420               NO_LLVM_ASSERTIONS: 1
 421             <<: *job-windows-xl
 422
 423           - name: x86_64-msvc-2
 424             env:
 425               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
 426               SCRIPT: make ci-subset-2
 427             <<: *job-windows-xl
 428
 429           - name: i686-msvc-1
 430             env:
 431               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
 432               SCRIPT: make ci-subset-1
 433               # FIXME(#59637)
 434               NO_DEBUG_ASSERTIONS: 1
 435               NO_LLVM_ASSERTIONS: 1
 436             <<: *job-windows-xl
 437
 438           - name: i686-msvc-2
 439             env:
 440               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
 441               SCRIPT: make ci-subset-2
 442               # FIXME(#59637)
 443               NO_DEBUG_ASSERTIONS: 1
 444               NO_LLVM_ASSERTIONS: 1
 445             <<: *job-windows-xl
 446
 447           - name: x86_64-msvc-cargo
 448             env:
 449               SCRIPT: python x.py test src/tools/cargotest src/tools/cargo
 450               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-lld
 451               VCVARS_BAT: vcvars64.bat
 452               # FIXME(#59637)
 453               NO_DEBUG_ASSERTIONS: 1
 454               NO_LLVM_ASSERTIONS: 1
 455             <<: *job-windows-xl
 456
 457           - name: x86_64-msvc-tools
 458             env:
 459               SCRIPT: src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows
 460               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstate/toolstates.json
 461             <<: *job-windows-xl
 462
 463           # 32/64-bit MinGW builds.
 464           #
 465           # We are using MinGW with posix threads since LLVM does not compile with
 466           # the win32 threads version due to missing support for C++'s std::thread.
 467           #
 468           # Instead of relying on the MinGW version installed on appveryor we download
 469           # and install one ourselves so we won't be surprised by changes to appveyor's
 470           # build image.
 471           #
 472           # Finally, note that the downloads below are all in the `rust-lang-ci` S3
 473           # bucket, but they cleraly didn't originate there! The downloads originally
 474           # came from the mingw-w64 SourceForge download site. Unfortunately
 475           # SourceForge is notoriously flaky, so we mirror it on our own infrastructure.
 476
 477           - name: i686-mingw-1
 478             env:
 479               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
 480               SCRIPT: make ci-mingw-subset-1
 481               CUSTOM_MINGW: 1
 482               # FIXME(#59637)
 483               NO_DEBUG_ASSERTIONS: 1
 484               NO_LLVM_ASSERTIONS: 1
 485             <<: *job-windows-xl
 486
 487           - name: i686-mingw-2
 488             env:
 489               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
 490               SCRIPT: make ci-mingw-subset-2
 491               CUSTOM_MINGW: 1
 492             <<: *job-windows-xl
 493
 494           - name: x86_64-mingw-1
 495             env:
 496               SCRIPT: make ci-mingw-subset-1
 497               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
 498               CUSTOM_MINGW: 1
 499               # FIXME(#59637)
 500               NO_DEBUG_ASSERTIONS: 1
 501               NO_LLVM_ASSERTIONS: 1
 502             <<: *job-windows-xl
 503
 504           - name: x86_64-mingw-2
 505             env:
 506               SCRIPT: make ci-mingw-subset-2
 507               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
 508               CUSTOM_MINGW: 1
 509             <<: *job-windows-xl
 510
 511           - name: dist-x86_64-msvc
 512             env:
 513               RUST_CONFIGURE_ARGS: >-
 514                 --build=x86_64-pc-windows-msvc
 515                 --target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc
 516                 --enable-full-tools
 517                 --enable-profiler
 518               SCRIPT: python x.py dist
 519               DIST_REQUIRE_ALL_TOOLS: 1
 520             <<: *job-windows-xl
 521
 522           - name: dist-i686-msvc
 523             env:
 524               RUST_CONFIGURE_ARGS: >-
 525                 --build=i686-pc-windows-msvc
 526                 --target=i586-pc-windows-msvc
 527                 --enable-full-tools
 528                 --enable-profiler
 529               SCRIPT: python x.py dist
 530               DIST_REQUIRE_ALL_TOOLS: 1
 531             <<: *job-windows-xl
 532
 533           - name: dist-i686-mingw
 534             env:
 535               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools --enable-profiler
 536               SCRIPT: python x.py dist
 537               CUSTOM_MINGW: 1
 538               DIST_REQUIRE_ALL_TOOLS: 1
 539             <<: *job-windows-xl
 540
 541           - name: dist-x86_64-mingw
 542             env:
 543               SCRIPT: python x.py dist
 544               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler
 545               CUSTOM_MINGW: 1
 546               DIST_REQUIRE_ALL_TOOLS: 1
 547             <<: *job-windows-xl
 548
 549           - name: dist-x86_64-msvc-alt
 550             env:
 551               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler
 552               SCRIPT: python x.py dist
 553             <<: *job-windows-xl
 554
 555   auto-fallible:
 556     <<: *base-ci-job
 557     name: auto-fallible
 558     env:
 559       <<: [*shared-ci-variables, *prod-variables]
 560     if: github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'
 561     strategy:
 562       matrix:
 563         include:
 564           ####################
 565           #  macOS Builders  #
 566           ####################
 567
 568           - name: dist-x86_64-apple
 569             env:
 570               SCRIPT: ./x.py dist
 571               RUST_CONFIGURE_ARGS: --target=aarch64-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc
 572               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
 573               MACOSX_DEPLOYMENT_TARGET: 10.7
 574               NO_LLVM_ASSERTIONS: 1
 575               NO_DEBUG_ASSERTIONS: 1
 576               DIST_REQUIRE_ALL_TOOLS: 1
 577             <<: *job-macos-xl
 578
 579           - name: dist-x86_64-apple-alt
 580             env:
 581               SCRIPT: ./x.py dist
 582               RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --set rust.jemalloc
 583               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
 584               MACOSX_DEPLOYMENT_TARGET: 10.7
 585               NO_LLVM_ASSERTIONS: 1
 586               NO_DEBUG_ASSERTIONS: 1
 587             <<: *job-macos-xl
 588
 589           - name: x86_64-apple
 590             env:
 591               SCRIPT: ./x.py test
 592               RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
 593               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
 594               MACOSX_DEPLOYMENT_TARGET: 10.8
 595               MACOSX_STD_DEPLOYMENT_TARGET: 10.7
 596               NO_LLVM_ASSERTIONS: 1
 597               NO_DEBUG_ASSERTIONS: 1
 598             <<: *job-macos-xl
 599
 600   master:
 601     name: master
 602     runs-on: ubuntu-latest
 603     env:
 604       <<: [*prod-variables]
 605     if: github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'rust-lang-ci/rust'
 606     steps:
 607       - name: checkout the source code
 608         uses: actions/checkout@v1
 609         with:
 610           fetch-depth: 2
 611
 612       - name: publish toolstate
 613         run: src/ci/publish_toolstate.sh
 614         shell: bash
 615         env:
 616           TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
 617         <<: *step
 618
 619   # These jobs don't actually test anything, but they're used to tell bors the
 620   # build completed, as there is no practical way to detect when a workflow is
 621   # successful listening to webhooks only.
 622   try-success:
 623     needs: [try]
 624     if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/try' && github.repository == 'rust-lang-ci/rust'"
 625     <<: *base-success-job
 626   try-failure:
 627     needs: [try]
 628     if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/try' && github.repository == 'rust-lang-ci/rust'"
 629     <<: *base-failure-job
 630   auto-success:
 631     needs: [auto]
 632     if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
 633     <<: *base-success-job
 634   auto-failure:
 635     needs: [auto]
 636     if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
 637     <<: *base-failure-job