]> git.lizzy.rs Git - rust.git/blob - src/ci/github-actions/ci.yml
ci: allow RLA to pick the right PR number
[rust.git] / src / ci / github-actions / ci.yml
1 ######################################################
2 #   WARNING! Action needed when changing this file   #
3 ######################################################
4
5 # Due to GitHub Actions limitations, we can't use YAML Anchors directly in the
6 # CI configuration stored on the repository. To work around that this file is
7 # expanded by a tool in the repository, and the expansion is committed as well.
8 #
9 # After you make any change to the file you'll need to run this command:
10 #
11 #   ./x.py run src/tools/expand-yaml-anchors
12 #
13 # ...and commit the file it updated in addition to this one. If you forget this
14 # step CI will fail.
15
16 ---
17
18 ###############################
19 #   YAML Anchors Definition   #
20 ###############################
21
22 # This key contains most of the YAML anchors that will be used later in the
23 # document. YAML anchors allows us to greatly reduce duplication inside the CI
24 # configuration by reusing parts of the configuration.
25 #
26 # YAML anchors work by defining an anchor with `&anchor-name` and reusing its
27 # content in another place with `*anchor-name`. The special `<<` map key merges
28 # the content of the map with the content of the anchor (or list of anchors).
29 #
30 # The expand-yaml-anchors tool will automatically remove this block from the
31 # output YAML file.
32 x--expand-yaml-anchors--remove:
33
34   - &shared-ci-variables
35     CI_JOB_NAME: ${{ matrix.name }}
36
37   - &public-variables
38     SCCACHE_BUCKET: rust-lang-gha-caches
39     TOOLSTATE_REPO: https://github.com/pietroalbini/rust-toolstate
40     CACHE_DOMAIN: ci-caches-gha.rust-lang.org
41
42   - &prod-variables
43     SCCACHE_BUCKET: rust-lang-gha-caches
44     DEPLOY_BUCKET: rust-lang-gha
45     TOOLSTATE_REPO: https://github.com/pietroalbini/rust-toolstate
46     TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/pietroalbini/rust-toolstate/issues
47     TOOLSTATE_PUBLISH: 1
48     # AWS_SECRET_ACCESS_KEYs are stored in GitHub's secrets storage, named
49     # AWS_SECRET_ACCESS_KEY_<keyid>. Including the key id in the name allows to
50     # rotate them in a single branch while keeping the old key in another
51     # branch, which wouldn't be possible if the key was named with the kind
52     # (caches, artifacts...).
53     CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZOMUQATD5
54     ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZH5AYXDVF
55     CACHE_DOMAIN: ci-caches-gha.rust-lang.org
56
57   - &base-job
58     env: {}
59
60   - &job-linux-xl
61     os: ubuntu-latest-xl
62     <<: *base-job
63
64   - &job-macos-xl
65     os: macos-latest  # We don't have an XL builder for this
66     <<: *base-job
67
68   - &job-windows-xl
69     os: windows-latest-xl
70     <<: *base-job
71
72   - &step
73     if: success() && !env.SKIP_JOB
74
75   - &base-ci-job
76     timeout-minutes: 600
77     runs-on: "${{ matrix.os }}"
78     env: *shared-ci-variables
79     steps:
80       - name: disable git crlf conversion
81         run: git config --global core.autocrlf false
82
83       - name: checkout the source code
84         uses: actions/checkout@v1
85         with:
86           fetch-depth: 2
87
88       - name: configure GitHub Actions to kill the build when outdated
89         uses: rust-lang/simpleinfra/github-actions/cancel-outdated-builds@master
90         with:
91           github_token: "${{ secrets.github_token }}"
92         if: success() && !env.SKIP_JOB && github.ref != 'refs/heads/try'
93         <<: *step
94
95       # Rust Log Analyzer can't currently detect the PR number of a GitHub
96       # Actions build on its own, so a hint in the log message is needed to
97       # point it in the right direction.
98       - name: configure the PR in which the error message will be posted
99         run: echo "[CI_PR_NUMBER=$num]"
100         env:
101           num: ${{ github.event.number }}
102         if: success() && !env.SKIP_JOBS && github.event_name == 'pull_request'
103
104       - name: add extra environment variables
105         run: src/ci/scripts/setup-environment.sh
106         env:
107           # Since it's not possible to merge `${{ matrix.env }}` with the other
108           # variables in `job.<name>.env`, the variables defined in the matrix
109           # are passed to the `setup-environment.sh` script encoded in JSON,
110           # which then uses log commands to actually set them.
111           EXTRA_VARIABLES: ${{ toJson(matrix.env) }}
112         <<: *step
113
114       - name: decide whether to skip this job
115         run: src/ci/scripts/should-skip-this.sh
116         <<: *step
117
118       - name: collect CPU statistics
119         run: src/ci/scripts/collect-cpu-stats.sh
120         <<: *step
121
122       - name: show the current environment
123         run: src/ci/scripts/dump-environment.sh
124         <<: *step
125
126       - name: install awscli
127         run: src/ci/scripts/install-awscli.sh
128         <<: *step
129
130       - name: install sccache
131         run: src/ci/scripts/install-sccache.sh
132         <<: *step
133
134       - name: install clang
135         run: src/ci/scripts/install-clang.sh
136         <<: *step
137
138       - name: install WIX
139         run: src/ci/scripts/install-wix.sh
140         <<: *step
141
142       - name: ensure the build happens on a partition with enough space
143         run: src/ci/scripts/symlink-build-dir.sh
144         <<: *step
145
146       - name: disable git crlf conversion
147         run: src/ci/scripts/disable-git-crlf-conversion.sh
148         <<: *step
149
150       - name: install MSYS2
151         run: src/ci/scripts/install-msys2.sh
152         <<: *step
153
154       - name: install MinGW
155         run: src/ci/scripts/install-mingw.sh
156         <<: *step
157
158       - name: install ninja
159         run: src/ci/scripts/install-ninja.sh
160         <<: *step
161
162       - name: enable ipv6 on Docker
163         run: src/ci/scripts/enable-docker-ipv6.sh
164         <<: *step
165
166       # Disable automatic line ending conversion (again). On Windows, when we're
167       # installing dependencies, something switches the git configuration directory or
168       # re-enables autocrlf. We've not tracked down the exact cause -- and there may
169       # be multiple -- but this should ensure submodules are checked out with the
170       # appropriate line endings.
171       - name: disable git crlf conversion
172         run: src/ci/scripts/disable-git-crlf-conversion.sh
173         <<: *step
174
175       - name: checkout submodules
176         run: src/ci/scripts/checkout-submodules.sh
177         <<: *step
178
179       - name: ensure line endings are correct
180         run: src/ci/scripts/verify-line-endings.sh
181         <<: *step
182
183       - name: run the build
184         run: src/ci/scripts/run-build-from-ci.sh
185         env:
186           AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }}
187           AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}
188           TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
189         <<: *step
190
191       - name: upload artifacts to S3
192         run: src/ci/scripts/upload-artifacts.sh
193         env:
194           AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}
195           AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}
196         # Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy
197         # builders *should* have the AWS credentials available. Still, explicitly
198         # adding the condition is helpful as this way CI will not silently skip
199         # deploying artifacts from a dist builder if the variables are misconfigured,
200         # erroring about invalid credentials instead.
201         if: success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')
202         <<: *step
203
204   # These snippets are used by the try-success, try-failure, auto-success and auto-failure jobs.
205   # Check out their documentation for more information on why they're needed.
206
207   - &base-outcome-job
208     name: bors build finished
209     runs-on: ubuntu-latest
210
211   - &base-success-job
212     steps:
213       - name: mark the job as a success
214         run: exit 0
215         shell: bash
216     <<: *base-outcome-job
217
218   - &base-failure-job
219     steps:
220       - name: mark the job as a failure
221         run: exit 1
222         shell: bash
223     <<: *base-outcome-job
224
225 ###########################
226 #   Builders definition   #
227 ###########################
228
229 name: CI
230 on:
231   push:
232     branches:
233       - auto
234       - try
235       - master
236   pull_request:
237     branches:
238       - "**"
239
240 defaults:
241   run:
242     # On Linux, macOS, and Windows, use the system-provided bash as the default
243     # shell. (This should only make a difference on Windows, where the default
244     # shell is PowerShell.)
245     shell: bash
246
247 jobs:
248   pr:
249     <<: *base-ci-job
250     name: PR
251     env:
252       <<: [*shared-ci-variables, *public-variables]
253     if: github.event_name == 'pull_request'
254     strategy:
255       matrix:
256         include:
257           - name: mingw-check
258             <<: *job-linux-xl
259
260           - name: x86_64-gnu-llvm-8
261             <<: *job-linux-xl
262
263           - name: x86_64-gnu-tools
264             env:
265               CI_ONLY_WHEN_SUBMODULES_CHANGED: 1
266             <<: *job-linux-xl
267
268   try:
269     <<: *base-ci-job
270     name: try
271     env:
272       <<: [*shared-ci-variables, *prod-variables]
273     if: github.event_name == 'push' && github.ref == 'refs/heads/try' && github.repository == 'rust-lang-ci/rust'
274     strategy:
275       matrix:
276         include:
277           - name: dist-x86_64-linux
278             <<: *job-linux-xl
279
280   auto:
281     <<: *base-ci-job
282     name: auto
283     env:
284       <<: [*shared-ci-variables, *prod-variables]
285     if: github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'
286     strategy:
287       matrix:
288         include:
289           #############################
290           #   Linux/Docker builders   #
291           #############################
292
293           - name: arm-android
294             <<: *job-linux-xl
295
296           - name: armhf-gnu
297             <<: *job-linux-xl
298
299           - name: dist-aarch64-linux
300             <<: *job-linux-xl
301
302           - name: dist-android
303             <<: *job-linux-xl
304
305           - name: dist-arm-linux
306             <<: *job-linux-xl
307
308           - name: dist-armhf-linux
309             <<: *job-linux-xl
310
311           - name: dist-armv7-linux
312             <<: *job-linux-xl
313
314           - name: dist-i586-gnu-i586-i686-musl
315             <<: *job-linux-xl
316
317           - name: dist-i686-freebsd
318             <<: *job-linux-xl
319
320           - name: dist-i686-linux
321             <<: *job-linux-xl
322
323           - name: dist-mips-linux
324             <<: *job-linux-xl
325
326           - name: dist-mips64-linux
327             <<: *job-linux-xl
328
329           - name: dist-mips64el-linux
330             <<: *job-linux-xl
331
332           - name: dist-mipsel-linux
333             <<: *job-linux-xl
334
335           - name: dist-powerpc-linux
336             <<: *job-linux-xl
337
338           - name: dist-powerpc64-linux
339             <<: *job-linux-xl
340
341           - name: dist-powerpc64le-linux
342             <<: *job-linux-xl
343
344           - name: dist-riscv64-linux
345             <<: *job-linux-xl
346
347           - name: dist-s390x-linux
348             <<: *job-linux-xl
349
350           - name: dist-various-1
351             <<: *job-linux-xl
352
353           - name: dist-various-2
354             <<: *job-linux-xl
355
356           - name: dist-x86_64-freebsd
357             <<: *job-linux-xl
358
359           - name: dist-x86_64-illumos
360             <<: *job-linux-xl
361
362           - name: dist-x86_64-linux
363             <<: *job-linux-xl
364
365           - name: dist-x86_64-linux-alt
366             env:
367               IMAGE: dist-x86_64-linux
368             <<: *job-linux-xl
369
370           - name: dist-x86_64-musl
371             <<: *job-linux-xl
372
373           - name: dist-x86_64-netbsd
374             <<: *job-linux-xl
375
376           - name: i686-gnu
377             <<: *job-linux-xl
378
379           - name: i686-gnu-nopt
380             <<: *job-linux-xl
381
382           - name: mingw-check
383             <<: *job-linux-xl
384
385           - name: test-various
386             <<: *job-linux-xl
387
388           - name: wasm32
389             <<: *job-linux-xl
390
391           - name: x86_64-gnu
392             <<: *job-linux-xl
393
394           - name: x86_64-gnu-aux
395             <<: *job-linux-xl
396
397           - name: x86_64-gnu-debug
398             <<: *job-linux-xl
399
400           - name: x86_64-gnu-distcheck
401             <<: *job-linux-xl
402
403           - name: x86_64-gnu-full-bootstrap
404             <<: *job-linux-xl
405
406           - name: x86_64-gnu-llvm-8
407             env:
408               RUST_BACKTRACE: 1
409             <<: *job-linux-xl
410
411           - name: x86_64-gnu-nopt
412             <<: *job-linux-xl
413
414           - name: x86_64-gnu-tools
415             env:
416               DEPLOY_TOOLSTATES_JSON: toolstates-linux.json
417             <<: *job-linux-xl
418
419           ######################
420           #  Windows Builders  #
421           ######################
422
423           - name: x86_64-msvc-1
424             env:
425               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
426               SCRIPT: make ci-subset-1
427               # FIXME(#59637)
428               NO_DEBUG_ASSERTIONS: 1
429               NO_LLVM_ASSERTIONS: 1
430             <<: *job-windows-xl
431
432           - name: x86_64-msvc-2
433             env:
434               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
435               SCRIPT: make ci-subset-2
436             <<: *job-windows-xl
437
438           - name: i686-msvc-1
439             env:
440               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
441               SCRIPT: make ci-subset-1
442               # FIXME(#59637)
443               NO_DEBUG_ASSERTIONS: 1
444               NO_LLVM_ASSERTIONS: 1
445             <<: *job-windows-xl
446
447           - name: i686-msvc-2
448             env:
449               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
450               SCRIPT: make ci-subset-2
451               # FIXME(#59637)
452               NO_DEBUG_ASSERTIONS: 1
453               NO_LLVM_ASSERTIONS: 1
454             <<: *job-windows-xl
455
456           - name: x86_64-msvc-cargo
457             env:
458               SCRIPT: python x.py test src/tools/cargotest src/tools/cargo
459               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-lld
460               VCVARS_BAT: vcvars64.bat
461               # FIXME(#59637)
462               NO_DEBUG_ASSERTIONS: 1
463               NO_LLVM_ASSERTIONS: 1
464             <<: *job-windows-xl
465
466           - name: x86_64-msvc-tools
467             env:
468               SCRIPT: src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows
469               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstate/toolstates.json
470             <<: *job-windows-xl
471
472           # 32/64-bit MinGW builds.
473           #
474           # We are using MinGW with posix threads since LLVM does not compile with
475           # the win32 threads version due to missing support for C++'s std::thread.
476           #
477           # Instead of relying on the MinGW version installed on appveryor we download
478           # and install one ourselves so we won't be surprised by changes to appveyor's
479           # build image.
480           #
481           # Finally, note that the downloads below are all in the `rust-lang-ci` S3
482           # bucket, but they cleraly didn't originate there! The downloads originally
483           # came from the mingw-w64 SourceForge download site. Unfortunately
484           # SourceForge is notoriously flaky, so we mirror it on our own infrastructure.
485
486           - name: i686-mingw-1
487             env:
488               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
489               SCRIPT: make ci-mingw-subset-1
490               CUSTOM_MINGW: 1
491               # FIXME(#59637)
492               NO_DEBUG_ASSERTIONS: 1
493               NO_LLVM_ASSERTIONS: 1
494             <<: *job-windows-xl
495
496           - name: i686-mingw-2
497             env:
498               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
499               SCRIPT: make ci-mingw-subset-2
500               CUSTOM_MINGW: 1
501             <<: *job-windows-xl
502
503           - name: x86_64-mingw-1
504             env:
505               SCRIPT: make ci-mingw-subset-1
506               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
507               CUSTOM_MINGW: 1
508               # FIXME(#59637)
509               NO_DEBUG_ASSERTIONS: 1
510               NO_LLVM_ASSERTIONS: 1
511             <<: *job-windows-xl
512
513           - name: x86_64-mingw-2
514             env:
515               SCRIPT: make ci-mingw-subset-2
516               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
517               CUSTOM_MINGW: 1
518             <<: *job-windows-xl
519
520           - name: dist-x86_64-msvc
521             env:
522               RUST_CONFIGURE_ARGS: >-
523                 --build=x86_64-pc-windows-msvc
524                 --target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc
525                 --enable-full-tools
526                 --enable-profiler
527               SCRIPT: python x.py dist
528               DIST_REQUIRE_ALL_TOOLS: 1
529             <<: *job-windows-xl
530
531           - name: dist-i686-msvc
532             env:
533               RUST_CONFIGURE_ARGS: >-
534                 --build=i686-pc-windows-msvc
535                 --target=i586-pc-windows-msvc
536                 --enable-full-tools
537                 --enable-profiler
538               SCRIPT: python x.py dist
539               DIST_REQUIRE_ALL_TOOLS: 1
540             <<: *job-windows-xl
541
542           - name: dist-i686-mingw
543             env:
544               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools --enable-profiler
545               SCRIPT: python x.py dist
546               CUSTOM_MINGW: 1
547               DIST_REQUIRE_ALL_TOOLS: 1
548             <<: *job-windows-xl
549
550           - name: dist-x86_64-mingw
551             env:
552               SCRIPT: python x.py dist
553               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler
554               CUSTOM_MINGW: 1
555               DIST_REQUIRE_ALL_TOOLS: 1
556             <<: *job-windows-xl
557
558           - name: dist-x86_64-msvc-alt
559             env:
560               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler
561               SCRIPT: python x.py dist
562             <<: *job-windows-xl
563
564   auto-fallible:
565     <<: *base-ci-job
566     name: auto-fallible
567     env:
568       <<: [*shared-ci-variables, *prod-variables]
569     if: github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'
570     strategy:
571       matrix:
572         include:
573           ####################
574           #  macOS Builders  #
575           ####################
576
577           - name: dist-x86_64-apple
578             env:
579               SCRIPT: ./x.py dist
580               RUST_CONFIGURE_ARGS: --target=aarch64-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc
581               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
582               MACOSX_DEPLOYMENT_TARGET: 10.7
583               NO_LLVM_ASSERTIONS: 1
584               NO_DEBUG_ASSERTIONS: 1
585               DIST_REQUIRE_ALL_TOOLS: 1
586             <<: *job-macos-xl
587
588           - name: dist-x86_64-apple-alt
589             env:
590               SCRIPT: ./x.py dist
591               RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --set rust.jemalloc
592               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
593               MACOSX_DEPLOYMENT_TARGET: 10.7
594               NO_LLVM_ASSERTIONS: 1
595               NO_DEBUG_ASSERTIONS: 1
596             <<: *job-macos-xl
597
598           - name: x86_64-apple
599             env:
600               SCRIPT: ./x.py test
601               RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
602               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
603               MACOSX_DEPLOYMENT_TARGET: 10.8
604               MACOSX_STD_DEPLOYMENT_TARGET: 10.7
605               NO_LLVM_ASSERTIONS: 1
606               NO_DEBUG_ASSERTIONS: 1
607             <<: *job-macos-xl
608
609   master:
610     name: master
611     runs-on: ubuntu-latest
612     env:
613       <<: [*prod-variables]
614     if: github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'rust-lang-ci/rust'
615     steps:
616       - name: checkout the source code
617         uses: actions/checkout@v1
618         with:
619           fetch-depth: 2
620
621       - name: publish toolstate
622         run: src/ci/publish_toolstate.sh
623         shell: bash
624         env:
625           TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
626         <<: *step
627
628   # These jobs don't actually test anything, but they're used to tell bors the
629   # build completed, as there is no practical way to detect when a workflow is
630   # successful listening to webhooks only.
631   try-success:
632     needs: [try]
633     if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/try' && github.repository == 'rust-lang-ci/rust'"
634     <<: *base-success-job
635   try-failure:
636     needs: [try]
637     if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/try' && github.repository == 'rust-lang-ci/rust'"
638     <<: *base-failure-job
639   auto-success:
640     needs: [auto]
641     if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
642     <<: *base-success-job
643   auto-failure:
644     needs: [auto]
645     if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
646     <<: *base-failure-job