]> git.lizzy.rs Git - rust.git/blob - src/ci/github-actions/ci.yml
Rollup merge of #74213 - pickfire:patch-1, r=jonas-schievink
[rust.git] / src / ci / github-actions / ci.yml
1 ######################################################
2 #   WARNING! Action needed when changing this file   #
3 ######################################################
4
5 # Due to GitHub Actions limitations, we can't use YAML Anchors directly in the
6 # CI configuration stored on the repository. To work around that this file is
7 # expanded by a tool in the repository, and the expansion is committed as well.
8 #
9 # After you make any change to the file you'll need to run this command:
10 #
11 #   ./x.py run src/tools/expand-yaml-anchors
12 #
13 # ...and commit the file it updated in addition to this one. If you forget this
14 # step CI will fail.
15
16 ---
17
18 ###############################
19 #   YAML Anchors Definition   #
20 ###############################
21
22 # This key contains most of the YAML anchors that will be used later in the
23 # document. YAML anchors allows us to greatly reduce duplication inside the CI
24 # configuration by reusing parts of the configuration.
25 #
26 # YAML anchors work by defining an anchor with `&anchor-name` and reusing its
27 # content in another place with `*anchor-name`. The special `<<` map key merges
28 # the content of the map with the content of the anchor (or list of anchors).
29 #
30 # The expand-yaml-anchors tool will automatically remove this block from the
31 # output YAML file.
32 x--expand-yaml-anchors--remove:
33
34   - &shared-ci-variables
35     CI_JOB_NAME: ${{ matrix.name }}
36
37   - &public-variables
38     SCCACHE_BUCKET: rust-lang-gha-caches
39     TOOLSTATE_REPO: https://github.com/pietroalbini/rust-toolstate
40     CACHE_DOMAIN: ci-caches-gha.rust-lang.org
41
42   - &prod-variables
43     SCCACHE_BUCKET: rust-lang-gha-caches
44     DEPLOY_BUCKET: rust-lang-gha
45     TOOLSTATE_REPO: https://github.com/pietroalbini/rust-toolstate
46     TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/pietroalbini/rust-toolstate/issues
47     TOOLSTATE_PUBLISH: 1
48     # AWS_SECRET_ACCESS_KEYs are stored in GitHub's secrets storage, named
49     # AWS_SECRET_ACCESS_KEY_<keyid>. Including the key id in the name allows to
50     # rotate them in a single branch while keeping the old key in another
51     # branch, which wouldn't be possible if the key was named with the kind
52     # (caches, artifacts...).
53     CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZOMUQATD5
54     ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZH5AYXDVF
55     CACHE_DOMAIN: ci-caches-gha.rust-lang.org
56
57   - &base-job
58     env: {}
59
60   - &job-linux-xl
61     os: ubuntu-latest-xl
62     <<: *base-job
63
64   - &job-macos-xl
65     os: macos-latest  # We don't have an XL builder for this
66     <<: *base-job
67
68   - &job-windows-xl
69     os: windows-latest-xl
70     <<: *base-job
71
72   - &step
73     if: success() && !env.SKIP_JOB
74
75   - &base-ci-job
76     timeout-minutes: 600
77     runs-on: "${{ matrix.os }}"
78     env: *shared-ci-variables
79     steps:
80       - name: disable git crlf conversion
81         run: git config --global core.autocrlf false
82         shell: bash
83
84       - name: checkout the source code
85         uses: actions/checkout@v1
86         with:
87           fetch-depth: 2
88
89       - name: configure GitHub Actions to kill the build when outdated
90         uses: rust-lang/simpleinfra/github-actions/cancel-outdated-builds@master
91         with:
92           github_token: "${{ secrets.github_token }}"
93         if: success() && !env.SKIP_JOB && github.ref != 'refs/heads/try'
94         <<: *step
95
96       - name: add extra environment variables
97         run: src/ci/scripts/setup-environment.sh
98         env:
99           # Since it's not possible to merge `${{ matrix.env }}` with the other
100           # variables in `job.<name>.env`, the variables defined in the matrix
101           # are passed to the `setup-environment.sh` script encoded in JSON,
102           # which then uses log commands to actually set them.
103           EXTRA_VARIABLES: ${{ toJson(matrix.env) }}
104         <<: *step
105
106       - name: decide whether to skip this job
107         run: src/ci/scripts/should-skip-this.sh
108         <<: *step
109
110       - name: collect CPU statistics
111         run: src/ci/scripts/collect-cpu-stats.sh
112         <<: *step
113
114       - name: show the current environment
115         run: src/ci/scripts/dump-environment.sh
116         <<: *step
117
118       - name: install awscli
119         run: src/ci/scripts/install-awscli.sh
120         <<: *step
121
122       - name: install sccache
123         run: src/ci/scripts/install-sccache.sh
124         <<: *step
125
126       - name: install clang
127         run: src/ci/scripts/install-clang.sh
128         <<: *step
129
130       - name: install WIX
131         run: src/ci/scripts/install-wix.sh
132         <<: *step
133
134       - name: ensure the build happens on a partition with enough space
135         run: src/ci/scripts/symlink-build-dir.sh
136         <<: *step
137
138       - name: disable git crlf conversion
139         run: src/ci/scripts/disable-git-crlf-conversion.sh
140         <<: *step
141
142       - name: install MSYS2
143         run: src/ci/scripts/install-msys2.sh
144         <<: *step
145
146       - name: install MinGW
147         run: src/ci/scripts/install-mingw.sh
148         <<: *step
149
150       - name: install ninja
151         run: src/ci/scripts/install-ninja.sh
152         <<: *step
153
154       - name: enable ipv6 on Docker
155         run: src/ci/scripts/enable-docker-ipv6.sh
156         <<: *step
157
158       # Disable automatic line ending conversion (again). On Windows, when we're
159       # installing dependencies, something switches the git configuration directory or
160       # re-enables autocrlf. We've not tracked down the exact cause -- and there may
161       # be multiple -- but this should ensure submodules are checked out with the
162       # appropriate line endings.
163       - name: disable git crlf conversion
164         run: src/ci/scripts/disable-git-crlf-conversion.sh
165         <<: *step
166
167       - name: checkout submodules
168         run: src/ci/scripts/checkout-submodules.sh
169         <<: *step
170
171       - name: ensure line endings are correct
172         run: src/ci/scripts/verify-line-endings.sh
173         <<: *step
174
175       - name: run the build
176         run: src/ci/scripts/run-build-from-ci.sh
177         env:
178           AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }}
179           AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}
180           TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
181         <<: *step
182
183       - name: upload artifacts to S3
184         run: src/ci/scripts/upload-artifacts.sh
185         env:
186           AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}
187           AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}
188         # Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy
189         # builders *should* have the AWS credentials available. Still, explicitly
190         # adding the condition is helpful as this way CI will not silently skip
191         # deploying artifacts from a dist builder if the variables are misconfigured,
192         # erroring about invalid credentials instead.
193         if: success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')
194         <<: *step
195
196   # These snippets are used by the try-success, try-failure, auto-success and auto-failure jobs.
197   # Check out their documentation for more information on why they're needed.
198
199   - &base-outcome-job
200     name: bors build finished
201     runs-on: ubuntu-latest
202
203   - &base-success-job
204     steps:
205       - name: mark the job as a success
206         run: exit 0
207         shell: bash
208     <<: *base-outcome-job
209
210   - &base-failure-job
211     steps:
212       - name: mark the job as a failure
213         run: exit 1
214         shell: bash
215     <<: *base-outcome-job
216
217 ###########################
218 #   Builders definition   #
219 ###########################
220
221 name: CI
222 on:
223   push:
224     branches:
225       - auto
226       - try
227       - master
228   pull_request:
229     branches:
230       - "**"
231
232 defaults:
233   run:
234     # While on Linux and macOS builders it just forwards the arguments to the
235     # system bash, this wrapper allows switching from the host's bash.exe to
236     # the one we install along with MSYS2 mid-build on Windows.
237     #
238     # Once the step to install MSYS2 is executed, the CI_OVERRIDE_SHELL
239     # environment variable is set pointing to our MSYS2's bash.exe. From that
240     # moment the host's bash.exe will not be called anymore.
241     #
242     # This is needed because we can't launch our own bash.exe from the host
243     # bash.exe, as that would load two different cygwin1.dll in memory, causing
244     # "cygwin heap mismatch" errors.
245     shell: python src/ci/exec-with-shell.py {0}
246
247 jobs:
248   pr:
249     <<: *base-ci-job
250     name: PR
251     env:
252       <<: [*shared-ci-variables, *public-variables]
253     if: github.event_name == 'pull_request'
254     strategy:
255       matrix:
256         include:
257           - name: mingw-check
258             <<: *job-linux-xl
259
260           - name: x86_64-gnu-llvm-8
261             <<: *job-linux-xl
262
263           - name: x86_64-gnu-tools
264             env:
265               CI_ONLY_WHEN_SUBMODULES_CHANGED: 1
266             <<: *job-linux-xl
267
268   try:
269     <<: *base-ci-job
270     name: try
271     env:
272       <<: [*shared-ci-variables, *prod-variables]
273     if: github.event_name == 'push' && github.ref == 'refs/heads/try' && github.repository == 'rust-lang-ci/rust'
274     strategy:
275       matrix:
276         include:
277           - name: dist-x86_64-linux
278             <<: *job-linux-xl
279
280   auto:
281     <<: *base-ci-job
282     name: auto
283     env:
284       <<: [*shared-ci-variables, *prod-variables]
285     if: github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'
286     strategy:
287       matrix:
288         include:
289           #############################
290           #   Linux/Docker builders   #
291           #############################
292
293           - name: arm-android
294             <<: *job-linux-xl
295
296           - name: armhf-gnu
297             <<: *job-linux-xl
298
299           - name: dist-aarch64-linux
300             <<: *job-linux-xl
301
302           - name: dist-android
303             <<: *job-linux-xl
304
305           - name: dist-arm-linux
306             <<: *job-linux-xl
307
308           - name: dist-armhf-linux
309             <<: *job-linux-xl
310
311           - name: dist-armv7-linux
312             <<: *job-linux-xl
313
314           - name: dist-i586-gnu-i586-i686-musl
315             <<: *job-linux-xl
316
317           - name: dist-i686-freebsd
318             <<: *job-linux-xl
319
320           - name: dist-i686-linux
321             <<: *job-linux-xl
322
323           - name: dist-mips-linux
324             <<: *job-linux-xl
325
326           - name: dist-mips64-linux
327             <<: *job-linux-xl
328
329           - name: dist-mips64el-linux
330             <<: *job-linux-xl
331
332           - name: dist-mipsel-linux
333             <<: *job-linux-xl
334
335           - name: dist-powerpc-linux
336             <<: *job-linux-xl
337
338           - name: dist-powerpc64-linux
339             <<: *job-linux-xl
340
341           - name: dist-powerpc64le-linux
342             <<: *job-linux-xl
343
344           - name: dist-s390x-linux
345             <<: *job-linux-xl
346
347           - name: dist-various-1
348             <<: *job-linux-xl
349
350           - name: dist-various-2
351             <<: *job-linux-xl
352
353           - name: dist-x86_64-freebsd
354             <<: *job-linux-xl
355
356           - name: dist-x86_64-linux
357             <<: *job-linux-xl
358
359           - name: dist-x86_64-linux-alt
360             env:
361               IMAGE: dist-x86_64-linux
362             <<: *job-linux-xl
363
364           - name: dist-x86_64-musl
365             <<: *job-linux-xl
366
367           - name: dist-x86_64-netbsd
368             <<: *job-linux-xl
369
370           - name: i686-gnu
371             <<: *job-linux-xl
372
373           - name: i686-gnu-nopt
374             <<: *job-linux-xl
375
376           - name: mingw-check
377             <<: *job-linux-xl
378
379           - name: test-various
380             <<: *job-linux-xl
381
382           - name: wasm32
383             <<: *job-linux-xl
384
385           - name: x86_64-gnu
386             <<: *job-linux-xl
387
388           - name: x86_64-gnu-aux
389             <<: *job-linux-xl
390
391           - name: x86_64-gnu-debug
392             <<: *job-linux-xl
393
394           - name: x86_64-gnu-distcheck
395             <<: *job-linux-xl
396
397           - name: x86_64-gnu-full-bootstrap
398             <<: *job-linux-xl
399
400           - name: x86_64-gnu-llvm-8
401             env:
402               RUST_BACKTRACE: 1
403             <<: *job-linux-xl
404
405           - name: x86_64-gnu-nopt
406             <<: *job-linux-xl
407
408           - name: x86_64-gnu-tools
409             env:
410               DEPLOY_TOOLSTATES_JSON: toolstates-linux.json
411             <<: *job-linux-xl
412
413           ######################
414           #  Windows Builders  #
415           ######################
416
417           - name: x86_64-msvc-1
418             env:
419               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
420               SCRIPT: make ci-subset-1
421               # FIXME(#59637)
422               NO_DEBUG_ASSERTIONS: 1
423               NO_LLVM_ASSERTIONS: 1
424             <<: *job-windows-xl
425
426           - name: x86_64-msvc-2
427             env:
428               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
429               SCRIPT: make ci-subset-2
430             <<: *job-windows-xl
431
432           - name: i686-msvc-1
433             env:
434               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
435               SCRIPT: make ci-subset-1
436               # FIXME(#59637)
437               NO_DEBUG_ASSERTIONS: 1
438               NO_LLVM_ASSERTIONS: 1
439             <<: *job-windows-xl
440
441           - name: i686-msvc-2
442             env:
443               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
444               SCRIPT: make ci-subset-2
445               # FIXME(#59637)
446               NO_DEBUG_ASSERTIONS: 1
447               NO_LLVM_ASSERTIONS: 1
448             <<: *job-windows-xl
449
450           - name: x86_64-msvc-cargo
451             env:
452               SCRIPT: python x.py test src/tools/cargotest src/tools/cargo
453               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-lld
454               VCVARS_BAT: vcvars64.bat
455               # FIXME(#59637)
456               NO_DEBUG_ASSERTIONS: 1
457               NO_LLVM_ASSERTIONS: 1
458             <<: *job-windows-xl
459
460           - name: x86_64-msvc-tools
461             env:
462               SCRIPT: src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows
463               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstate/toolstates.json
464             <<: *job-windows-xl
465
466           # 32/64-bit MinGW builds.
467           #
468           # We are using MinGW with posix threads since LLVM does not compile with
469           # the win32 threads version due to missing support for C++'s std::thread.
470           #
471           # Instead of relying on the MinGW version installed on appveryor we download
472           # and install one ourselves so we won't be surprised by changes to appveyor's
473           # build image.
474           #
475           # Finally, note that the downloads below are all in the `rust-lang-ci` S3
476           # bucket, but they cleraly didn't originate there! The downloads originally
477           # came from the mingw-w64 SourceForge download site. Unfortunately
478           # SourceForge is notoriously flaky, so we mirror it on our own infrastructure.
479
480           - name: i686-mingw-1
481             env:
482               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
483               SCRIPT: make ci-mingw-subset-1
484               CUSTOM_MINGW: 1
485               # FIXME(#59637)
486               NO_DEBUG_ASSERTIONS: 1
487               NO_LLVM_ASSERTIONS: 1
488             <<: *job-windows-xl
489
490           - name: i686-mingw-2
491             env:
492               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
493               SCRIPT: make ci-mingw-subset-2
494               CUSTOM_MINGW: 1
495             <<: *job-windows-xl
496
497           - name: x86_64-mingw-1
498             env:
499               SCRIPT: make ci-mingw-subset-1
500               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
501               CUSTOM_MINGW: 1
502               # FIXME(#59637)
503               NO_DEBUG_ASSERTIONS: 1
504               NO_LLVM_ASSERTIONS: 1
505             <<: *job-windows-xl
506
507           - name: x86_64-mingw-2
508             env:
509               SCRIPT: make ci-mingw-subset-2
510               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
511               CUSTOM_MINGW: 1
512             <<: *job-windows-xl
513
514           - name: dist-x86_64-msvc
515             env:
516               RUST_CONFIGURE_ARGS: >-
517                 --build=x86_64-pc-windows-msvc
518                 --target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc
519                 --enable-full-tools
520                 --enable-profiler
521               SCRIPT: python x.py dist
522               DIST_REQUIRE_ALL_TOOLS: 1
523             <<: *job-windows-xl
524
525           - name: dist-i686-msvc
526             env:
527               RUST_CONFIGURE_ARGS: >-
528                 --build=i686-pc-windows-msvc
529                 --target=i586-pc-windows-msvc
530                 --enable-full-tools
531                 --enable-profiler
532               SCRIPT: python x.py dist
533               DIST_REQUIRE_ALL_TOOLS: 1
534             <<: *job-windows-xl
535
536           - name: dist-i686-mingw
537             env:
538               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools --enable-profiler
539               SCRIPT: python x.py dist
540               CUSTOM_MINGW: 1
541               DIST_REQUIRE_ALL_TOOLS: 1
542             <<: *job-windows-xl
543
544           - name: dist-x86_64-mingw
545             env:
546               SCRIPT: python x.py dist
547               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler
548               CUSTOM_MINGW: 1
549               DIST_REQUIRE_ALL_TOOLS: 1
550             <<: *job-windows-xl
551
552           - name: dist-x86_64-msvc-alt
553             env:
554               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler
555               SCRIPT: python x.py dist
556             <<: *job-windows-xl
557
558   auto-fallible:
559     <<: *base-ci-job
560     name: auto-fallible
561     env:
562       <<: [*shared-ci-variables, *prod-variables]
563     if: github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'
564     strategy:
565       matrix:
566         include:
567           ####################
568           #  macOS Builders  #
569           ####################
570
571           - name: dist-x86_64-apple
572             env:
573               SCRIPT: ./x.py dist
574               RUST_CONFIGURE_ARGS: --target=aarch64-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc
575               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
576               MACOSX_DEPLOYMENT_TARGET: 10.7
577               NO_LLVM_ASSERTIONS: 1
578               NO_DEBUG_ASSERTIONS: 1
579               DIST_REQUIRE_ALL_TOOLS: 1
580             <<: *job-macos-xl
581
582           - name: dist-x86_64-apple-alt
583             env:
584               SCRIPT: ./x.py dist
585               RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --set rust.jemalloc
586               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
587               MACOSX_DEPLOYMENT_TARGET: 10.7
588               NO_LLVM_ASSERTIONS: 1
589               NO_DEBUG_ASSERTIONS: 1
590             <<: *job-macos-xl
591
592           - name: x86_64-apple
593             env:
594               SCRIPT: ./x.py test
595               RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
596               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
597               MACOSX_DEPLOYMENT_TARGET: 10.8
598               MACOSX_STD_DEPLOYMENT_TARGET: 10.7
599               NO_LLVM_ASSERTIONS: 1
600               NO_DEBUG_ASSERTIONS: 1
601             <<: *job-macos-xl
602
603   master:
604     name: master
605     runs-on: ubuntu-latest
606     env:
607       <<: [*prod-variables]
608     if: github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'rust-lang-ci/rust'
609     steps:
610       - name: checkout the source code
611         uses: actions/checkout@v1
612         with:
613           fetch-depth: 2
614
615       - name: publish toolstate
616         run: src/ci/publish_toolstate.sh
617         env:
618           TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
619         <<: *step
620
621   # These jobs don't actually test anything, but they're used to tell bors the
622   # build completed, as there is no practical way to detect when a workflow is
623   # successful listening to webhooks only.
624   try-success:
625     needs: [try]
626     if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/try' && github.repository == 'rust-lang-ci/rust'"
627     <<: *base-success-job
628   try-failure:
629     needs: [try]
630     if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/try' && github.repository == 'rust-lang-ci/rust'"
631     <<: *base-failure-job
632   auto-success:
633     needs: [auto]
634     if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
635     <<: *base-success-job
636   auto-failure:
637     needs: [auto]
638     if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
639     <<: *base-failure-job