]> git.lizzy.rs Git - rust.git/blob - src/ci/github-actions/ci.yml
Auto merge of #72091 - RalfJung:miri, r=RalfJung
[rust.git] / src / ci / github-actions / ci.yml
1 ######################################################
2 #   WARNING! Action needed when changing this file   #
3 ######################################################
4
5 # Due to GitHub Actions limitations, we can't use YAML Anchors directly in the
6 # CI configuration stored on the repository. To work around that this file is
7 # expanded by a tool in the repository, and the expansion is committed as well.
8 #
9 # After you make any change to the file you'll need to run this command:
10 #
11 #   ./x.py run src/tools/expand-yaml-anchors
12 #
13 # ...and commit the file it updated in addition to this one. If you forget this
14 # step CI will fail.
15
16 ---
17
18 ###############################
19 #   YAML Anchors Definition   #
20 ###############################
21
22 # This key contains most of the YAML anchors that will be used later in the
23 # document. YAML anchors allows us to greatly reduce duplication inside the CI
24 # configuration by reusing parts of the configuration.
25 #
26 # YAML anchors work by defining an anchor with `&anchor-name` and reusing its
27 # content in another place with `*anchor-name`. The special `<<` map key merges
28 # the content of the map with the content of the anchor (or list of anchors).
29 #
30 # The expand-yaml-anchors tool will automatically remove this block from the
31 # output YAML file.
32 x--expand-yaml-anchors--remove:
33
34   - &shared-ci-variables
35     CI_JOB_NAME: ${{ matrix.name }}
36
37   - &public-variables
38     SCCACHE_BUCKET: rust-lang-gha-caches
39     TOOLSTATE_REPO: https://github.com/pietroalbini/rust-toolstate
40     CACHE_DOMAIN: ci-caches-gha.rust-lang.org
41
42   - &prod-variables
43     SCCACHE_BUCKET: rust-lang-gha-caches
44     DEPLOY_BUCKET: rust-lang-gha
45     TOOLSTATE_REPO: https://github.com/pietroalbini/rust-toolstate
46     TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/pietroalbini/rust-toolstate/issues
47     TOOLSTATE_PUBLISH: 1
48     # AWS_SECRET_ACCESS_KEYs are stored in GitHub's secrets storage, named
49     # AWS_SECRET_ACCESS_KEY_<keyid>. Including the key id in the name allows to
50     # rotate them in a single branch while keeping the old key in another
51     # branch, which wouldn't be possible if the key was named with the kind
52     # (caches, artifacts...).
53     CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZOMUQATD5
54     ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZH5AYXDVF
55     CACHE_DOMAIN: ci-caches-gha.rust-lang.org
56
57   - &base-job
58     env: {}
59
60   - &job-linux-xl
61     os: ubuntu-latest-xl
62     <<: *base-job
63
64   - &job-macos-xl
65     os: macos-latest  # We don't have an XL builder for this
66     <<: *base-job
67
68   - &job-windows-xl
69     os: windows-latest-xl
70     <<: *base-job
71
72   - &step
73     if: success() && !env.SKIP_JOB
74
75   - &base-ci-job
76     timeout-minutes: 600
77     runs-on: "${{ matrix.os }}"
78     env: *shared-ci-variables
79     steps:
80       - name: disable git crlf conversion
81         run: git config --global core.autocrlf false
82         shell: bash
83
84       - name: checkout the source code
85         uses: actions/checkout@v1
86         with:
87           fetch-depth: 2
88
89       - name: configure GitHub Actions to kill the build when outdated
90         uses: rust-lang/simpleinfra/github-actions/cancel-outdated-builds@master
91         with:
92           github_token: "${{ secrets.github_token }}"
93         if: success() && !env.SKIP_JOB && github.ref != 'refs/heads/try'
94         <<: *step
95
96       - name: add extra environment variables
97         run: src/ci/scripts/setup-environment.sh
98         env:
99           # Since it's not possible to merge `${{ matrix.env }}` with the other
100           # variables in `job.<name>.env`, the variables defined in the matrix
101           # are passed to the `setup-environment.sh` script encoded in JSON,
102           # which then uses log commands to actually set them.
103           EXTRA_VARIABLES: ${{ toJson(matrix.env) }}
104         <<: *step
105
106       - name: decide whether to skip this job
107         run: src/ci/scripts/should-skip-this.sh
108         <<: *step
109
110       - name: collect CPU statistics
111         run: src/ci/scripts/collect-cpu-stats.sh
112         <<: *step
113
114       - name: show the current environment
115         run: src/ci/scripts/dump-environment.sh
116         <<: *step
117
118       - name: install awscli
119         run: src/ci/scripts/install-awscli.sh
120         <<: *step
121
122       - name: install sccache
123         run: src/ci/scripts/install-sccache.sh
124         <<: *step
125
126       - name: install clang
127         run: src/ci/scripts/install-clang.sh
128         <<: *step
129
130       - name: install WIX
131         run: src/ci/scripts/install-wix.sh
132         <<: *step
133
134       - name: install InnoSetup
135         run: src/ci/scripts/install-innosetup.sh
136         <<: *step
137
138       - name: ensure the build happens on a partition with enough space
139         run: src/ci/scripts/symlink-build-dir.sh
140         <<: *step
141
142       - name: disable git crlf conversion
143         run: src/ci/scripts/disable-git-crlf-conversion.sh
144         <<: *step
145
146       - name: install MSYS2
147         run: src/ci/scripts/install-msys2.sh
148         <<: *step
149
150       - name: install MSYS2 packages
151         run: src/ci/scripts/install-msys2-packages.sh
152         <<: *step
153
154       - name: install MinGW
155         run: src/ci/scripts/install-mingw.sh
156         <<: *step
157
158       - name: install ninja
159         run: src/ci/scripts/install-ninja.sh
160         <<: *step
161
162       - name: enable ipv6 on Docker
163         run: src/ci/scripts/enable-docker-ipv6.sh
164         <<: *step
165
166       # Disable automatic line ending conversion (again). On Windows, when we're
167       # installing dependencies, something switches the git configuration directory or
168       # re-enables autocrlf. We've not tracked down the exact cause -- and there may
169       # be multiple -- but this should ensure submodules are checked out with the
170       # appropriate line endings.
171       - name: disable git crlf conversion
172         run: src/ci/scripts/disable-git-crlf-conversion.sh
173         <<: *step
174
175       - name: checkout submodules
176         run: src/ci/scripts/checkout-submodules.sh
177         <<: *step
178
179       - name: ensure line endings are correct
180         run: src/ci/scripts/verify-line-endings.sh
181         <<: *step
182
183       - name: run the build
184         run: src/ci/scripts/run-build-from-ci.sh
185         env:
186           AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }}
187           AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}
188           TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
189         <<: *step
190
191       - name: upload artifacts to S3
192         run: src/ci/scripts/upload-artifacts.sh
193         env:
194           AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}
195           AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}
196         # Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy
197         # builders *should* have the AWS credentials available. Still, explicitly
198         # adding the condition is helpful as this way CI will not silently skip
199         # deploying artifacts from a dist builder if the variables are misconfigured,
200         # erroring about invalid credentials instead.
201         if: success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')
202         <<: *step
203
204   # These snippets are used by the try-success, try-failure, auto-success and auto-failure jobs.
205   # Check out their documentation for more information on why they're needed.
206
207   - &base-outcome-job
208     name: bors build finished
209     runs-on: ubuntu-latest
210
211   - &base-success-job
212     steps:
213       - name: mark the job as a success
214         run: exit 0
215         shell: bash
216     <<: *base-outcome-job
217
218   - &base-failure-job
219     steps:
220       - name: mark the job as a failure
221         run: exit 1
222         shell: bash
223     <<: *base-outcome-job
224
225 ###########################
226 #   Builders definition   #
227 ###########################
228
229 name: CI
230 on:
231   push:
232     branches:
233       - auto
234       - try
235       - master
236   pull_request:
237     branches:
238       - "**"
239
240 defaults:
241   run:
242     # While on Linux and macOS builders it just forwards the arguments to the
243     # system bash, this wrapper allows switching from the host's bash.exe to
244     # the one we install along with MSYS2 mid-build on Windows.
245     #
246     # Once the step to install MSYS2 is executed, the CI_OVERRIDE_SHELL
247     # environment variable is set pointing to our MSYS2's bash.exe. From that
248     # moment the host's bash.exe will not be called anymore.
249     #
250     # This is needed because we can't launch our own bash.exe from the host
251     # bash.exe, as that would load two different cygwin1.dll in memory, causing
252     # "cygwin heap mismatch" errors.
253     shell: python src/ci/exec-with-shell.py {0}
254
255 jobs:
256   pr:
257     <<: *base-ci-job
258     name: PR
259     env:
260       <<: [*shared-ci-variables, *public-variables]
261     if: github.event_name == 'pull_request'
262     strategy:
263       matrix:
264         include:
265           - name: mingw-check
266             <<: *job-linux-xl
267
268           - name: x86_64-gnu-llvm-8
269             <<: *job-linux-xl
270
271           - name: x86_64-gnu-tools
272             env:
273               CI_ONLY_WHEN_SUBMODULES_CHANGED: 1
274             <<: *job-linux-xl
275
276   try:
277     <<: *base-ci-job
278     name: try
279     env:
280       <<: [*shared-ci-variables, *prod-variables]
281     if: github.event_name == 'push' && github.ref == 'refs/heads/try' && github.repository == 'rust-lang-ci/rust'
282     strategy:
283       matrix:
284         include:
285           - name: dist-x86_64-linux
286             <<: *job-linux-xl
287
288           - name: dist-x86_64-linux-alt
289             env:
290               IMAGE: dist-x86_64-linux
291             <<: *job-linux-xl
292
293   auto:
294     <<: *base-ci-job
295     name: auto
296     env:
297       <<: [*shared-ci-variables, *prod-variables]
298     if: github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'
299     strategy:
300       matrix:
301         include:
302           #############################
303           #   Linux/Docker builders   #
304           #############################
305
306           - name: arm-android
307             <<: *job-linux-xl
308
309           - name: armhf-gnu
310             <<: *job-linux-xl
311
312           - name: dist-aarch64-linux
313             <<: *job-linux-xl
314
315           - name: dist-android
316             <<: *job-linux-xl
317
318           - name: dist-arm-linux
319             <<: *job-linux-xl
320
321           - name: dist-armhf-linux
322             <<: *job-linux-xl
323
324           - name: dist-armv7-linux
325             <<: *job-linux-xl
326
327           - name: dist-i586-gnu-i586-i686-musl
328             <<: *job-linux-xl
329
330           - name: dist-i686-freebsd
331             <<: *job-linux-xl
332
333           - name: dist-i686-linux
334             <<: *job-linux-xl
335
336           - name: dist-mips-linux
337             <<: *job-linux-xl
338
339           - name: dist-mips64-linux
340             <<: *job-linux-xl
341
342           - name: dist-mips64el-linux
343             <<: *job-linux-xl
344
345           - name: dist-mipsel-linux
346             <<: *job-linux-xl
347
348           - name: dist-powerpc-linux
349             <<: *job-linux-xl
350
351           - name: dist-powerpc64-linux
352             <<: *job-linux-xl
353
354           - name: dist-powerpc64le-linux
355             <<: *job-linux-xl
356
357           - name: dist-s390x-linux
358             <<: *job-linux-xl
359
360           - name: dist-various-1
361             <<: *job-linux-xl
362
363           - name: dist-various-2
364             <<: *job-linux-xl
365
366           - name: dist-x86_64-freebsd
367             <<: *job-linux-xl
368
369           - name: dist-x86_64-linux
370             <<: *job-linux-xl
371
372           - name: dist-x86_64-linux-alt
373             env:
374               IMAGE: dist-x86_64-linux
375             <<: *job-linux-xl
376
377           - name: dist-x86_64-musl
378             <<: *job-linux-xl
379
380           - name: dist-x86_64-netbsd
381             <<: *job-linux-xl
382
383           - name: i686-gnu
384             <<: *job-linux-xl
385
386           - name: i686-gnu-nopt
387             <<: *job-linux-xl
388
389           - name: mingw-check
390             <<: *job-linux-xl
391
392           - name: test-various
393             <<: *job-linux-xl
394
395           - name: wasm32
396             <<: *job-linux-xl
397
398           - name: x86_64-gnu
399             <<: *job-linux-xl
400
401           - name: x86_64-gnu-aux
402             <<: *job-linux-xl
403
404           - name: x86_64-gnu-debug
405             <<: *job-linux-xl
406
407           - name: x86_64-gnu-distcheck
408             <<: *job-linux-xl
409
410           - name: x86_64-gnu-full-bootstrap
411             <<: *job-linux-xl
412
413           - name: x86_64-gnu-llvm-8
414             env:
415               RUST_BACKTRACE: 1
416             <<: *job-linux-xl
417
418           - name: x86_64-gnu-nopt
419             <<: *job-linux-xl
420
421           - name: x86_64-gnu-tools
422             env:
423               DEPLOY_TOOLSTATES_JSON: toolstates-linux.json
424             <<: *job-linux-xl
425
426           ####################
427           #  macOS Builders  #
428           ####################
429
430           - name: dist-x86_64-apple
431             env:
432               SCRIPT: ./x.py dist
433               RUST_CONFIGURE_ARGS: --target=aarch64-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc
434               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
435               MACOSX_DEPLOYMENT_TARGET: 10.7
436               NO_LLVM_ASSERTIONS: 1
437               NO_DEBUG_ASSERTIONS: 1
438               DIST_REQUIRE_ALL_TOOLS: 1
439             <<: *job-macos-xl
440
441           - name: dist-x86_64-apple-alt
442             env:
443               SCRIPT: ./x.py dist
444               RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --set rust.jemalloc
445               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
446               MACOSX_DEPLOYMENT_TARGET: 10.7
447               NO_LLVM_ASSERTIONS: 1
448               NO_DEBUG_ASSERTIONS: 1
449             <<: *job-macos-xl
450
451           - name: x86_64-apple
452             env:
453               SCRIPT: ./x.py test
454               RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
455               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
456               MACOSX_DEPLOYMENT_TARGET: 10.8
457               MACOSX_STD_DEPLOYMENT_TARGET: 10.7
458               NO_LLVM_ASSERTIONS: 1
459               NO_DEBUG_ASSERTIONS: 1
460             <<: *job-macos-xl
461
462           ######################
463           #  Windows Builders  #
464           ######################
465
466           - name: x86_64-msvc-1
467             env:
468               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
469               SCRIPT: make ci-subset-1
470               # FIXME(#59637)
471               NO_DEBUG_ASSERTIONS: 1
472               NO_LLVM_ASSERTIONS: 1
473             <<: *job-windows-xl
474
475           - name: x86_64-msvc-2
476             env:
477               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
478               SCRIPT: make ci-subset-2
479             <<: *job-windows-xl
480
481           - name: i686-msvc-1
482             env:
483               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
484               SCRIPT: make ci-subset-1
485               # FIXME(#59637)
486               NO_DEBUG_ASSERTIONS: 1
487               NO_LLVM_ASSERTIONS: 1
488             <<: *job-windows-xl
489
490           - name: i686-msvc-2
491             env:
492               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
493               SCRIPT: make ci-subset-2
494               # FIXME(#59637)
495               NO_DEBUG_ASSERTIONS: 1
496               NO_LLVM_ASSERTIONS: 1
497             <<: *job-windows-xl
498
499           - name: x86_64-msvc-aux
500             env:
501               RUST_CHECK_TARGET: check-aux EXCLUDE_CARGO=1
502               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc
503             <<: *job-windows-xl
504
505           - name: x86_64-msvc-cargo
506             env:
507               SCRIPT: python x.py test src/tools/cargotest src/tools/cargo
508               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc
509               VCVARS_BAT: vcvars64.bat
510               # FIXME(#59637)
511               NO_DEBUG_ASSERTIONS: 1
512               NO_LLVM_ASSERTIONS: 1
513             <<: *job-windows-xl
514
515           - name: x86_64-msvc-tools
516             env:
517               SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows
518               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstate/toolstates.json
519             <<: *job-windows-xl
520
521           # 32/64-bit MinGW builds.
522           #
523           # We are using MinGW with posix threads since LLVM does not compile with
524           # the win32 threads version due to missing support for C++'s std::thread.
525           #
526           # Instead of relying on the MinGW version installed on appveryor we download
527           # and install one ourselves so we won't be surprised by changes to appveyor's
528           # build image.
529           #
530           # Finally, note that the downloads below are all in the `rust-lang-ci` S3
531           # bucket, but they cleraly didn't originate there! The downloads originally
532           # came from the mingw-w64 SourceForge download site. Unfortunately
533           # SourceForge is notoriously flaky, so we mirror it on our own infrastructure.
534
535           - name: i686-mingw-1
536             env:
537               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
538               SCRIPT: make ci-mingw-subset-1
539               CUSTOM_MINGW: 1
540               # FIXME(#59637)
541               NO_DEBUG_ASSERTIONS: 1
542               NO_LLVM_ASSERTIONS: 1
543             <<: *job-windows-xl
544
545           - name: i686-mingw-2
546             env:
547               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
548               SCRIPT: make ci-mingw-subset-2
549               CUSTOM_MINGW: 1
550             <<: *job-windows-xl
551
552           - name: x86_64-mingw-1
553             env:
554               SCRIPT: make ci-mingw-subset-1
555               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
556               CUSTOM_MINGW: 1
557               # FIXME(#59637)
558               NO_DEBUG_ASSERTIONS: 1
559               NO_LLVM_ASSERTIONS: 1
560             <<: *job-windows-xl
561
562           - name: x86_64-mingw-2
563             env:
564               SCRIPT: make ci-mingw-subset-2
565               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
566               CUSTOM_MINGW: 1
567             <<: *job-windows-xl
568
569           - name: dist-x86_64-msvc
570             env:
571               RUST_CONFIGURE_ARGS: >-
572                 --build=x86_64-pc-windows-msvc
573                 --target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc
574                 --enable-full-tools
575                 --enable-profiler
576               SCRIPT: python x.py dist
577               DIST_REQUIRE_ALL_TOOLS: 1
578             <<: *job-windows-xl
579
580           - name: dist-i686-msvc
581             env:
582               RUST_CONFIGURE_ARGS: >-
583                 --build=i686-pc-windows-msvc
584                 --target=i586-pc-windows-msvc
585                 --enable-full-tools
586                 --enable-profiler
587               SCRIPT: python x.py dist
588               DIST_REQUIRE_ALL_TOOLS: 1
589             <<: *job-windows-xl
590
591           - name: dist-i686-mingw
592             env:
593               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools --enable-profiler
594               SCRIPT: python x.py dist
595               CUSTOM_MINGW: 1
596               DIST_REQUIRE_ALL_TOOLS: 1
597             <<: *job-windows-xl
598
599           - name: dist-x86_64-mingw
600             env:
601               SCRIPT: python x.py dist
602               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler
603               CUSTOM_MINGW: 1
604               DIST_REQUIRE_ALL_TOOLS: 1
605             <<: *job-windows-xl
606
607           - name: dist-x86_64-msvc-alt
608             env:
609               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler
610               SCRIPT: python x.py dist
611             <<: *job-windows-xl
612
613   master:
614     name: master
615     runs-on: ubuntu-latest
616     env:
617       <<: [*prod-variables]
618     if: github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'rust-lang-ci/rust'
619     steps:
620       - name: checkout the source code
621         uses: actions/checkout@v1
622         with:
623           fetch-depth: 2
624
625       - name: publish toolstate
626         run: src/ci/publish_toolstate.sh
627         env:
628           TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
629         <<: *step
630
631   # These jobs don't actually test anything, but they're used to tell bors the
632   # build completed, as there is no practical way to detect when a workflow is
633   # successful listening to webhooks only.
634   try-success:
635     needs: [try]
636     if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/try' && github.repository == 'rust-lang-ci/rust'"
637     <<: *base-success-job
638   try-failure:
639     needs: [try]
640     if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/try' && github.repository == 'rust-lang-ci/rust'"
641     <<: *base-failure-job
642   auto-success:
643     needs: [auto]
644     if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
645     <<: *base-success-job
646   auto-failure:
647     needs: [auto]
648     if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
649     <<: *base-failure-job