]> git.lizzy.rs Git - rust.git/blob - src/ci/github-actions/ci.yml
Auto merge of #69530 - Aaron1011:perf/skip-coerce-var, r=nikomatsakis
[rust.git] / src / ci / github-actions / ci.yml
1 ######################################################
2 #   WARNING! Action needed when changing this file   #
3 ######################################################
4
5 # Due to GitHub Actions limitations, we can't use YAML Anchors directly in the
6 # CI configuration stored on the repository. To work around that this file is
7 # expanded by a tool in the repository, and the expansion is committed as well.
8 #
9 # After you make any change to the file you'll need to run this command:
10 #
11 #   ./x.py run src/tools/expand-yaml-anchors
12 #
13 # ...and commit the file it updated in addition to this one. If you forget this
14 # step CI will fail.
15
16 ---
17
18 ###############################
19 #   YAML Anchors Definition   #
20 ###############################
21
22 # This key contains most of the YAML anchors that will be used later in the
23 # document. YAML anchors allows us to greatly reduce duplication inside the CI
24 # configuration by reusing parts of the configuration.
25 #
26 # YAML anchors work by defining an anchor with `&anchor-name` and reusing its
27 # content in another place with `*anchor-name`. The special `<<` map key merges
28 # the content of the map with the content of the anchor (or list of anchors).
29 #
30 # The expand-yaml-anchors tool will automatically remove this block from the
31 # output YAML file.
32 x--expand-yaml-anchors--remove:
33
34   - &shared-ci-variables
35     CI_JOB_NAME: ${{ matrix.name }}
36
37   - &public-variables
38     SCCACHE_BUCKET: rust-lang-gha-caches
39     TOOLSTATE_REPO: https://github.com/pietroalbini/rust-toolstate
40
41   - &prod-variables
42     SCCACHE_BUCKET: rust-lang-gha-caches
43     DEPLOY_BUCKET: rust-lang-gha
44     TOOLSTATE_REPO: https://github.com/pietroalbini/rust-toolstate
45     TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/pietroalbini/rust-toolstate/issues
46     TOOLSTATE_PUBLISH: 1
47     # AWS_SECRET_ACCESS_KEYs are stored in GitHub's secrets storage, named
48     # AWS_SECRET_ACCESS_KEY_<keyid>. Including the key id in the name allows to
49     # rotate them in a single branch while keeping the old key in another
50     # branch, which wouldn't be possible if the key was named with the kind
51     # (caches, artifacts...).
52     CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZOMUQATD5
53     ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZH5AYXDVF
54
55   - &base-job
56     env: {}
57
58   - &job-linux-xl
59     os: ubuntu-latest-xl
60     <<: *base-job
61
62   - &job-macos-xl
63     os: macos-latest  # We don't have an XL builder for this
64     <<: *base-job
65
66   - &job-windows-xl
67     os: windows-latest-xl
68     <<: *base-job
69
70   - &step
71     if: success() && !env.SKIP_JOB
72
73   - &base-ci-job
74     timeout-minutes: 600
75     runs-on: "${{ matrix.os }}"
76     env: *shared-ci-variables
77     steps:
78       - name: disable git crlf conversion
79         run: git config --global core.autocrlf false
80         shell: bash
81
82       - name: checkout the source code
83         uses: actions/checkout@v1
84         with:
85           fetch-depth: 2
86
87       - name: configure GitHub Actions to kill the build when outdated
88         uses: rust-lang/simpleinfra/github-actions/cancel-outdated-builds@master
89         with:
90           github_token: "${{ secrets.github_token }}"
91         if: success() && !env.SKIP_JOB && github.ref != 'refs/heads/try'
92         <<: *step
93
94       - name: add extra environment variables
95         run: src/ci/scripts/setup-environment.sh
96         env:
97           # Since it's not possible to merge `${{ matrix.env }}` with the other
98           # variables in `job.<name>.env`, the variables defined in the matrix
99           # are passed to the `setup-environment.sh` script encoded in JSON,
100           # which then uses log commands to actually set them.
101           EXTRA_VARIABLES: ${{ toJson(matrix.env) }}
102         <<: *step
103
104       - name: decide whether to skip this job
105         run: src/ci/scripts/should-skip-this.sh
106         <<: *step
107
108       - name: collect CPU statistics
109         run: src/ci/scripts/collect-cpu-stats.sh
110         <<: *step
111
112       - name: show the current environment
113         run: src/ci/scripts/dump-environment.sh
114         <<: *step
115
116       - name: install awscli
117         run: src/ci/scripts/install-awscli.sh
118         <<: *step
119
120       - name: install sccache
121         run: src/ci/scripts/install-sccache.sh
122         <<: *step
123
124       - name: install clang
125         run: src/ci/scripts/install-clang.sh
126         <<: *step
127
128       - name: install WIX
129         run: src/ci/scripts/install-wix.sh
130         <<: *step
131
132       - name: install InnoSetup
133         run: src/ci/scripts/install-innosetup.sh
134         <<: *step
135
136       - name: ensure the build happens on a partition with enough space
137         run: src/ci/scripts/symlink-build-dir.sh
138         <<: *step
139
140       - name: disable git crlf conversion
141         run: src/ci/scripts/disable-git-crlf-conversion.sh
142         <<: *step
143
144       - name: install MSYS2
145         run: src/ci/scripts/install-msys2.sh
146         <<: *step
147
148       - name: install MSYS2 packages
149         run: src/ci/scripts/install-msys2-packages.sh
150         <<: *step
151
152       - name: install MinGW
153         run: src/ci/scripts/install-mingw.sh
154         <<: *step
155
156       - name: install ninja
157         run: src/ci/scripts/install-ninja.sh
158         <<: *step
159
160       - name: enable ipv6 on Docker
161         run: src/ci/scripts/enable-docker-ipv6.sh
162         <<: *step
163
164       # Disable automatic line ending conversion (again). On Windows, when we're
165       # installing dependencies, something switches the git configuration directory or
166       # re-enables autocrlf. We've not tracked down the exact cause -- and there may
167       # be multiple -- but this should ensure submodules are checked out with the
168       # appropriate line endings.
169       - name: disable git crlf conversion
170         run: src/ci/scripts/disable-git-crlf-conversion.sh
171         <<: *step
172
173       - name: checkout submodules
174         run: src/ci/scripts/checkout-submodules.sh
175         <<: *step
176
177       - name: ensure line endings are correct
178         run: src/ci/scripts/verify-line-endings.sh
179         <<: *step
180
181       - name: run the build
182         run: src/ci/scripts/run-build-from-ci.sh
183         env:
184           AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }}
185           AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}
186           TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
187         <<: *step
188
189       - name: upload artifacts to S3
190         run: src/ci/scripts/upload-artifacts.sh
191         env:
192           AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}
193           AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}
194         # Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy
195         # builders *should* have the AWS credentials available. Still, explicitly
196         # adding the condition is helpful as this way CI will not silently skip
197         # deploying artifacts from a dist builder if the variables are misconfigured,
198         # erroring about invalid credentials instead.
199         if: success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')
200         <<: *step
201
202   # These snippets are used by the try-success, try-failure, auto-success and auto-failure jobs.
203   # Check out their documentation for more information on why they're needed.
204
205   - &base-outcome-job
206     name: bors build finished
207     runs-on: ubuntu-latest
208
209   - &base-success-job
210     steps:
211       - name: mark the job as a success
212         run: exit 0
213         shell: bash
214     <<: *base-outcome-job
215
216   - &base-failure-job
217     steps:
218       - name: mark the job as a failure
219         run: exit 1
220         shell: bash
221     <<: *base-outcome-job
222
223 ###########################
224 #   Builders definition   #
225 ###########################
226
227 name: CI
228 on:
229   push:
230     branches:
231       - auto
232       - try
233       - master
234   pull_request:
235     branches:
236       - "**"
237
238 defaults:
239   run:
240     # While on Linux and macOS builders it just forwards the arguments to the
241     # system bash, this wrapper allows switching from the host's bash.exe to
242     # the one we install along with MSYS2 mid-build on Windows.
243     #
244     # Once the step to install MSYS2 is executed, the CI_OVERRIDE_SHELL
245     # environment variable is set pointing to our MSYS2's bash.exe. From that
246     # moment the host's bash.exe will not be called anymore.
247     #
248     # This is needed because we can't launch our own bash.exe from the host
249     # bash.exe, as that would load two different cygwin1.dll in memory, causing
250     # "cygwin heap mismatch" errors.
251     shell: python src/ci/exec-with-shell.py {0}
252
253 jobs:
254   pr:
255     <<: *base-ci-job
256     name: PR
257     env:
258       <<: [*shared-ci-variables, *public-variables]
259     if: github.event_name == 'pull_request'
260     strategy:
261       matrix:
262         include:
263           - name: mingw-check
264             <<: *job-linux-xl
265
266           - name: x86_64-gnu-llvm-8
267             <<: *job-linux-xl
268
269           - name: x86_64-gnu-tools
270             env:
271               CI_ONLY_WHEN_SUBMODULES_CHANGED: 1
272             <<: *job-linux-xl
273
274   try:
275     <<: *base-ci-job
276     name: try
277     env:
278       <<: [*shared-ci-variables, *prod-variables]
279     if: github.event_name == 'push' && github.ref == 'refs/heads/try' && github.repository == 'rust-lang-ci/rust'
280     strategy:
281       matrix:
282         include:
283           - name: dist-x86_64-linux
284             <<: *job-linux-xl
285
286           - name: dist-x86_64-linux-alt
287             env:
288               IMAGE: dist-x86_64-linux
289             <<: *job-linux-xl
290
291   auto:
292     <<: *base-ci-job
293     name: auto
294     env:
295       <<: [*shared-ci-variables, *prod-variables]
296     if: github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'
297     strategy:
298       matrix:
299         include:
300           #############################
301           #   Linux/Docker builders   #
302           #############################
303
304           - name: arm-android
305             <<: *job-linux-xl
306
307           - name: armhf-gnu
308             <<: *job-linux-xl
309
310           - name: dist-aarch64-linux
311             <<: *job-linux-xl
312
313           - name: dist-android
314             <<: *job-linux-xl
315
316           - name: dist-arm-linux
317             <<: *job-linux-xl
318
319           - name: dist-armhf-linux
320             <<: *job-linux-xl
321
322           - name: dist-armv7-linux
323             <<: *job-linux-xl
324
325           - name: dist-i586-gnu-i586-i686-musl
326             <<: *job-linux-xl
327
328           - name: dist-i686-freebsd
329             <<: *job-linux-xl
330
331           - name: dist-i686-linux
332             <<: *job-linux-xl
333
334           - name: dist-mips-linux
335             <<: *job-linux-xl
336
337           - name: dist-mips64-linux
338             <<: *job-linux-xl
339
340           - name: dist-mips64el-linux
341             <<: *job-linux-xl
342
343           - name: dist-mipsel-linux
344             <<: *job-linux-xl
345
346           - name: dist-powerpc-linux
347             <<: *job-linux-xl
348
349           - name: dist-powerpc64-linux
350             <<: *job-linux-xl
351
352           - name: dist-powerpc64le-linux
353             <<: *job-linux-xl
354
355           - name: dist-s390x-linux
356             <<: *job-linux-xl
357
358           - name: dist-various-1
359             <<: *job-linux-xl
360
361           - name: dist-various-2
362             <<: *job-linux-xl
363
364           - name: dist-x86_64-freebsd
365             <<: *job-linux-xl
366
367           - name: dist-x86_64-linux
368             <<: *job-linux-xl
369
370           - name: dist-x86_64-linux-alt
371             env:
372               IMAGE: dist-x86_64-linux
373             <<: *job-linux-xl
374
375           - name: dist-x86_64-musl
376             <<: *job-linux-xl
377
378           - name: dist-x86_64-netbsd
379             <<: *job-linux-xl
380
381           - name: i686-gnu
382             <<: *job-linux-xl
383
384           - name: i686-gnu-nopt
385             <<: *job-linux-xl
386
387           - name: mingw-check
388             <<: *job-linux-xl
389
390           - name: test-various
391             <<: *job-linux-xl
392
393           - name: wasm32
394             <<: *job-linux-xl
395
396           - name: x86_64-gnu
397             <<: *job-linux-xl
398
399           - name: x86_64-gnu-aux
400             <<: *job-linux-xl
401
402           - name: x86_64-gnu-debug
403             <<: *job-linux-xl
404
405           - name: x86_64-gnu-distcheck
406             <<: *job-linux-xl
407
408           - name: x86_64-gnu-full-bootstrap
409             <<: *job-linux-xl
410
411           - name: x86_64-gnu-llvm-8
412             env:
413               RUST_BACKTRACE: 1
414             <<: *job-linux-xl
415
416           - name: x86_64-gnu-nopt
417             <<: *job-linux-xl
418
419           - name: x86_64-gnu-tools
420             env:
421               DEPLOY_TOOLSTATES_JSON: toolstates-linux.json
422             <<: *job-linux-xl
423
424           ####################
425           #  macOS Builders  #
426           ####################
427
428           - name: dist-x86_64-apple
429             env:
430               SCRIPT: ./x.py dist
431               RUST_CONFIGURE_ARGS: --target=aarch64-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc
432               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
433               MACOSX_DEPLOYMENT_TARGET: 10.7
434               NO_LLVM_ASSERTIONS: 1
435               NO_DEBUG_ASSERTIONS: 1
436               DIST_REQUIRE_ALL_TOOLS: 1
437             <<: *job-macos-xl
438
439           - name: dist-x86_64-apple-alt
440             env:
441               SCRIPT: ./x.py dist
442               RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --set rust.jemalloc
443               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
444               MACOSX_DEPLOYMENT_TARGET: 10.7
445               NO_LLVM_ASSERTIONS: 1
446               NO_DEBUG_ASSERTIONS: 1
447             <<: *job-macos-xl
448
449           - name: x86_64-apple
450             env:
451               SCRIPT: ./x.py test
452               RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
453               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
454               MACOSX_DEPLOYMENT_TARGET: 10.8
455               MACOSX_STD_DEPLOYMENT_TARGET: 10.7
456               NO_LLVM_ASSERTIONS: 1
457               NO_DEBUG_ASSERTIONS: 1
458             <<: *job-macos-xl
459
460           ######################
461           #  Windows Builders  #
462           ######################
463
464           - name: x86_64-msvc-1
465             env:
466               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
467               SCRIPT: make ci-subset-1
468               # FIXME(#59637)
469               NO_DEBUG_ASSERTIONS: 1
470               NO_LLVM_ASSERTIONS: 1
471             <<: *job-windows-xl
472
473           - name: x86_64-msvc-2
474             env:
475               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
476               SCRIPT: make ci-subset-2
477             <<: *job-windows-xl
478
479           - name: i686-msvc-1
480             env:
481               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
482               SCRIPT: make ci-subset-1
483               # FIXME(#59637)
484               NO_DEBUG_ASSERTIONS: 1
485               NO_LLVM_ASSERTIONS: 1
486             <<: *job-windows-xl
487
488           - name: i686-msvc-2
489             env:
490               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
491               SCRIPT: make ci-subset-2
492               # FIXME(#59637)
493               NO_DEBUG_ASSERTIONS: 1
494               NO_LLVM_ASSERTIONS: 1
495             <<: *job-windows-xl
496
497           - name: x86_64-msvc-aux
498             env:
499               RUST_CHECK_TARGET: check-aux EXCLUDE_CARGO=1
500               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc
501             <<: *job-windows-xl
502
503           - name: x86_64-msvc-cargo
504             env:
505               SCRIPT: python x.py test src/tools/cargotest src/tools/cargo
506               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc
507               VCVARS_BAT: vcvars64.bat
508               # FIXME(#59637)
509               NO_DEBUG_ASSERTIONS: 1
510               NO_LLVM_ASSERTIONS: 1
511             <<: *job-windows-xl
512
513           - name: x86_64-msvc-tools
514             env:
515               SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows
516               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstate/toolstates.json
517             <<: *job-windows-xl
518
519           # 32/64-bit MinGW builds.
520           #
521           # We are using MinGW with posix threads since LLVM does not compile with
522           # the win32 threads version due to missing support for C++'s std::thread.
523           #
524           # Instead of relying on the MinGW version installed on appveryor we download
525           # and install one ourselves so we won't be surprised by changes to appveyor's
526           # build image.
527           #
528           # Finally, note that the downloads below are all in the `rust-lang-ci` S3
529           # bucket, but they cleraly didn't originate there! The downloads originally
530           # came from the mingw-w64 SourceForge download site. Unfortunately
531           # SourceForge is notoriously flaky, so we mirror it on our own infrastructure.
532
533           - name: i686-mingw-1
534             env:
535               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
536               SCRIPT: make ci-mingw-subset-1
537               CUSTOM_MINGW: 1
538               # FIXME(#59637)
539               NO_DEBUG_ASSERTIONS: 1
540               NO_LLVM_ASSERTIONS: 1
541             <<: *job-windows-xl
542
543           - name: i686-mingw-2
544             env:
545               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
546               SCRIPT: make ci-mingw-subset-2
547               CUSTOM_MINGW: 1
548             <<: *job-windows-xl
549
550           - name: x86_64-mingw-1
551             env:
552               SCRIPT: make ci-mingw-subset-1
553               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
554               CUSTOM_MINGW: 1
555               # FIXME(#59637)
556               NO_DEBUG_ASSERTIONS: 1
557               NO_LLVM_ASSERTIONS: 1
558             <<: *job-windows-xl
559
560           - name: x86_64-mingw-2
561             env:
562               SCRIPT: make ci-mingw-subset-2
563               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
564               CUSTOM_MINGW: 1
565             <<: *job-windows-xl
566
567           - name: dist-x86_64-msvc
568             env:
569               RUST_CONFIGURE_ARGS: >-
570                 --build=x86_64-pc-windows-msvc
571                 --target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc
572                 --enable-full-tools
573                 --enable-profiler
574               SCRIPT: python x.py dist
575               DIST_REQUIRE_ALL_TOOLS: 1
576             <<: *job-windows-xl
577
578           - name: dist-i686-msvc
579             env:
580               RUST_CONFIGURE_ARGS: >-
581                 --build=i686-pc-windows-msvc
582                 --target=i586-pc-windows-msvc
583                 --enable-full-tools
584                 --enable-profiler
585               SCRIPT: python x.py dist
586               DIST_REQUIRE_ALL_TOOLS: 1
587             <<: *job-windows-xl
588
589           - name: dist-i686-mingw
590             env:
591               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools --enable-profiler
592               SCRIPT: python x.py dist
593               CUSTOM_MINGW: 1
594               DIST_REQUIRE_ALL_TOOLS: 1
595             <<: *job-windows-xl
596
597           - name: dist-x86_64-mingw
598             env:
599               SCRIPT: python x.py dist
600               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler
601               CUSTOM_MINGW: 1
602               DIST_REQUIRE_ALL_TOOLS: 1
603             <<: *job-windows-xl
604
605           - name: dist-x86_64-msvc-alt
606             env:
607               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler
608               SCRIPT: python x.py dist
609             <<: *job-windows-xl
610
611   master:
612     name: master
613     runs-on: ubuntu-latest
614     env:
615       <<: [*prod-variables]
616     if: github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'rust-lang-ci/rust'
617     steps:
618       - name: checkout the source code
619         uses: actions/checkout@v1
620         with:
621           fetch-depth: 2
622
623       - name: publish toolstate
624         run: src/ci/publish_toolstate.sh
625         env:
626           TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
627         <<: *step
628
629   # These jobs don't actually test anything, but they're used to tell bors the
630   # build completed, as there is no practical way to detect when a workflow is
631   # successful listening to webhooks only.
632   try-success:
633     needs: [try]
634     if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/try' && github.repository == 'rust-lang-ci/rust'"
635     <<: *base-success-job
636   try-failure:
637     needs: [try]
638     if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/try' && github.repository == 'rust-lang-ci/rust'"
639     <<: *base-failure-job
640   auto-success:
641     needs: [auto]
642     if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
643     <<: *base-success-job
644   auto-failure:
645     needs: [auto]
646     if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
647     <<: *base-failure-job