]> git.lizzy.rs Git - rust.git/blob - src/ci/github-actions/ci.yml
Auto merge of #74565 - pietroalbini:build-on-gha, r=Mark-Simulacrum
[rust.git] / src / ci / github-actions / ci.yml
1 ######################################################
2 #   WARNING! Action needed when changing this file   #
3 ######################################################
4
5 # Due to GitHub Actions limitations, we can't use YAML Anchors directly in the
6 # CI configuration stored on the repository. To work around that this file is
7 # expanded by a tool in the repository, and the expansion is committed as well.
8 #
9 # After you make any change to the file you'll need to run this command:
10 #
11 #   ./x.py run src/tools/expand-yaml-anchors
12 #
13 # ...and commit the file it updated in addition to this one. If you forget this
14 # step CI will fail.
15
16 ---
17
18 ###############################
19 #   YAML Anchors Definition   #
20 ###############################
21
22 # This key contains most of the YAML anchors that will be used later in the
23 # document. YAML anchors allows us to greatly reduce duplication inside the CI
24 # configuration by reusing parts of the configuration.
25 #
26 # YAML anchors work by defining an anchor with `&anchor-name` and reusing its
27 # content in another place with `*anchor-name`. The special `<<` map key merges
28 # the content of the map with the content of the anchor (or list of anchors).
29 #
30 # The expand-yaml-anchors tool will automatically remove this block from the
31 # output YAML file.
32 x--expand-yaml-anchors--remove:
33
34   - &shared-ci-variables
35     CI_JOB_NAME: ${{ matrix.name }}
36
37   - &public-variables
38     SCCACHE_BUCKET: rust-lang-ci-sccache2
39     TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate
40     CACHE_DOMAIN: ci-caches.rust-lang.org
41
42   - &prod-variables
43     SCCACHE_BUCKET: rust-lang-ci-sccache2
44     DEPLOY_BUCKET: rust-lang-ci2
45     TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate
46     TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/rust-lang/rust/issues
47     TOOLSTATE_PUBLISH: 1
48     # AWS_SECRET_ACCESS_KEYs are stored in GitHub's secrets storage, named
49     # AWS_SECRET_ACCESS_KEY_<keyid>. Including the key id in the name allows to
50     # rotate them in a single branch while keeping the old key in another
51     # branch, which wouldn't be possible if the key was named with the kind
52     # (caches, artifacts...).
53     CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL
54     ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55
55     CACHE_DOMAIN: ci-caches.rust-lang.org
56
57   - &dummy-variables
58     SCCACHE_BUCKET: rust-lang-gha-caches
59     DEPLOY_BUCKET: rust-lang-gha
60     TOOLSTATE_REPO: https://github.com/pietroalbini/rust-toolstate
61     TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/pietroalbini/rust-toolstate/issues
62     TOOLSTATE_PUBLISH: 1
63     # AWS_SECRET_ACCESS_KEYs are stored in GitHub's secrets storage, named
64     # AWS_SECRET_ACCESS_KEY_<keyid>. Including the key id in the name allows to
65     # rotate them in a single branch while keeping the old key in another
66     # branch, which wouldn't be possible if the key was named with the kind
67     # (caches, artifacts...).
68     CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZOMUQATD5
69     ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZH5AYXDVF
70     CACHE_DOMAIN: ci-caches-gha.rust-lang.org
71
72   - &base-job
73     env: {}
74
75   - &job-linux-xl
76     os: ubuntu-latest-xl
77     <<: *base-job
78
79   - &job-macos-xl
80     os: macos-latest  # We don't have an XL builder for this
81     <<: *base-job
82
83   - &job-windows-xl
84     os: windows-latest-xl
85     <<: *base-job
86
87   - &step
88     if: success() && !env.SKIP_JOB
89
90   - &base-ci-job
91     timeout-minutes: 600
92     runs-on: "${{ matrix.os }}"
93     env: *shared-ci-variables
94     steps:
95       - name: disable git crlf conversion
96         run: git config --global core.autocrlf false
97
98       - name: checkout the source code
99         uses: actions/checkout@v1
100         with:
101           fetch-depth: 2
102
103       - name: configure GitHub Actions to kill the build when outdated
104         uses: rust-lang/simpleinfra/github-actions/cancel-outdated-builds@master
105         with:
106           github_token: "${{ secrets.github_token }}"
107         if: success() && !env.SKIP_JOB && github.ref != 'refs/heads/try'
108         <<: *step
109
110       # Rust Log Analyzer can't currently detect the PR number of a GitHub
111       # Actions build on its own, so a hint in the log message is needed to
112       # point it in the right direction.
113       - name: configure the PR in which the error message will be posted
114         run: echo "[CI_PR_NUMBER=$num]"
115         env:
116           num: ${{ github.event.number }}
117         if: success() && !env.SKIP_JOBS && github.event_name == 'pull_request'
118
119       - name: add extra environment variables
120         run: src/ci/scripts/setup-environment.sh
121         env:
122           # Since it's not possible to merge `${{ matrix.env }}` with the other
123           # variables in `job.<name>.env`, the variables defined in the matrix
124           # are passed to the `setup-environment.sh` script encoded in JSON,
125           # which then uses log commands to actually set them.
126           EXTRA_VARIABLES: ${{ toJson(matrix.env) }}
127         <<: *step
128
129       - name: decide whether to skip this job
130         run: src/ci/scripts/should-skip-this.sh
131         <<: *step
132
133       - name: collect CPU statistics
134         run: src/ci/scripts/collect-cpu-stats.sh
135         <<: *step
136
137       - name: show the current environment
138         run: src/ci/scripts/dump-environment.sh
139         <<: *step
140
141       - name: install awscli
142         run: src/ci/scripts/install-awscli.sh
143         <<: *step
144
145       - name: install sccache
146         run: src/ci/scripts/install-sccache.sh
147         <<: *step
148
149       - name: install clang
150         run: src/ci/scripts/install-clang.sh
151         <<: *step
152
153       - name: install WIX
154         run: src/ci/scripts/install-wix.sh
155         <<: *step
156
157       - name: ensure the build happens on a partition with enough space
158         run: src/ci/scripts/symlink-build-dir.sh
159         <<: *step
160
161       - name: disable git crlf conversion
162         run: src/ci/scripts/disable-git-crlf-conversion.sh
163         <<: *step
164
165       - name: install MSYS2
166         run: src/ci/scripts/install-msys2.sh
167         <<: *step
168
169       - name: install MinGW
170         run: src/ci/scripts/install-mingw.sh
171         <<: *step
172
173       - name: install ninja
174         run: src/ci/scripts/install-ninja.sh
175         <<: *step
176
177       - name: enable ipv6 on Docker
178         run: src/ci/scripts/enable-docker-ipv6.sh
179         <<: *step
180
181       # Disable automatic line ending conversion (again). On Windows, when we're
182       # installing dependencies, something switches the git configuration directory or
183       # re-enables autocrlf. We've not tracked down the exact cause -- and there may
184       # be multiple -- but this should ensure submodules are checked out with the
185       # appropriate line endings.
186       - name: disable git crlf conversion
187         run: src/ci/scripts/disable-git-crlf-conversion.sh
188         <<: *step
189
190       - name: checkout submodules
191         run: src/ci/scripts/checkout-submodules.sh
192         <<: *step
193
194       - name: ensure line endings are correct
195         run: src/ci/scripts/verify-line-endings.sh
196         <<: *step
197
198       - name: run the build
199         run: src/ci/scripts/run-build-from-ci.sh
200         env:
201           AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }}
202           AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}
203           TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
204         <<: *step
205
206       - name: upload artifacts to S3
207         run: src/ci/scripts/upload-artifacts.sh
208         env:
209           AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}
210           AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}
211         # Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy
212         # builders *should* have the AWS credentials available. Still, explicitly
213         # adding the condition is helpful as this way CI will not silently skip
214         # deploying artifacts from a dist builder if the variables are misconfigured,
215         # erroring about invalid credentials instead.
216         if: success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')
217         <<: *step
218
219   # These snippets are used by the try-success, try-failure, auto-success and auto-failure jobs.
220   # Check out their documentation for more information on why they're needed.
221
222   - &base-outcome-job
223     name: bors build finished
224     runs-on: ubuntu-latest
225
226   - &base-success-job
227     steps:
228       - name: mark the job as a success
229         run: exit 0
230         shell: bash
231     <<: *base-outcome-job
232
233   - &base-failure-job
234     steps:
235       - name: mark the job as a failure
236         run: exit 1
237         shell: bash
238     <<: *base-outcome-job
239
240 ###########################
241 #   Builders definition   #
242 ###########################
243
244 name: CI
245 on:
246   push:
247     branches:
248       - auto
249       - try
250       - master
251   pull_request:
252     branches:
253       - "**"
254
255 defaults:
256   run:
257     # On Linux, macOS, and Windows, use the system-provided bash as the default
258     # shell. (This should only make a difference on Windows, where the default
259     # shell is PowerShell.)
260     shell: bash
261
262 jobs:
263   pr:
264     <<: *base-ci-job
265     name: PR
266     env:
267       <<: [*shared-ci-variables, *public-variables]
268     if: github.event_name == 'pull_request'
269     strategy:
270       matrix:
271         include:
272           - name: mingw-check
273             <<: *job-linux-xl
274
275           - name: x86_64-gnu-llvm-8
276             <<: *job-linux-xl
277
278           - name: x86_64-gnu-tools
279             env:
280               CI_ONLY_WHEN_SUBMODULES_CHANGED: 1
281             <<: *job-linux-xl
282
283   try:
284     <<: *base-ci-job
285     name: try
286     env:
287       <<: [*shared-ci-variables, *prod-variables]
288     if: github.event_name == 'push' && github.ref == 'refs/heads/try' && github.repository == 'rust-lang-ci/rust'
289     strategy:
290       matrix:
291         include:
292           - name: dist-x86_64-linux
293             <<: *job-linux-xl
294
295   auto:
296     <<: *base-ci-job
297     name: auto
298     env:
299       <<: [*shared-ci-variables, *prod-variables]
300     if: github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'
301     strategy:
302       matrix:
303         include:
304           #############################
305           #   Linux/Docker builders   #
306           #############################
307
308           - name: arm-android
309             <<: *job-linux-xl
310
311           - name: armhf-gnu
312             <<: *job-linux-xl
313
314           - name: dist-aarch64-linux
315             <<: *job-linux-xl
316
317           - name: dist-android
318             <<: *job-linux-xl
319
320           - name: dist-arm-linux
321             <<: *job-linux-xl
322
323           - name: dist-armhf-linux
324             <<: *job-linux-xl
325
326           - name: dist-armv7-linux
327             <<: *job-linux-xl
328
329           - name: dist-i586-gnu-i586-i686-musl
330             <<: *job-linux-xl
331
332           - name: dist-i686-freebsd
333             <<: *job-linux-xl
334
335           - name: dist-i686-linux
336             <<: *job-linux-xl
337
338           - name: dist-mips-linux
339             <<: *job-linux-xl
340
341           - name: dist-mips64-linux
342             <<: *job-linux-xl
343
344           - name: dist-mips64el-linux
345             <<: *job-linux-xl
346
347           - name: dist-mipsel-linux
348             <<: *job-linux-xl
349
350           - name: dist-powerpc-linux
351             <<: *job-linux-xl
352
353           - name: dist-powerpc64-linux
354             <<: *job-linux-xl
355
356           - name: dist-powerpc64le-linux
357             <<: *job-linux-xl
358
359           - name: dist-riscv64-linux
360             <<: *job-linux-xl
361
362           - name: dist-s390x-linux
363             <<: *job-linux-xl
364
365           - name: dist-various-1
366             <<: *job-linux-xl
367
368           - name: dist-various-2
369             <<: *job-linux-xl
370
371           - name: dist-x86_64-freebsd
372             <<: *job-linux-xl
373
374           - name: dist-x86_64-illumos
375             <<: *job-linux-xl
376
377           - name: dist-x86_64-linux
378             <<: *job-linux-xl
379
380           - name: dist-x86_64-linux-alt
381             env:
382               IMAGE: dist-x86_64-linux
383             <<: *job-linux-xl
384
385           - name: dist-x86_64-musl
386             <<: *job-linux-xl
387
388           - name: dist-x86_64-netbsd
389             <<: *job-linux-xl
390
391           - name: i686-gnu
392             <<: *job-linux-xl
393
394           - name: i686-gnu-nopt
395             <<: *job-linux-xl
396
397           - name: mingw-check
398             <<: *job-linux-xl
399
400           - name: test-various
401             <<: *job-linux-xl
402
403           - name: wasm32
404             <<: *job-linux-xl
405
406           - name: x86_64-gnu
407             <<: *job-linux-xl
408
409           - name: x86_64-gnu-aux
410             <<: *job-linux-xl
411
412           - name: x86_64-gnu-debug
413             <<: *job-linux-xl
414
415           - name: x86_64-gnu-distcheck
416             <<: *job-linux-xl
417
418           - name: x86_64-gnu-full-bootstrap
419             <<: *job-linux-xl
420
421           - name: x86_64-gnu-llvm-8
422             env:
423               RUST_BACKTRACE: 1
424             <<: *job-linux-xl
425
426           - name: x86_64-gnu-nopt
427             <<: *job-linux-xl
428
429           - name: x86_64-gnu-tools
430             env:
431               DEPLOY_TOOLSTATES_JSON: toolstates-linux.json
432             <<: *job-linux-xl
433
434           ######################
435           #  Windows Builders  #
436           ######################
437
438           - name: x86_64-msvc-1
439             env:
440               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
441               SCRIPT: make ci-subset-1
442               # FIXME(#59637)
443               NO_DEBUG_ASSERTIONS: 1
444               NO_LLVM_ASSERTIONS: 1
445             <<: *job-windows-xl
446
447           - name: x86_64-msvc-2
448             env:
449               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
450               SCRIPT: make ci-subset-2
451             <<: *job-windows-xl
452
453           - name: i686-msvc-1
454             env:
455               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
456               SCRIPT: make ci-subset-1
457               # FIXME(#59637)
458               NO_DEBUG_ASSERTIONS: 1
459               NO_LLVM_ASSERTIONS: 1
460             <<: *job-windows-xl
461
462           - name: i686-msvc-2
463             env:
464               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
465               SCRIPT: make ci-subset-2
466               # FIXME(#59637)
467               NO_DEBUG_ASSERTIONS: 1
468               NO_LLVM_ASSERTIONS: 1
469             <<: *job-windows-xl
470
471           - name: x86_64-msvc-cargo
472             env:
473               SCRIPT: python x.py test src/tools/cargotest src/tools/cargo
474               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-lld
475               VCVARS_BAT: vcvars64.bat
476               # FIXME(#59637)
477               NO_DEBUG_ASSERTIONS: 1
478               NO_LLVM_ASSERTIONS: 1
479             <<: *job-windows-xl
480
481           - name: x86_64-msvc-tools
482             env:
483               SCRIPT: src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows
484               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstate/toolstates.json
485             <<: *job-windows-xl
486
487           # 32/64-bit MinGW builds.
488           #
489           # We are using MinGW with posix threads since LLVM does not compile with
490           # the win32 threads version due to missing support for C++'s std::thread.
491           #
492           # Instead of relying on the MinGW version installed on appveryor we download
493           # and install one ourselves so we won't be surprised by changes to appveyor's
494           # build image.
495           #
496           # Finally, note that the downloads below are all in the `rust-lang-ci` S3
497           # bucket, but they cleraly didn't originate there! The downloads originally
498           # came from the mingw-w64 SourceForge download site. Unfortunately
499           # SourceForge is notoriously flaky, so we mirror it on our own infrastructure.
500
501           - name: i686-mingw-1
502             env:
503               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
504               SCRIPT: make ci-mingw-subset-1
505               CUSTOM_MINGW: 1
506               # FIXME(#59637)
507               NO_DEBUG_ASSERTIONS: 1
508               NO_LLVM_ASSERTIONS: 1
509             <<: *job-windows-xl
510
511           - name: i686-mingw-2
512             env:
513               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
514               SCRIPT: make ci-mingw-subset-2
515               CUSTOM_MINGW: 1
516             <<: *job-windows-xl
517
518           - name: x86_64-mingw-1
519             env:
520               SCRIPT: make ci-mingw-subset-1
521               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
522               CUSTOM_MINGW: 1
523               # FIXME(#59637)
524               NO_DEBUG_ASSERTIONS: 1
525               NO_LLVM_ASSERTIONS: 1
526             <<: *job-windows-xl
527
528           - name: x86_64-mingw-2
529             env:
530               SCRIPT: make ci-mingw-subset-2
531               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
532               CUSTOM_MINGW: 1
533             <<: *job-windows-xl
534
535           - name: dist-x86_64-msvc
536             env:
537               RUST_CONFIGURE_ARGS: >-
538                 --build=x86_64-pc-windows-msvc
539                 --target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc
540                 --enable-full-tools
541                 --enable-profiler
542               SCRIPT: python x.py dist
543               DIST_REQUIRE_ALL_TOOLS: 1
544             <<: *job-windows-xl
545
546           - name: dist-i686-msvc
547             env:
548               RUST_CONFIGURE_ARGS: >-
549                 --build=i686-pc-windows-msvc
550                 --target=i586-pc-windows-msvc
551                 --enable-full-tools
552                 --enable-profiler
553               SCRIPT: python x.py dist
554               DIST_REQUIRE_ALL_TOOLS: 1
555             <<: *job-windows-xl
556
557           - name: dist-i686-mingw
558             env:
559               RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools --enable-profiler
560               SCRIPT: python x.py dist
561               CUSTOM_MINGW: 1
562               DIST_REQUIRE_ALL_TOOLS: 1
563             <<: *job-windows-xl
564
565           - name: dist-x86_64-mingw
566             env:
567               SCRIPT: python x.py dist
568               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler
569               CUSTOM_MINGW: 1
570               DIST_REQUIRE_ALL_TOOLS: 1
571             <<: *job-windows-xl
572
573           - name: dist-x86_64-msvc-alt
574             env:
575               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler
576               SCRIPT: python x.py dist
577             <<: *job-windows-xl
578
579   auto-fallible:
580     <<: *base-ci-job
581     name: auto-fallible
582     env:
583       <<: [*shared-ci-variables, *dummy-variables]
584     if: github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'
585     strategy:
586       matrix:
587         include:
588           ####################
589           #  macOS Builders  #
590           ####################
591
592           - name: dist-x86_64-apple
593             env:
594               SCRIPT: ./x.py dist
595               RUST_CONFIGURE_ARGS: --target=aarch64-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc
596               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
597               MACOSX_DEPLOYMENT_TARGET: 10.7
598               NO_LLVM_ASSERTIONS: 1
599               NO_DEBUG_ASSERTIONS: 1
600               DIST_REQUIRE_ALL_TOOLS: 1
601             <<: *job-macos-xl
602
603           - name: dist-x86_64-apple-alt
604             env:
605               SCRIPT: ./x.py dist
606               RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --set rust.jemalloc
607               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
608               MACOSX_DEPLOYMENT_TARGET: 10.7
609               NO_LLVM_ASSERTIONS: 1
610               NO_DEBUG_ASSERTIONS: 1
611             <<: *job-macos-xl
612
613           - name: x86_64-apple
614             env:
615               SCRIPT: ./x.py test
616               RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
617               RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
618               MACOSX_DEPLOYMENT_TARGET: 10.8
619               MACOSX_STD_DEPLOYMENT_TARGET: 10.7
620               NO_LLVM_ASSERTIONS: 1
621               NO_DEBUG_ASSERTIONS: 1
622             <<: *job-macos-xl
623
624   master:
625     name: master
626     runs-on: ubuntu-latest
627     env:
628       <<: [*prod-variables]
629     if: github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'rust-lang-ci/rust'
630     steps:
631       - name: checkout the source code
632         uses: actions/checkout@v1
633         with:
634           fetch-depth: 2
635
636       - name: publish toolstate
637         run: src/ci/publish_toolstate.sh
638         shell: bash
639         env:
640           TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
641         <<: *step
642
643   # These jobs don't actually test anything, but they're used to tell bors the
644   # build completed, as there is no practical way to detect when a workflow is
645   # successful listening to webhooks only.
646   try-success:
647     needs: [try]
648     if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/try' && github.repository == 'rust-lang-ci/rust'"
649     <<: *base-success-job
650   try-failure:
651     needs: [try]
652     if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/try' && github.repository == 'rust-lang-ci/rust'"
653     <<: *base-failure-job
654   auto-success:
655     needs: [auto]
656     if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
657     <<: *base-success-job
658   auto-failure:
659     needs: [auto]
660     if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
661     <<: *base-failure-job