1 # Copyright 2012 The Rust Project Developers. See the COPYRIGHT
2 # file at the top-level directory of this distribution and at
3 # http://rust-lang.org/COPYRIGHT.
5 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 # http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 # <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 # option. This file may not be copied, modified, or distributed
9 # except according to those terms.
12 ######################################################################
14 ######################################################################
16 # The names of crates that must be tested
17 TEST_TARGET_CRATES = std extra rustuv
18 TEST_HOST_CRATES = rustpkg rustc rustdoc syntax
19 TEST_CRATES = $(TEST_TARGET_CRATES) $(TEST_HOST_CRATES)
21 # Markdown files under doc/ that should have their code extracted and run
22 DOC_TEST_NAMES = tutorial tutorial-ffi tutorial-macros tutorial-borrowed-ptr \
23 tutorial-tasks tutorial-conditions tutorial-container rust
25 ######################################################################
26 # Environment configuration
27 ######################################################################
29 # The arguments to all test runners
31 TESTARGS += $(TESTNAME)
40 # Arguments to the cfail/rfail/rpass/bench tests
42 CTEST_RUNTOOL = --runtool "$(CFG_VALGRIND)"
50 # Arguments to the perf tests
52 CTEST_PERF_RUNTOOL = --runtool "$(CFG_PERF_TOOL)"
55 CTEST_TESTARGS := $(TESTARGS)
58 CTEST_TESTARGS += --verbose
61 # If we're running perf then set this environment variable
62 # to put the benchmarks into 'hard mode'
63 ifeq ($(MAKECMDGOALS),perf)
68 TEST_LOG_FILE=tmp/check-stage$(1)-T-$(2)-H-$(3)-$(4).log
69 TEST_OK_FILE=tmp/check-stage$(1)-T-$(2)-H-$(3)-$(4).ok
71 TEST_RATCHET_FILE=tmp/check-stage$(1)-T-$(2)-H-$(3)-$(4)-metrics.json
72 TEST_RATCHET_NOISE_PERCENT=10.0
74 # Whether to ratchet or merely save benchmarks
75 ifdef CFG_RATCHET_BENCH
76 CRATE_TEST_EXTRA_ARGS=\
77 --test $(TEST_BENCH) \
78 --ratchet-metrics $(call TEST_RATCHET_FILE,$(1),$(2),$(3),$(4)) \
79 --ratchet-noise-percent $(TEST_RATCHET_NOISE_PERCENT)
81 CRATE_TEST_EXTRA_ARGS=\
82 --test $(TEST_BENCH) \
83 --save-metrics $(call TEST_RATCHET_FILE,$(1),$(2),$(3),$(4))
86 # If we're sharding the testsuite between parallel testers,
87 # pass this argument along to the compiletest and crate test
90 CTEST_TESTARGS += --test-shard=$(TEST_SHARD)
91 CRATE_TEST_EXTRA_ARGS += --test-shard=$(TEST_SHARD)
94 define DEF_TARGET_COMMANDS
97 CFG_RUN_TEST_$(1)=$$(call CFG_RUN_$(1),,$$(CFG_VALGRIND) $$(1))
100 ifdef CFG_WINDOWSY_$(1)
101 CFG_TESTLIB_$(1)=$$(CFG_BUILD_DIR)/$$(2)/$$(strip \
102 $$(if $$(findstring stage0,$$(1)), \
103 stage0/$$(CFG_LIBDIR), \
104 $$(if $$(findstring stage1,$$(1)), \
105 stage1/$$(CFG_LIBDIR), \
106 $$(if $$(findstring stage2,$$(1)), \
107 stage2/$$(CFG_LIBDIR), \
108 $$(if $$(findstring stage3,$$(1)), \
109 stage3/$$(CFG_LIBDIR), \
110 )))))/rustc/$$(CFG_BUILD)/$$(CFG_LIBDIR)
111 CFG_RUN_TEST_$(1)=$$(call CFG_RUN_$(1),$$(call CFG_TESTLIB_$(1),$$(1),$$(3)),$$(1))
114 # Run the compiletest runner itself under valgrind
116 CFG_RUN_CTEST_$(1)=$$(call CFG_RUN_TEST_$$(CFG_BUILD),$$(2),$$(3))
118 CFG_RUN_CTEST_$(1)=$$(call CFG_RUN_$$(CFG_BUILD),$$(TLIB$$(1)_T_$$(3)_H_$$(3)),$$(2))
123 $(foreach target,$(CFG_TARGET), \
124 $(eval $(call DEF_TARGET_COMMANDS,$(target))))
126 # Target platform specific variables
127 # for arm-linux-androidabi
128 define DEF_ADB_DEVICE_STATUS
129 CFG_ADB_DEVICE_STATUS=$(1)
132 $(foreach target,$(CFG_TARGET), \
133 $(if $(findstring $(target),"arm-linux-androideabi"), \
134 $(if $(findstring adb,$(CFG_ADB)), \
135 $(if $(findstring device,$(shell $(CFG_ADB) devices 2>/dev/null | grep -E '^[_A-Za-z0-9-]+[[:blank:]]+device')), \
136 $(info check: $(target) test enabled \
137 $(info check: android device attached) \
138 $(eval $(call DEF_ADB_DEVICE_STATUS, true))), \
139 $(info check: $(target) test disabled \
140 $(info check: android device not attached) \
141 $(eval $(call DEF_ADB_DEVICE_STATUS, false))) \
143 $(info check: $(target) test disabled \
144 $(info check: adb not found) \
145 $(eval $(call DEF_ADB_DEVICE_STATUS, false))) \
150 ifeq ($(CFG_ADB_DEVICE_STATUS),true)
151 CFG_ADB_TEST_DIR=/data/tmp
153 $(info check: android device test dir $(CFG_ADB_TEST_DIR) ready \
154 $(shell adb remount 1>/dev/null) \
155 $(shell adb shell rm -r $(CFG_ADB_TEST_DIR) >/dev/null) \
156 $(shell adb shell mkdir $(CFG_ADB_TEST_DIR)) \
157 $(shell adb shell mkdir $(CFG_ADB_TEST_DIR)/tmp) \
158 $(shell adb push $(S)src/etc/adb_run_wrapper.sh $(CFG_ADB_TEST_DIR) 1>/dev/null) \
159 $(shell adb push $(CFG_ANDROID_CROSS_PATH)/arm-linux-androideabi/lib/armv7-a/libgnustl_shared.so \
160 $(CFG_ADB_TEST_DIR) 1>/dev/null) \
161 $(shell adb push $(TLIB2_T_arm-linux-androideabi_H_$(CFG_BUILD))/$(CFG_RUNTIME_arm-linux-androideabi) \
162 $(CFG_ADB_TEST_DIR)) \
163 $(shell adb push $(TLIB2_T_arm-linux-androideabi_H_$(CFG_BUILD))/$(STDLIB_GLOB_arm-linux-androideabi) \
164 $(CFG_ADB_TEST_DIR)) \
165 $(shell adb push $(TLIB2_T_arm-linux-androideabi_H_$(CFG_BUILD))/$(EXTRALIB_GLOB_arm-linux-androideabi) \
166 $(CFG_ADB_TEST_DIR)) \
167 $(shell adb push $(TLIB2_T_arm-linux-androideabi_H_$(CFG_BUILD_TRIPLE))/$(LIBRUSTUV_GLOB_arm-linux-androideabi) \
168 $(CFG_ADB_TEST_DIR)) \
175 ######################################################################
177 ######################################################################
179 check: cleantestlibs cleantmptestlogs tidy all check-stage2
180 $(Q)$(CFG_PYTHON) $(S)src/etc/check-summary.py tmp/*.log
182 check-notidy: cleantestlibs cleantmptestlogs all check-stage2
183 $(Q)$(CFG_PYTHON) $(S)src/etc/check-summary.py tmp/*.log
185 check-full: cleantestlibs cleantmptestlogs tidy \
186 all check-stage1 check-stage2 check-stage3
187 $(Q)$(CFG_PYTHON) $(S)src/etc/check-summary.py tmp/*.log
189 check-test: cleantestlibs cleantmptestlogs all check-stage2-rfail
190 $(Q)$(CFG_PYTHON) $(S)src/etc/check-summary.py tmp/*.log
192 check-lite: cleantestlibs cleantmptestlogs \
193 check-stage2-std check-stage2-extra check-stage2-rpass \
194 check-stage2-rustuv \
195 check-stage2-rustpkg \
196 check-stage2-rfail check-stage2-cfail
197 $(Q)$(CFG_PYTHON) $(S)src/etc/check-summary.py tmp/*.log
199 .PHONY: cleantmptestlogs cleantestlibs
205 $(Q)find $(CFG_BUILD)/test \
206 -name '*.[odasS]' -o \
213 -name '*.libaux' -o \
216 -name '*.debugger.script' \
220 ######################################################################
222 ######################################################################
228 ALL_CS := $(wildcard $(S)src/rt/*.cpp \
230 $(S)src/rt/*/*/*.cpp \
231 $(S)src/rustllvm/*.cpp)
232 ALL_CS := $(filter-out $(S)src/rt/miniz.cpp \
233 $(wildcard $(S)src/rt/sundown/src/*.c) \
234 $(wildcard $(S)src/rt/sundown/html/*.c) \
236 ALL_HS := $(wildcard $(S)src/rt/*.h \
239 $(S)src/rustllvm/*.h)
240 ALL_HS := $(filter-out $(S)src/rt/vg/valgrind.h \
241 $(S)src/rt/vg/memcheck.h \
242 $(S)src/rt/msvc/typeof.h \
243 $(S)src/rt/msvc/stdint.h \
244 $(S)src/rt/msvc/inttypes.h \
245 $(wildcard $(S)src/rt/sundown/src/*.h) \
246 $(wildcard $(S)src/rt/sundown/html/*.h) \
249 # Run the tidy script in multiple parts to avoid huge 'echo' commands
251 @$(call E, check: formatting)
252 $(Q)find $(S)src -name '*.r[sc]' \
253 | grep '^$(S)src/test' -v \
254 | grep '^$(S)src/libuv' -v \
255 | grep '^$(S)src/llvm' -v \
256 | grep '^$(S)src/gyp' -v \
257 | xargs -n 10 $(CFG_PYTHON) $(S)src/etc/tidy.py
258 $(Q)find $(S)src/etc -name '*.py' \
259 | xargs -n 10 $(CFG_PYTHON) $(S)src/etc/tidy.py
261 | xargs -n 10 $(CFG_PYTHON) $(S)src/etc/tidy.py
263 | xargs -n 10 $(CFG_PYTHON) $(S)src/etc/tidy.py
264 $(Q)find $(S)src -type f -perm +111 \
265 -not -name '*.rs' -and -not -name '*.py' \
266 -and -not -name '*.sh' \
267 | grep '^$(S)src/llvm' -v \
268 | grep '^$(S)src/libuv' -v \
269 | grep '^$(S)src/gyp' -v \
270 | grep '^$(S)src/etc' -v \
271 | xargs $(CFG_PYTHON) $(S)src/etc/check-binaries.py
276 ######################################################################
278 ######################################################################
282 check-stage$(1)-T-$(2)-H-$(3)-exec: \
283 check-stage$(1)-T-$(2)-H-$(3)-rpass-exec \
284 check-stage$(1)-T-$(2)-H-$(3)-rfail-exec \
285 check-stage$(1)-T-$(2)-H-$(3)-cfail-exec \
286 check-stage$(1)-T-$(2)-H-$(3)-rpass-full-exec \
287 check-stage$(1)-T-$(2)-H-$(3)-crates-exec \
288 check-stage$(1)-T-$(2)-H-$(3)-bench-exec \
289 check-stage$(1)-T-$(2)-H-$(3)-debuginfo-exec \
290 check-stage$(1)-T-$(2)-H-$(3)-codegen-exec \
291 check-stage$(1)-T-$(2)-H-$(3)-doc-exec \
292 check-stage$(1)-T-$(2)-H-$(3)-pretty-exec
294 # Only test the compiler-dependent crates when the target is
295 # able to build a compiler (when the target triple is in the set of host triples)
296 ifneq ($$(findstring $(2),$$(CFG_HOST)),)
298 check-stage$(1)-T-$(2)-H-$(3)-crates-exec: \
299 $$(foreach crate,$$(TEST_CRATES), \
300 check-stage$(1)-T-$(2)-H-$(3)-$$(crate)-exec)
304 check-stage$(1)-T-$(2)-H-$(3)-crates-exec: \
305 $$(foreach crate,$$(TEST_TARGET_CRATES), \
306 check-stage$(1)-T-$(2)-H-$(3)-$$(crate)-exec)
310 check-stage$(1)-T-$(2)-H-$(3)-doc-exec: \
311 $$(foreach docname,$$(DOC_TEST_NAMES), \
312 check-stage$(1)-T-$(2)-H-$(3)-doc-$$(docname)-exec)
314 check-stage$(1)-T-$(2)-H-$(3)-pretty-exec: \
315 check-stage$(1)-T-$(2)-H-$(3)-pretty-rpass-exec \
316 check-stage$(1)-T-$(2)-H-$(3)-pretty-rpass-full-exec \
317 check-stage$(1)-T-$(2)-H-$(3)-pretty-rfail-exec \
318 check-stage$(1)-T-$(2)-H-$(3)-pretty-bench-exec \
319 check-stage$(1)-T-$(2)-H-$(3)-pretty-pretty-exec
323 $(foreach host,$(CFG_HOST), \
324 $(foreach target,$(CFG_TARGET), \
325 $(foreach stage,$(STAGES), \
326 $(eval $(call DEF_TEST_SETS,$(stage),$(target),$(host))))))
329 ######################################################################
331 ######################################################################
335 # If NO_REBUILD is set then break the dependencies on extra so we can
336 # test crates without rebuilding std and extra first
337 ifeq ($(NO_REBUILD),)
338 STDTESTDEP_$(1)_$(2)_$(3) = $$(SREQ$(1)_T_$(2)_H_$(3)) \
339 $$(TLIB$(1)_T_$(2)_H_$(3))/$$(CFG_EXTRALIB_$(2)) \
340 $$(TLIB$(1)_T_$(2)_H_$(3))/$$(CFG_LIBRUSTUV_$(2))
342 STDTESTDEP_$(1)_$(2)_$(3) =
345 $(3)/stage$(1)/test/stdtest-$(2)$$(X_$(2)): \
346 $$(STDLIB_CRATE) $$(STDLIB_INPUTS) \
347 $$(STDTESTDEP_$(1)_$(2)_$(3))
348 @$$(call E, compile_and_link: $$@)
349 $$(STAGE$(1)_T_$(2)_H_$(3)) -o $$@ $$< --test
351 $(3)/stage$(1)/test/extratest-$(2)$$(X_$(2)): \
352 $$(EXTRALIB_CRATE) $$(EXTRALIB_INPUTS) \
353 $$(STDTESTDEP_$(1)_$(2)_$(3))
354 @$$(call E, compile_and_link: $$@)
355 $$(STAGE$(1)_T_$(2)_H_$(3)) -o $$@ $$< --test
357 $(3)/stage$(1)/test/rustuvtest-$(2)$$(X_$(2)): \
358 $$(LIBRUSTUV_CRATE) $$(LIBRUSTUV_INPUTS) \
359 $$(STDTESTDEP_$(1)_$(2)_$(3))
360 @$$(call E, compile_and_link: $$@)
361 $$(STAGE$(1)_T_$(2)_H_$(3)) -o $$@ $$< --test
363 $(3)/stage$(1)/test/syntaxtest-$(2)$$(X_$(2)): \
364 $$(LIBSYNTAX_CRATE) $$(LIBSYNTAX_INPUTS) \
365 $$(STDTESTDEP_$(1)_$(2)_$(3))
366 @$$(call E, compile_and_link: $$@)
367 $$(STAGE$(1)_T_$(2)_H_$(3)) -o $$@ $$< --test
369 $(3)/stage$(1)/test/rustctest-$(2)$$(X_$(2)): CFG_COMPILER = $(2)
370 $(3)/stage$(1)/test/rustctest-$(2)$$(X_$(2)): \
371 $$(COMPILER_CRATE) $$(COMPILER_INPUTS) \
372 $$(SREQ$(1)_T_$(2)_H_$(3)) \
373 $$(TLIB$(1)_T_$(2)_H_$(3))/$$(CFG_RUSTLLVM_$(2)) \
374 $$(TLIB$(1)_T_$(2)_H_$(3))/$$(CFG_LIBSYNTAX_$(2))
375 @$$(call E, compile_and_link: $$@)
376 $$(STAGE$(1)_T_$(2)_H_$(3)) -o $$@ $$< --test
378 $(3)/stage$(1)/test/rustpkgtest-$(2)$$(X_$(2)): \
379 $$(RUSTPKG_LIB) $$(RUSTPKG_INPUTS) \
380 $$(SREQ$(1)_T_$(2)_H_$(3)) \
381 $$(TLIB$(1)_T_$(2)_H_$(3))/$$(CFG_LIBSYNTAX_$(2)) \
382 $$(TLIB$(1)_T_$(2)_H_$(3))/$$(CFG_LIBRUSTC_$(2)) \
383 $$(HBIN$(1)_H_$(3))/rustpkg$$(X_$(2)) \
384 $$(TBIN$(1)_T_$(2)_H_$(3))/rustpkg$$(X_$(2)) \
385 $$(TBIN$(1)_T_$(2)_H_$(3))/rustc$$(X_$(2))
386 @$$(call E, compile_and_link: $$@)
387 $$(STAGE$(1)_T_$(2)_H_$(3)) -o $$@ $$< --test
389 $(3)/stage$(1)/test/rustdoctest-$(2)$$(X_$(2)): \
390 $$(RUSTDOC_LIB) $$(RUSTDOC_INPUTS) \
391 $$(SREQ$(1)_T_$(2)_H_$(3)) \
392 $$(TLIB$(1)_T_$(2)_H_$(3))/$$(CFG_LIBSYNTAX_$(2)) \
393 $$(TLIB$(1)_T_$(2)_H_$(3))/$$(CFG_LIBRUSTC_$(2))
394 @$$(call E, compile_and_link: $$@)
395 $$(STAGE$(1)_T_$(2)_H_$(3)) -o $$@ $$< --test
399 $(foreach host,$(CFG_HOST), \
400 $(eval $(foreach target,$(CFG_TARGET), \
401 $(eval $(foreach stage,$(STAGES), \
402 $(eval $(call TEST_RUNNER,$(stage),$(target),$(host))))))))
404 define DEF_TEST_CRATE_RULES
405 check-stage$(1)-T-$(2)-H-$(3)-$(4)-exec: $$(call TEST_OK_FILE,$(1),$(2),$(3),$(4))
407 check-stage$(1)-T-$(2)-H-$(3)-$(4)-exec: $$(call TEST_OK_FILE,$(1),$(2),$(3),$(4))
409 $$(call TEST_OK_FILE,$(1),$(2),$(3),$(4)): \
410 $(3)/stage$(1)/test/$(4)test-$(2)$$(X_$(2))
411 @$$(call E, run: $$<)
412 $$(Q)$$(call CFG_RUN_TEST_$(2),$$<,$(2),$(3)) $$(TESTARGS) \
413 --logfile $$(call TEST_LOG_FILE,$(1),$(2),$(3),$(4)) \
414 $$(call CRATE_TEST_EXTRA_ARGS,$(1),$(2),$(3),$(4)) \
418 define DEF_TEST_CRATE_RULES_arm-linux-androideabi
419 check-stage$(1)-T-$(2)-H-$(3)-$(4)-exec: $$(call TEST_OK_FILE,$(1),$(2),$(3),$(4))
421 $$(call TEST_OK_FILE,$(1),$(2),$(3),$(4)): \
422 $(3)/stage$(1)/test/$(4)test-$(2)$$(X_$(2))
423 @$$(call E, run: $$< via adb)
424 @$(CFG_ADB) push $$< $(CFG_ADB_TEST_DIR)
425 @$(CFG_ADB) shell '(cd $(CFG_ADB_TEST_DIR); LD_LIBRARY_PATH=. \
427 --logfile $(CFG_ADB_TEST_DIR)/check-stage$(1)-T-$(2)-H-$(3)-$(4).log \
428 $$(call CRATE_TEST_EXTRA_ARGS,$(1),$(2),$(3),$(4)))' \
429 > tmp/check-stage$(1)-T-$(2)-H-$(3)-$(4).tmp
430 @cat tmp/check-stage$(1)-T-$(2)-H-$(3)-$(4).tmp
431 @touch tmp/check-stage$(1)-T-$(2)-H-$(3)-$(4).log
432 @$(CFG_ADB) pull $(CFG_ADB_TEST_DIR)/check-stage$(1)-T-$(2)-H-$(3)-$(4).log tmp/
433 @$(CFG_ADB) shell rm $(CFG_ADB_TEST_DIR)/check-stage$(1)-T-$(2)-H-$(3)-$(4).log
434 @$(CFG_ADB) pull $(CFG_ADB_TEST_DIR)/$$(call TEST_RATCHET_FILE,$(1),$(2),$(3),$(4)) tmp/
435 @if grep -q "result: ok" tmp/check-stage$(1)-T-$(2)-H-$(3)-$(4).tmp; \
437 rm tmp/check-stage$(1)-T-$(2)-H-$(3)-$(4).tmp; \
440 rm tmp/check-stage$(1)-T-$(2)-H-$(3)-$(4).tmp; \
445 define DEF_TEST_CRATE_RULES_null
446 check-stage$(1)-T-$(2)-H-$(3)-$(4)-exec: $$(call TEST_OK_FILE,$(1),$(2),$(3),$(4))
448 $$(call TEST_OK_FILE,$(1),$(2),$(3),$(4)): \
449 $(3)/stage$(1)/test/$(4)test-$(2)$$(X_$(2))
450 @$$(call E, run: skipped $$< )
454 $(foreach host,$(CFG_HOST), \
455 $(foreach target,$(CFG_TARGET), \
456 $(foreach stage,$(STAGES), \
457 $(foreach crate, $(TEST_CRATES), \
458 $(if $(findstring $(target),$(CFG_BUILD)), \
459 $(eval $(call DEF_TEST_CRATE_RULES,$(stage),$(target),$(host),$(crate))), \
460 $(if $(findstring $(target),"arm-linux-androideabi"), \
461 $(if $(findstring $(CFG_ADB_DEVICE_STATUS),"true"), \
462 $(eval $(call DEF_TEST_CRATE_RULES_arm-linux-androideabi,$(stage),$(target),$(host),$(crate))), \
463 $(eval $(call DEF_TEST_CRATE_RULES_null,$(stage),$(target),$(host),$(crate))) \
465 $(eval $(call DEF_TEST_CRATE_RULES,$(stage),$(target),$(host),$(crate))) \
468 # FIXME (#10104): Raise the stack size to work around rustpkg bypassing
469 # the code in rustc that would take care of it.
470 define DEF_RUSTPKG_STACK_FIX
471 $$(call TEST_OK_FILE,$(1),$(2),$(3),rustpkg): export RUST_MIN_STACK=8000000
474 $(foreach host,$(CFG_HOST_TRIPLES), \
475 $(foreach target,$(CFG_TARGET_TRIPLES), \
476 $(foreach stage,$(STAGES), \
477 $(eval $(call DEF_RUSTPKG_STACK_FIX,$(stage),$(target),$(host))))))
480 ######################################################################
481 # Rules for the compiletest tests (rpass, rfail, etc.)
482 ######################################################################
484 RPASS_RC := $(wildcard $(S)src/test/run-pass/*.rc)
485 RPASS_RS := $(wildcard $(S)src/test/run-pass/*.rs)
486 RPASS_FULL_RC := $(wildcard $(S)src/test/run-pass-fulldeps/*.rc)
487 RPASS_FULL_RS := $(wildcard $(S)src/test/run-pass-fulldeps/*.rs)
488 RFAIL_RC := $(wildcard $(S)src/test/run-fail/*.rc)
489 RFAIL_RS := $(wildcard $(S)src/test/run-fail/*.rs)
490 CFAIL_RC := $(wildcard $(S)src/test/compile-fail/*.rc)
491 CFAIL_RS := $(wildcard $(S)src/test/compile-fail/*.rs)
492 BENCH_RS := $(wildcard $(S)src/test/bench/*.rs)
493 PRETTY_RS := $(wildcard $(S)src/test/pretty/*.rs)
494 DEBUGINFO_RS := $(wildcard $(S)src/test/debug-info/*.rs)
495 CODEGEN_RS := $(wildcard $(S)src/test/codegen/*.rs)
496 CODEGEN_CC := $(wildcard $(S)src/test/codegen/*.cc)
498 # perf tests are the same as bench tests only they run under
499 # a performance monitor.
500 PERF_RS := $(wildcard $(S)src/test/bench/*.rs)
502 RPASS_TESTS := $(RPASS_RC) $(RPASS_RS)
503 RPASS_FULL_TESTS := $(RPASS_FULL_RC) $(RPASS_FULL_RS)
504 RFAIL_TESTS := $(RFAIL_RC) $(RFAIL_RS)
505 CFAIL_TESTS := $(CFAIL_RC) $(CFAIL_RS)
506 BENCH_TESTS := $(BENCH_RS)
507 PERF_TESTS := $(PERF_RS)
508 PRETTY_TESTS := $(PRETTY_RS)
509 DEBUGINFO_TESTS := $(DEBUGINFO_RS)
510 CODEGEN_TESTS := $(CODEGEN_RS) $(CODEGEN_CC)
512 CTEST_SRC_BASE_rpass = run-pass
513 CTEST_BUILD_BASE_rpass = run-pass
514 CTEST_MODE_rpass = run-pass
515 CTEST_RUNTOOL_rpass = $(CTEST_RUNTOOL)
517 CTEST_SRC_BASE_rpass-full = run-pass-full
518 CTEST_BUILD_BASE_rpass-full = run-pass-full
519 CTEST_MODE_rpass-full = run-pass
520 CTEST_RUNTOOL_rpass-full = $(CTEST_RUNTOOL)
522 CTEST_SRC_BASE_rfail = run-fail
523 CTEST_BUILD_BASE_rfail = run-fail
524 CTEST_MODE_rfail = run-fail
525 CTEST_RUNTOOL_rfail = $(CTEST_RUNTOOL)
527 CTEST_SRC_BASE_cfail = compile-fail
528 CTEST_BUILD_BASE_cfail = compile-fail
529 CTEST_MODE_cfail = compile-fail
530 CTEST_RUNTOOL_cfail = $(CTEST_RUNTOOL)
532 CTEST_SRC_BASE_bench = bench
533 CTEST_BUILD_BASE_bench = bench
534 CTEST_MODE_bench = run-pass
535 CTEST_RUNTOOL_bench = $(CTEST_RUNTOOL)
537 CTEST_SRC_BASE_perf = bench
538 CTEST_BUILD_BASE_perf = perf
539 CTEST_MODE_perf = run-pass
540 CTEST_RUNTOOL_perf = $(CTEST_PERF_RUNTOOL)
542 CTEST_SRC_BASE_debuginfo = debug-info
543 CTEST_BUILD_BASE_debuginfo = debug-info
544 CTEST_MODE_debuginfo = debug-info
545 CTEST_RUNTOOL_debuginfo = $(CTEST_RUNTOOL)
547 CTEST_SRC_BASE_codegen = codegen
548 CTEST_BUILD_BASE_codegen = codegen
549 CTEST_MODE_codegen = codegen
550 CTEST_RUNTOOL_codegen = $(CTEST_RUNTOOL)
553 CTEST_DISABLE_debuginfo = "no gdb found"
557 CTEST_DISABLE_codegen = "no clang found"
560 ifeq ($(CFG_OSTYPE),apple-darwin)
561 CTEST_DISABLE_debuginfo = "gdb on darwing needs root"
564 define DEF_CTEST_VARS
566 # All the per-stage build rules you might want to call from the
569 # $(1) is the stage number
570 # $(2) is the target triple to test
571 # $(3) is the host triple to test
573 # Prerequisites for compiletest tests
574 TEST_SREQ$(1)_T_$(2)_H_$(3) = \
575 $$(HBIN$(1)_H_$(3))/compiletest$$(X_$(3)) \
576 $$(SREQ$(1)_T_$(2)_H_$(3))
578 # Rules for the cfail/rfail/rpass/bench/perf test runner
580 # The tests select when to use debug configuration on their own;
581 # remove directive, if present, from CFG_RUSTC_FLAGS (issue #7898).
582 CTEST_RUSTC_FLAGS := $$(subst --cfg ndebug,,$$(CFG_RUSTC_FLAGS))
584 # The tests can not be optimized while the rest of the compiler is optimized, so
585 # filter out the optimization (if any) from rustc and then figure out if we need
587 CTEST_RUSTC_FLAGS := $$(subst -O,,$$(CTEST_RUSTC_FLAGS))
588 ifndef CFG_DISABLE_OPTIMIZE_TESTS
589 CTEST_RUSTC_FLAGS += -O
592 CTEST_COMMON_ARGS$(1)-T-$(2)-H-$(3) := \
593 --compile-lib-path $$(HLIB$(1)_H_$(3)) \
594 --run-lib-path $$(TLIB$(1)_T_$(2)_H_$(3)) \
595 --rustc-path $$(HBIN$(1)_H_$(3))/rustc$$(X_$(3)) \
596 --clang-path $(if $(CFG_CLANG),$(CFG_CLANG),clang) \
597 --llvm-bin-path $(CFG_LLVM_INST_DIR_$(CFG_BUILD))/bin \
598 --aux-base $$(S)src/test/auxiliary/ \
599 --stage-id stage$(1)-$(2) \
601 --adb-path=$(CFG_ADB) \
602 --adb-test-dir=$(CFG_ADB_TEST_DIR) \
603 --rustcflags "$(RUSTC_FLAGS_$(2)) $$(CTEST_RUSTC_FLAGS) --target=$(2)" \
606 CTEST_DEPS_rpass_$(1)-T-$(2)-H-$(3) = $$(RPASS_TESTS)
607 CTEST_DEPS_rpass_full_$(1)-T-$(2)-H-$(3) = $$(RPASS_FULL_TESTS) $$(TLIBRUSTC_DEFAULT$(1)_T_$(2)_H_$(3))
608 CTEST_DEPS_rfail_$(1)-T-$(2)-H-$(3) = $$(RFAIL_TESTS)
609 CTEST_DEPS_cfail_$(1)-T-$(2)-H-$(3) = $$(CFAIL_TESTS)
610 CTEST_DEPS_bench_$(1)-T-$(2)-H-$(3) = $$(BENCH_TESTS)
611 CTEST_DEPS_perf_$(1)-T-$(2)-H-$(3) = $$(PERF_TESTS)
612 CTEST_DEPS_debuginfo_$(1)-T-$(2)-H-$(3) = $$(DEBUGINFO_TESTS)
613 CTEST_DEPS_codegen_$(1)-T-$(2)-H-$(3) = $$(CODEGEN_TESTS)
617 $(foreach host,$(CFG_HOST), \
618 $(eval $(foreach target,$(CFG_TARGET), \
619 $(eval $(foreach stage,$(STAGES), \
620 $(eval $(call DEF_CTEST_VARS,$(stage),$(target),$(host))))))))
622 define DEF_RUN_COMPILETEST
624 CTEST_ARGS$(1)-T-$(2)-H-$(3)-$(4) := \
625 $$(CTEST_COMMON_ARGS$(1)-T-$(2)-H-$(3)) \
626 --src-base $$(S)src/test/$$(CTEST_SRC_BASE_$(4))/ \
627 --build-base $(3)/test/$$(CTEST_BUILD_BASE_$(4))/ \
628 --ratchet-metrics $(call TEST_RATCHET_FILE,$(1),$(2),$(3),$(4)) \
629 --mode $$(CTEST_MODE_$(4)) \
630 $$(CTEST_RUNTOOL_$(4))
632 check-stage$(1)-T-$(2)-H-$(3)-$(4)-exec: $$(call TEST_OK_FILE,$(1),$(2),$(3),$(4))
634 ifeq ($$(CTEST_DISABLE_$(4)),)
636 $$(call TEST_OK_FILE,$(1),$(2),$(3),$(4)): \
637 $$(TEST_SREQ$(1)_T_$(2)_H_$(3)) \
638 $$(CTEST_DEPS_$(4)_$(1)-T-$(2)-H-$(3))
639 @$$(call E, run $(4) [$(2)]: $$<)
640 $$(Q)$$(call CFG_RUN_CTEST_$(2),$(1),$$<,$(3)) \
641 $$(CTEST_ARGS$(1)-T-$(2)-H-$(3)-$(4)) \
642 --logfile $$(call TEST_LOG_FILE,$(1),$(2),$(3),$(4)) \
647 $$(call TEST_OK_FILE,$(1),$(2),$(3),$(4)): \
648 $$(TEST_SREQ$(1)_T_$(2)_H_$(3)) \
649 $$(CTEST_DEPS_$(4)_$(1)-T-$(2)-H-$(3))
650 @$$(call E, run $(4) [$(2)]: $$<)
651 @$$(call E, warning: tests disabled: $$(CTEST_DISABLE_$(4)))
658 CTEST_NAMES = rpass rpass-full rfail cfail bench perf debuginfo codegen
660 $(foreach host,$(CFG_HOST), \
661 $(eval $(foreach target,$(CFG_TARGET), \
662 $(eval $(foreach stage,$(STAGES), \
663 $(eval $(foreach name,$(CTEST_NAMES), \
664 $(eval $(call DEF_RUN_COMPILETEST,$(stage),$(target),$(host),$(name))))))))))
666 PRETTY_NAMES = pretty-rpass pretty-rpass-full pretty-rfail pretty-bench pretty-pretty
667 PRETTY_DEPS_pretty-rpass = $(RPASS_TESTS)
668 PRETTY_DEPS_pretty-rpass-full = $(RPASS_FULL_TESTS)
669 PRETTY_DEPS_pretty-rfail = $(RFAIL_TESTS)
670 PRETTY_DEPS_pretty-bench = $(BENCH_TESTS)
671 PRETTY_DEPS_pretty-pretty = $(PRETTY_TESTS)
672 PRETTY_DIRNAME_pretty-rpass = run-pass
673 PRETTY_DIRNAME_pretty-rpass-full = run-pass-full
674 PRETTY_DIRNAME_pretty-rfail = run-fail
675 PRETTY_DIRNAME_pretty-bench = bench
676 PRETTY_DIRNAME_pretty-pretty = pretty
678 define DEF_RUN_PRETTY_TEST
680 PRETTY_ARGS$(1)-T-$(2)-H-$(3)-$(4) := \
681 $$(CTEST_COMMON_ARGS$(1)-T-$(2)-H-$(3)) \
682 --src-base $$(S)src/test/$$(PRETTY_DIRNAME_$(4))/ \
683 --build-base $(3)/test/$$(PRETTY_DIRNAME_$(4))/ \
686 check-stage$(1)-T-$(2)-H-$(3)-$(4)-exec: $$(call TEST_OK_FILE,$(1),$(2),$(3),$(4))
688 $$(call TEST_OK_FILE,$(1),$(2),$(3),$(4)): \
689 $$(TEST_SREQ$(1)_T_$(2)_H_$(3)) \
691 @$$(call E, run pretty-rpass [$(2)]: $$<)
692 $$(Q)$$(call CFG_RUN_CTEST_$(2),$(1),$$<,$(3)) \
693 $$(PRETTY_ARGS$(1)-T-$(2)-H-$(3)-$(4)) \
694 --logfile $$(call TEST_LOG_FILE,$(1),$(2),$(3),$(4)) \
699 $(foreach host,$(CFG_HOST), \
700 $(foreach target,$(CFG_TARGET), \
701 $(foreach stage,$(STAGES), \
702 $(foreach pretty-name,$(PRETTY_NAMES), \
703 $(eval $(call DEF_RUN_PRETTY_TEST,$(stage),$(target),$(host),$(pretty-name)))))))
705 define DEF_RUN_DOC_TEST
707 DOC_TEST_ARGS$(1)-T-$(2)-H-$(3)-doc-$(4) := \
708 $$(CTEST_COMMON_ARGS$(1)-T-$(2)-H-$(3)) \
709 --src-base $(3)/test/doc-$(4)/ \
710 --build-base $(3)/test/doc-$(4)/ \
713 check-stage$(1)-T-$(2)-H-$(3)-doc-$(4)-exec: $$(call TEST_OK_FILE,$(1),$(2),$(3),doc-$(4))
715 $$(call TEST_OK_FILE,$(1),$(2),$(3),doc-$(4)): \
716 $$(TEST_SREQ$(1)_T_$(2)_H_$(3)) \
718 @$$(call E, run doc-$(4) [$(2)]: $$<)
719 $$(Q)$$(call CFG_RUN_CTEST_$(2),$(1),$$<,$(3)) \
720 $$(DOC_TEST_ARGS$(1)-T-$(2)-H-$(3)-doc-$(4)) \
721 --logfile $$(call TEST_LOG_FILE,$(1),$(2),$(3),doc-$(4)) \
726 $(foreach host,$(CFG_HOST), \
727 $(foreach target,$(CFG_TARGET), \
728 $(foreach stage,$(STAGES), \
729 $(foreach docname,$(DOC_TEST_NAMES), \
730 $(eval $(call DEF_RUN_DOC_TEST,$(stage),$(target),$(host),$(docname)))))))
733 ######################################################################
734 # Extracting tests for docs
735 ######################################################################
737 EXTRACT_TESTS := "$(CFG_PYTHON)" $(S)src/etc/extract-tests.py
739 define DEF_DOC_TEST_HOST
741 doc-$(2)-extract$(1):
742 @$$(call E, extract: $(2) tests)
743 $$(Q)rm -f $(1)/test/doc-$(2)/*.rs
744 $$(Q)$$(EXTRACT_TESTS) $$(S)doc/$(2).md $(1)/test/doc-$(2)
748 $(foreach host,$(CFG_HOST), \
749 $(foreach docname,$(DOC_TEST_NAMES), \
750 $(eval $(call DEF_DOC_TEST_HOST,$(host),$(docname)))))
753 ######################################################################
755 ######################################################################
759 $(foreach crate,$(TEST_CRATES),$(crate)) \
769 $(foreach docname,$(DOC_TEST_NAMES),doc-$(docname)) \
778 define DEF_CHECK_FOR_STAGE_AND_TARGET_AND_HOST
779 check-stage$(1)-T-$(2)-H-$(3): check-stage$(1)-T-$(2)-H-$(3)-exec
782 $(foreach stage,$(STAGES), \
783 $(foreach target,$(CFG_TARGET), \
784 $(foreach host,$(CFG_HOST), \
785 $(eval $(call DEF_CHECK_FOR_STAGE_AND_TARGET_AND_HOST,$(stage),$(target),$(host))))))
787 define DEF_CHECK_FOR_STAGE_AND_TARGET_AND_HOST_AND_GROUP
788 check-stage$(1)-T-$(2)-H-$(3)-$(4): check-stage$(1)-T-$(2)-H-$(3)-$(4)-exec
791 $(foreach stage,$(STAGES), \
792 $(foreach target,$(CFG_TARGET), \
793 $(foreach host,$(CFG_HOST), \
794 $(foreach group,$(TEST_GROUPS), \
795 $(eval $(call DEF_CHECK_FOR_STAGE_AND_TARGET_AND_HOST_AND_GROUP,$(stage),$(target),$(host),$(group)))))))
797 define DEF_CHECK_FOR_STAGE
798 check-stage$(1): check-stage$(1)-H-$$(CFG_BUILD)
799 check-stage$(1)-H-all: $$(foreach target,$$(CFG_TARGET), \
800 check-stage$(1)-H-$$(target))
803 $(foreach stage,$(STAGES), \
804 $(eval $(call DEF_CHECK_FOR_STAGE,$(stage))))
806 define DEF_CHECK_FOR_STAGE_AND_GROUP
807 check-stage$(1)-$(2): check-stage$(1)-H-$$(CFG_BUILD)-$(2)
808 check-stage$(1)-H-all-$(2): $$(foreach target,$$(CFG_TARGET), \
809 check-stage$(1)-H-$$(target)-$(2))
812 $(foreach stage,$(STAGES), \
813 $(foreach group,$(TEST_GROUPS), \
814 $(eval $(call DEF_CHECK_FOR_STAGE_AND_GROUP,$(stage),$(group)))))
817 define DEF_CHECK_FOR_STAGE_AND_HOSTS
818 check-stage$(1)-H-$(2): $$(foreach target,$$(CFG_TARGET), \
819 check-stage$(1)-T-$$(target)-H-$(2))
822 $(foreach stage,$(STAGES), \
823 $(foreach host,$(CFG_HOST), \
824 $(eval $(call DEF_CHECK_FOR_STAGE_AND_HOSTS,$(stage),$(host)))))
826 define DEF_CHECK_FOR_STAGE_AND_HOSTS_AND_GROUP
827 check-stage$(1)-H-$(2)-$(3): $$(foreach target,$$(CFG_TARGET), \
828 check-stage$(1)-T-$$(target)-H-$(2)-$(3))
831 $(foreach stage,$(STAGES), \
832 $(foreach host,$(CFG_HOST), \
833 $(foreach group,$(TEST_GROUPS), \
834 $(eval $(call DEF_CHECK_FOR_STAGE_AND_HOSTS_AND_GROUP,$(stage),$(host),$(group))))))
836 ######################################################################
838 ######################################################################
840 FT := run_pass_stage2
841 FT_LIB := $(call CFG_LIB_NAME_$(CFG_BUILD),$(FT))
842 FT_DRIVER := $(FT)_driver
844 GENERATED += tmp/$(FT).rc tmp/$(FT_DRIVER).rs
846 tmp/$(FT).rc tmp/$(FT_DRIVER).rs: \
848 $(S)src/etc/combine-tests.py
849 @$(call E, check: building combined stage2 test runner)
850 $(Q)$(CFG_PYTHON) $(S)src/etc/combine-tests.py
852 define DEF_CHECK_FAST_FOR_T_H
857 $$(TLIB2_T_$(2)_H_$(3))/$$(FT_LIB): \
859 $$(SREQ2_T_$(2)_H_$(3))
860 @$$(call E, compile_and_link: $$@)
861 $$(STAGE2_T_$(2)_H_$(3)) --lib -o $$@ $$<
863 $(3)/test/$$(FT_DRIVER)-$(2)$$(X_$(2)): \
864 tmp/$$(FT_DRIVER).rs \
865 $$(TLIB2_T_$(2)_H_$(3))/$$(FT_LIB) \
866 $$(SREQ2_T_$(2)_H_$(3))
867 @$$(call E, compile_and_link: $$@ $$<)
868 $$(STAGE2_T_$(2)_H_$(3)) -o $$@ $$<
870 $(3)/test/$$(FT_DRIVER)-$(2).out: \
871 $(3)/test/$$(FT_DRIVER)-$(2)$$(X_$(2)) \
872 $$(SREQ2_T_$(2)_H_$(3))
873 $$(Q)$$(call CFG_RUN_TEST_$(2),$$<,$(2),$(3)) \
874 --logfile tmp/$$(FT_DRIVER)-$(2).log
876 check-fast-T-$(2)-H-$(3): \
877 $(3)/test/$$(FT_DRIVER)-$(2).out
881 $(foreach host,$(CFG_HOST), \
882 $(eval $(foreach target,$(CFG_TARGET), \
883 $(eval $(call DEF_CHECK_FAST_FOR_T_H,,$(target),$(host))))))
885 check-fast: tidy check-fast-H-$(CFG_BUILD) check-stage2-std check-stage2-extra
886 $(Q)$(CFG_PYTHON) $(S)src/etc/check-summary.py tmp/*.log
888 define DEF_CHECK_FAST_FOR_H
890 check-fast-H-$(1): check-fast-T-$(1)-H-$(1)
894 $(foreach host,$(CFG_HOST), \
895 $(eval $(call DEF_CHECK_FAST_FOR_H,$(host))))