src/test/run-make-fulldeps/coverage-reports-base/Makefile

   1 # needs-profiler-support
   2 # ignore-windows-gnu
   3
   4 # FIXME(mati865): MinGW GCC miscompiles compiler-rt profiling library but with Clang it works
   5 # properly. Since we only have GCC on the CI ignore the test for now.
   6
   7 # ISSUE(76038): When targeting MSVC, Rust binaries built with both `-Z instrument-coverage` and
   8 # `-C link-dead-code` typically crash (with a seg-fault) or at best generate an empty `*.profraw`.
   9 # See ../coverage/coverage_tools.mk for more information.
  10
  11 -include ../coverage/coverage_tools.mk
  12
  13 BASEDIR=../coverage-reports-base
  14 SOURCEDIR=../coverage
  15
  16 ifeq ($(UNAME),Darwin)
  17 # FIXME(richkadel): It appears that --debug is not available on MacOS even when not running
  18 # under CI.
  19 NO_LLVM_ASSERTIONS=1
  20 endif
  21
  22 # The `llvm-cov show` flag `--debug`, used to generate the `counters` output files, is only enabled
  23 # if LLVM assertions are enabled. Some CI builds disable debug assertions.
  24 ifndef NO_LLVM_ASSERTIONS
  25 DEBUG_FLAG=--debug
  26 endif
  27
  28 # When generating `expected_*` results (using `x.py test --bless`), the `--debug` flag is forced.
  29 # If assertions are disabled, the command will fail with an error, rather than attempt to generate
  30 # only partial results.
  31 ifdef RUSTC_BLESS_TEST
  32 DEBUG_FLAG=--debug
  33 endif
  34
  35 all: $(patsubst $(SOURCEDIR)/%.rs,%,$(wildcard $(SOURCEDIR)/*.rs))
  36
  37 # Ensure there are no `expected` results for tests that may have been removed or renamed
  38 .PHONY: clear_expected_if_blessed
  39 clear_expected_if_blessed:
  40 ifdef RUSTC_BLESS_TEST
  41         rm -f expected_export_coverage.*.json
  42         rm -f expected_show_coverage.*.txt
  43         rm -f expected_show_coverage_counters.*.txt
  44 endif
  45
  46 -include clear_expected_if_blessed
  47
  48 %: $(SOURCEDIR)/%.rs
  49         # Compile the test program with coverage instrumentation and generate relevant MIR.
  50         $(RUSTC) $(SOURCEDIR)/$@.rs \
  51                         -Zinstrument-coverage \
  52                         -Clink-dead-code=$(LINK_DEAD_CODE)
  53
  54         # Run it in order to generate some profiling data,
  55         # with `LLVM_PROFILE_FILE=<profdata_file>` environment variable set to
  56         # output the coverage stats for this run.
  57         LLVM_PROFILE_FILE="$(TMPDIR)"/$@.profraw \
  58                         $(call RUN,$@) || \
  59                         ( \
  60                                 status=$$?; \
  61                                 grep -q "^\/\/ expect-exit-status-$$status" $(SOURCEDIR)/$@.rs || \
  62                                 ( >&2 echo "program exited with an unexpected exit status: $$status"; \
  63                                         false \
  64                                 ) \
  65                         )
  66
  67         # Postprocess the profiling data so it can be used by the llvm-cov tool
  68         "$(LLVM_BIN_DIR)"/llvm-profdata merge --sparse \
  69                         "$(TMPDIR)"/$@.profraw \
  70                         -o "$(TMPDIR)"/$@.profdata
  71
  72         # Generate a coverage report using `llvm-cov show`.
  73         "$(LLVM_BIN_DIR)"/llvm-cov show \
  74                         $(DEBUG_FLAG) \
  75                         --Xdemangler="$(RUST_DEMANGLER)" \
  76                         --show-line-counts-or-regions \
  77                         --instr-profile="$(TMPDIR)"/$@.profdata \
  78                         $(call BIN,"$(TMPDIR)"/$@) \
  79                 > "$(TMPDIR)"/actual_show_coverage.$@.txt \
  80                 2> "$(TMPDIR)"/show_coverage_stderr.$@.txt || \
  81         ( status=$$? ; \
  82                 >&2 cat "$(TMPDIR)"/show_coverage_stderr.$@.txt ; \
  83                 exit $$status \
  84         )
  85
  86 ifdef DEBUG_FLAG
  87         # The first line (beginning with "Args:" contains hard-coded, build-specific
  88         # file paths. Strip that line and keep the remaining lines with counter debug
  89         # data.
  90         tail -n +2 "$(TMPDIR)"/show_coverage_stderr.$@.txt \
  91                 > "$(TMPDIR)"/actual_show_coverage_counters.$@.txt
  92 endif
  93
  94 ifdef RUSTC_BLESS_TEST
  95         cp "$(TMPDIR)"/actual_show_coverage.$@.txt \
  96                         expected_show_coverage.$@.txt
  97         cp "$(TMPDIR)"/actual_show_coverage_counters.$@.txt \
  98                         expected_show_coverage_counters.$@.txt
  99 else
 100         # Compare the show coverage output (`--bless` refreshes `typical` files)
 101         # Note `llvm-cov show` output for some programs can vary, but can be ignored
 102         # by inserting `// ignore-llvm-cov-show-diffs` at the top of the source file.
 103
 104         $(DIFF) expected_show_coverage.$@.txt "$(TMPDIR)"/actual_show_coverage.$@.txt || \
 105                 ( grep -q '^\/\/ ignore-llvm-cov-show-diffs' $(SOURCEDIR)/$@.rs && \
 106                         >&2 echo 'diff failed, but suppressed with `// ignore-llvm-cov-show-diffs` in $(SOURCEDIR)/$@.rs' \
 107                 ) || \
 108                 ( >&2 echo 'diff failed, and not suppressed without `// ignore-llvm-cov-show-diffs` in $(SOURCEDIR)/$@.rs'; \
 109                         false \
 110                 )
 111
 112 ifdef DEBUG_FLAG
 113         $(DIFF) expected_show_coverage_counters.$@.txt "$(TMPDIR)"/actual_show_coverage_counters.$@.txt || \
 114                 ( grep -q '^\/\/ ignore-llvm-cov-show-diffs' $(SOURCEDIR)/$@.rs && \
 115                         >&2 echo 'diff failed, but suppressed with `// ignore-llvm-cov-show-diffs` in $(SOURCEDIR)/$@.rs' \
 116                 ) || \
 117                 ( >&2 echo 'diff failed, and not suppressed without `// ignore-llvm-cov-show-diffs` in $(SOURCEDIR)/$@.rs'; \
 118                         >&2 echo '(Ignore anyway until mangled function names in "counters" files are demangled.)' \
 119                 )
 120
 121                 # FIXME(richkadel): Apply the demangler to the `*_show_coverage_counters.*.txt` files,
 122                 # so the crate disambiguator differences will be stripped away. At that point, these files
 123                 # will be less likely to vary, and the last `echo` above (starting with "Ignore anyway")
 124                 # can be replaced with `false` to fail the test.
 125 endif
 126
 127 endif
 128
 129         # Generate a coverage report in JSON, using `llvm-cov export`, and fail if
 130         # there are differences from the expected output.
 131         "$(LLVM_BIN_DIR)"/llvm-cov export \
 132                         --summary-only \
 133                         --instr-profile="$(TMPDIR)"/$@.profdata \
 134                         $(call BIN,"$(TMPDIR)"/$@) \
 135                 | "$(PYTHON)" $(BASEDIR)/prettify_json.py \
 136                 > "$(TMPDIR)"/actual_export_coverage.$@.json
 137
 138 ifdef RUSTC_BLESS_TEST
 139         cp "$(TMPDIR)"/actual_export_coverage.$@.json expected_export_coverage.$@.json
 140 else
 141         # Check that exported JSON coverage data matches what we expect (`--bless` refreshes `expected`)
 142         $(DIFF) expected_export_coverage.$@.json "$(TMPDIR)"/actual_export_coverage.$@.json
 143 endif