util/travis/run-clang-tidy.py

   1 #!/usr/bin/env python2
   2 #
   3 # ===- run-clang-tidy.py - Parallel clang-tidy runner ---------*- python -*--===#
   4 #
   5 #                     The LLVM Compiler Infrastructure
   6 #
   7 # This file is distributed under the University of Illinois Open Source
   8 # License. See LICENSE.TXT for details.
   9 #
  10 # ===------------------------------------------------------------------------===#
  11 # FIXME: Integrate with clang-tidy-diff.py
  12
  13 """
  14 Parallel clang-tidy runner
  15 ==========================
  16
  17 Runs clang-tidy over all files in a compilation database. Requires clang-tidy
  18 and clang-apply-replacements in $PATH.
  19
  20 Example invocations.
  21 - Run clang-tidy on all files in the current working directory with a default
  22   set of checks and show warnings in the cpp files and all project headers.
  23     run-clang-tidy.py $PWD
  24
  25 - Fix all header guards.
  26     run-clang-tidy.py -fix -checks=-*,llvm-header-guard
  27
  28 - Fix all header guards included from clang-tidy and header guards
  29   for clang-tidy headers.
  30     run-clang-tidy.py -fix -checks=-*,llvm-header-guard extra/clang-tidy \
  31                       -header-filter=extra/clang-tidy
  32
  33 Compilation database setup:
  34 http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html
  35 """
  36
  37 from __future__ import print_function
  38 import argparse
  39 import json
  40 import multiprocessing
  41 import os
  42 import Queue
  43 import re
  44 import shutil
  45 import subprocess
  46 import sys
  47 import tempfile
  48 import threading
  49 import traceback
  50
  51
  52 class TidyQueue(Queue.Queue):
  53     def __init__(self, max_task):
  54         Queue.Queue.__init__(self, max_task)
  55         self.has_error = False
  56
  57
  58 def find_compilation_database(path):
  59     """Adjusts the directory until a compilation database is found."""
  60     result = './'
  61     while not os.path.isfile(os.path.join(result, path)):
  62         if os.path.realpath(result) == '/':
  63             print('Error: could not find compilation database.')
  64             sys.exit(1)
  65         result += '../'
  66     return os.path.realpath(result)
  67
  68
  69 def get_tidy_invocation(f, clang_tidy_binary, checks, warningsaserrors,
  70                         tmpdir, build_path,
  71                         header_filter, extra_arg, extra_arg_before, quiet):
  72     """Gets a command line for clang-tidy."""
  73     start = [clang_tidy_binary]
  74     if header_filter is not None:
  75         start.append('-header-filter=' + header_filter)
  76     else:
  77         # Show warnings in all in-project headers by default.
  78         start.append('-header-filter=^' + build_path + '/.*')
  79     if checks:
  80         start.append('-checks=' + checks)
  81     if warningsaserrors:
  82         start.append('-warnings-as-errors=' + warningsaserrors)
  83     if tmpdir is not None:
  84         start.append('-export-fixes')
  85         # Get a temporary file. We immediately close the handle so clang-tidy can
  86         # overwrite it.
  87         (handle, name) = tempfile.mkstemp(suffix='.yaml', dir=tmpdir)
  88         os.close(handle)
  89         start.append(name)
  90     for arg in extra_arg:
  91         start.append('-extra-arg=%s' % arg)
  92     for arg in extra_arg_before:
  93         start.append('-extra-arg-before=%s' % arg)
  94     start.append('-p=' + build_path)
  95     if quiet:
  96         start.append('-quiet')
  97     start.append(f)
  98     return start
  99
 100
 101 def check_clang_apply_replacements_binary(args):
 102     """Checks if invoking supplied clang-apply-replacements binary works."""
 103     try:
 104         subprocess.check_call([args.clang_apply_replacements_binary, '--version'])
 105     except:
 106         print('Unable to run clang-apply-replacements. Is clang-apply-replacements '
 107               'binary correctly specified?', file=sys.stderr)
 108         traceback.print_exc()
 109         sys.exit(1)
 110
 111
 112 def apply_fixes(args, tmpdir):
 113     """Calls clang-apply-fixes on a given directory. Deletes the dir when done."""
 114     invocation = [args.clang_apply_replacements_binary]
 115     if args.format:
 116         invocation.append('-format')
 117     if args.style:
 118         invocation.append('-style=' + args.style)
 119     invocation.append(tmpdir)
 120     subprocess.call(invocation)
 121
 122
 123 def run_tidy(args, tmpdir, build_path, queue):
 124     """Takes filenames out of queue and runs clang-tidy on them."""
 125     while True:
 126         name = queue.get()
 127         invocation = get_tidy_invocation(name, args.clang_tidy_binary, args.checks,
 128                                          args.warningsaserrors, tmpdir, build_path,
 129                                          args.header_filter, args.extra_arg,
 130                                          args.extra_arg_before, args.quiet)
 131         if not args.no_command_on_stdout:
 132             sys.stdout.write(' '.join(invocation) + '\n')
 133         try:
 134             subprocess.check_call(invocation)
 135         except subprocess.CalledProcessError:
 136             queue.has_error = True
 137         queue.task_done()
 138
 139
 140 def main():
 141     parser = argparse.ArgumentParser(description='Runs clang-tidy over all files '
 142                                                  'in a compilation database. Requires '
 143                                                  'clang-tidy and clang-apply-replacements in '
 144                                                  '$PATH.')
 145     parser.add_argument('-clang-tidy-binary', metavar='PATH',
 146                         default='clang-tidy',
 147                         help='path to clang-tidy binary')
 148     parser.add_argument('-clang-apply-replacements-binary', metavar='PATH',
 149                         default='clang-apply-replacements',
 150                         help='path to clang-apply-replacements binary')
 151     parser.add_argument('-checks', default=None,
 152                         help='checks filter, when not specified, use clang-tidy '
 153                              'default')
 154     parser.add_argument('-warningsaserrors', default=None,
 155                         help='warnings-as-errors filter, when not specified, '
 156                              'use clang-tidy default')
 157     parser.add_argument('-header-filter', default=None,
 158                         help='regular expression matching the names of the '
 159                              'headers to output diagnostics from. Diagnostics from '
 160                              'the main file of each translation unit are always '
 161                              'displayed.')
 162     parser.add_argument('-j', type=int, default=0,
 163                         help='number of tidy instances to be run in parallel.')
 164     parser.add_argument('files', nargs='*', default=['.*'],
 165                         help='files to be processed (regex on path)')
 166     parser.add_argument('-fix', action='store_true', help='apply fix-its')
 167     parser.add_argument('-format', action='store_true', help='Reformat code '
 168                                                              'after applying fixes')
 169     parser.add_argument('-style', default='file', help='The style of reformat '
 170                                                        'code after applying fixes')
 171     parser.add_argument('-p', dest='build_path',
 172                         help='Path used to read a compile command database.')
 173     parser.add_argument('-extra-arg', dest='extra_arg',
 174                         action='append', default=[],
 175                         help='Additional argument to append to the compiler '
 176                              'command line.')
 177     parser.add_argument('-extra-arg-before', dest='extra_arg_before',
 178                         action='append', default=[],
 179                         help='Additional argument to prepend to the compiler '
 180                              'command line.')
 181     parser.add_argument('-quiet', action='store_true',
 182                         help='Run clang-tidy in quiet mode')
 183     parser.add_argument('-no-command-on-stdout', action='store_true',
 184                         help='Run clang-tidy without printing invocation on '
 185                              'stdout')
 186     args = parser.parse_args()
 187
 188     db_path = 'compile_commands.json'
 189
 190     if args.build_path is not None:
 191         build_path = args.build_path
 192     else:
 193         # Find our database
 194         build_path = find_compilation_database(db_path)
 195
 196     try:
 197         invocation = [args.clang_tidy_binary, '-list-checks', '-p=' + build_path]
 198         if args.checks:
 199             invocation.append('-checks=' + args.checks)
 200         if args.warningsaserrors:
 201             invocation.append('-warnings-as-errors=' + args.warningsaserrors)
 202         invocation.append('-')
 203         print(subprocess.check_output(invocation))
 204     except:
 205         print("Unable to run clang-tidy.", file=sys.stderr)
 206         sys.exit(1)
 207
 208     # Load the database and extract all files.
 209     database = json.load(open(os.path.join(build_path, db_path)))
 210     files = [entry['file'] for entry in database]
 211
 212     max_task = args.j
 213     if max_task == 0:
 214         max_task = multiprocessing.cpu_count()
 215
 216     tmpdir = None
 217     if args.fix:
 218         check_clang_apply_replacements_binary(args)
 219         tmpdir = tempfile.mkdtemp()
 220
 221     # Build up a big regexy filter from all command line arguments.
 222     file_name_re = re.compile('|'.join(args.files))
 223
 224     try:
 225         # Spin up a bunch of tidy-launching threads.
 226         queue = TidyQueue(max_task)
 227         for _ in range(max_task):
 228             t = threading.Thread(target=run_tidy,
 229                                  args=(args, tmpdir, build_path, queue))
 230             t.daemon = True
 231             t.start()
 232
 233         # Fill the queue with files.
 234         for name in files:
 235             if file_name_re.search(name):
 236                 queue.put(name)
 237
 238         # Wait for all threads to be done.
 239         queue.join()
 240
 241         # If one clang-tidy process found and error, exit with non-zero
 242         # status
 243         if queue.has_error:
 244             sys.exit(2)
 245
 246     except KeyboardInterrupt:
 247         # This is a sad hack. Unfortunately subprocess goes
 248         # bonkers with ctrl-c and we start forking merrily.
 249         print('\nCtrl-C detected, goodbye.')
 250         if args.fix:
 251             shutil.rmtree(tmpdir)
 252         os.kill(0, 9)
 253
 254     if args.fix:
 255         print('Applying fixes ...')
 256         successfully_applied = False
 257
 258         try:
 259             apply_fixes(args, tmpdir)
 260             successfully_applied = True
 261         except:
 262             print('Error applying fixes.\n', file=sys.stderr)
 263             traceback.print_exc()
 264
 265         shutil.rmtree(tmpdir)
 266         if not successfully_applied:
 267             sys.exit(1)
 268
 269
 270 if __name__ == '__main__':
 271     main()