src/tools/publish_toolstate.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 ## This script publishes the new "current" toolstate in the toolstate repo (not to be
   5 ## confused with publishing the test results, which happens in
   6 ## `src/ci/docker/x86_64-gnu-tools/checktools.sh`).
   7 ## It is set as callback for `src/ci/docker/x86_64-gnu-tools/repo.sh` by the CI scripts
   8 ## when a new commit lands on `master` (i.e., after it passed all checks on `auto`).
   9
  10 import sys
  11 import re
  12 import os
  13 import json
  14 import datetime
  15 import collections
  16 import textwrap
  17 try:
  18     import urllib2
  19 except ImportError:
  20     import urllib.request as urllib2
  21
  22 # List of people to ping when the status of a tool or a book changed.
  23 MAINTAINERS = {
  24     'miri': '@oli-obk @RalfJung @eddyb',
  25     'clippy-driver': '@Manishearth @llogiq @mcarton @oli-obk @phansch',
  26     'rls': '@Xanewok',
  27     'rustfmt': '@topecongiro',
  28     'book': '@carols10cents @steveklabnik',
  29     'nomicon': '@frewsxcv @Gankro',
  30     'reference': '@steveklabnik @Havvy @matthewjasper @ehuss',
  31     'rust-by-example': '@steveklabnik @marioidival @projektir',
  32     'embedded-book': (
  33         '@adamgreig @andre-richter @jamesmunns @korken89 '
  34         '@ryankurte @thejpster @therealprof'
  35     ),
  36     'edition-guide': '@ehuss @Centril @steveklabnik',
  37     'rustc-guide': '@mark-i-m @spastorino'
  38 }
  39
  40 REPOS = {
  41     'miri': 'https://github.com/rust-lang/miri',
  42     'clippy-driver': 'https://github.com/rust-lang/rust-clippy',
  43     'rls': 'https://github.com/rust-lang/rls',
  44     'rustfmt': 'https://github.com/rust-lang/rustfmt',
  45     'book': 'https://github.com/rust-lang/book',
  46     'nomicon': 'https://github.com/rust-lang-nursery/nomicon',
  47     'reference': 'https://github.com/rust-lang-nursery/reference',
  48     'rust-by-example': 'https://github.com/rust-lang/rust-by-example',
  49     'embedded-book': 'https://github.com/rust-embedded/book',
  50     'edition-guide': 'https://github.com/rust-lang-nursery/edition-guide',
  51     'rustc-guide': 'https://github.com/rust-lang/rustc-guide',
  52 }
  53
  54
  55 def read_current_status(current_commit, path):
  56     '''Reads build status of `current_commit` from content of `history/*.tsv`
  57     '''
  58     with open(path, 'rU') as f:
  59         for line in f:
  60             (commit, status) = line.split('\t', 1)
  61             if commit == current_commit:
  62                 return json.loads(status)
  63     return {}
  64
  65 def gh_url():
  66     return os.environ['TOOLSTATE_ISSUES_API_URL']
  67
  68 def maybe_delink(message):
  69     if os.environ.get('TOOLSTATE_SKIP_MENTIONS') is not None:
  70         return message.replace("@", "")
  71     return message
  72
  73 def issue(
  74     tool,
  75     status,
  76     maintainers,
  77     relevant_pr_number,
  78     relevant_pr_user,
  79     pr_reviewer,
  80 ):
  81     # Open an issue about the toolstate failure.
  82     assignees = [x.strip() for x in maintainers.split('@') if x != '']
  83     if status == 'test-fail':
  84         status_description = 'has failing tests'
  85     else:
  86         status_description = 'no longer builds'
  87     request = json.dumps({
  88         'body': maybe_delink(textwrap.dedent('''\
  89         Hello, this is your friendly neighborhood mergebot.
  90         After merging PR {}, I observed that the tool {} {}.
  91         A follow-up PR to the repository {} is needed to fix the fallout.
  92
  93         cc @{}, do you think you would have time to do the follow-up work?
  94         If so, that would be great!
  95
  96         cc @{}, the PR reviewer, and @rust-lang/compiler -- nominating for prioritization.
  97
  98         ''').format(
  99             relevant_pr_number, tool, status_description,
 100             REPOS.get(tool), relevant_pr_user, pr_reviewer
 101         )),
 102         'title': '`{}` no longer builds after {}'.format(tool, relevant_pr_number),
 103         'assignees': assignees,
 104         'labels': ['T-compiler', 'I-nominated'],
 105     })
 106     print("Creating issue:\n{}".format(request))
 107     response = urllib2.urlopen(urllib2.Request(
 108         gh_url(),
 109         request,
 110         {
 111             'Authorization': 'token ' + github_token,
 112             'Content-Type': 'application/json',
 113         }
 114     ))
 115     response.read()
 116
 117 def update_latest(
 118     current_commit,
 119     relevant_pr_number,
 120     relevant_pr_url,
 121     relevant_pr_user,
 122     pr_reviewer,
 123     current_datetime
 124 ):
 125     '''Updates `_data/latest.json` to match build result of the given commit.
 126     '''
 127     with open('_data/latest.json', 'rb+') as f:
 128         latest = json.load(f, object_pairs_hook=collections.OrderedDict)
 129
 130         current_status = {
 131             os: read_current_status(current_commit, 'history/' + os + '.tsv')
 132             for os in ['windows', 'linux']
 133         }
 134
 135         slug = 'rust-lang/rust'
 136         message = textwrap.dedent('''\
 137             📣 Toolstate changed by {}!
 138
 139             Tested on commit {}@{}.
 140             Direct link to PR: <{}>
 141
 142         ''').format(relevant_pr_number, slug, current_commit, relevant_pr_url)
 143         anything_changed = False
 144         for status in latest:
 145             tool = status['tool']
 146             changed = False
 147             create_issue_for_status = None # set to the status that caused the issue
 148
 149             for os, s in current_status.items():
 150                 old = status[os]
 151                 new = s.get(tool, old)
 152                 status[os] = new
 153                 if new > old: # comparing the strings, but they are ordered appropriately!
 154                     # things got fixed or at least the status quo improved
 155                     changed = True
 156                     message += '🎉 {} on {}: {} → {} (cc {}, @rust-lang/infra).\n' \
 157                         .format(tool, os, old, new, MAINTAINERS.get(tool))
 158                 elif new < old:
 159                     # tests or builds are failing and were not failing before
 160                     changed = True
 161                     title = '💔 {} on {}: {} → {}' \
 162                         .format(tool, os, old, new)
 163                     message += '{} (cc {}, @rust-lang/infra).\n' \
 164                         .format(title, MAINTAINERS.get(tool))
 165                     # Most tools only create issues for build failures.
 166                     # Other failures can be spurious.
 167                     if new == 'build-fail' or (tool == 'miri' and new == 'test-fail'):
 168                         create_issue_for_status = new
 169
 170             if create_issue_for_status is not None:
 171                 try:
 172                     issue(
 173                         tool, create_issue_for_status, MAINTAINERS.get(tool, ''),
 174                         relevant_pr_number, relevant_pr_user, pr_reviewer,
 175                     )
 176                 except urllib2.HTTPError as e:
 177                     # network errors will simply end up not creating an issue, but that's better
 178                     # than failing the entire build job
 179                     print("HTTPError when creating issue for status regression: {0}\n{1}"
 180                           .format(e, e.read()))
 181                 except IOError as e:
 182                     print("I/O error when creating issue for status regression: {0}".format(e))
 183                 except:
 184                     print("Unexpected error when creating issue for status regression: {0}"
 185                           .format(sys.exc_info()[0]))
 186                     raise
 187
 188             if changed:
 189                 status['commit'] = current_commit
 190                 status['datetime'] = current_datetime
 191                 anything_changed = True
 192
 193         if not anything_changed:
 194             return ''
 195
 196         f.seek(0)
 197         f.truncate(0)
 198         json.dump(latest, f, indent=4, separators=(',', ': '))
 199         return message
 200
 201
 202 if __name__ == '__main__':
 203     cur_commit = sys.argv[1]
 204     cur_datetime = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
 205     cur_commit_msg = sys.argv[2]
 206     save_message_to_path = sys.argv[3]
 207     github_token = sys.argv[4]
 208
 209     # assume that PR authors are also owners of the repo where the branch lives
 210     relevant_pr_match = re.search(
 211         r'Auto merge of #([0-9]+) - ([^:]+):[^,]+, r=(\S+)',
 212         cur_commit_msg,
 213     )
 214     if relevant_pr_match:
 215         number = relevant_pr_match.group(1)
 216         relevant_pr_user = relevant_pr_match.group(2)
 217         relevant_pr_number = 'rust-lang/rust#' + number
 218         relevant_pr_url = 'https://github.com/rust-lang/rust/pull/' + number
 219         pr_reviewer = relevant_pr_match.group(3)
 220     else:
 221         number = '-1'
 222         relevant_pr_user = 'ghost'
 223         relevant_pr_number = '<unknown PR>'
 224         relevant_pr_url = '<unknown>'
 225         pr_reviewer = 'ghost'
 226
 227     message = update_latest(
 228         cur_commit,
 229         relevant_pr_number,
 230         relevant_pr_url,
 231         relevant_pr_user,
 232         pr_reviewer,
 233         cur_datetime
 234     )
 235     if not message:
 236         print('<Nothing changed>')
 237         sys.exit(0)
 238
 239     print(message)
 240
 241     if not github_token:
 242         print('Dry run only, not committing anything')
 243         sys.exit(0)
 244
 245     with open(save_message_to_path, 'w') as f:
 246         f.write(message)
 247
 248     # Write the toolstate comment on the PR as well.
 249     issue_url = gh_url() + '/{}/comments'.format(number)
 250     response = urllib2.urlopen(urllib2.Request(
 251         issue_url,
 252         json.dumps({'body': maybe_delink(message)}),
 253         {
 254             'Authorization': 'token ' + github_token,
 255             'Content-Type': 'application/json',
 256         }
 257     ))
 258     response.read()