2 # -*- coding: utf-8 -*-
4 # This script computes the new "current" toolstate for the toolstate repo (not to be
5 # confused with publishing the test results, which happens in `src/bootstrap/toolstate.rs`).
6 # It gets called from `src/ci/publish_toolstate.sh` when a new commit lands on `master`
7 # (i.e., after it passed all checks on `auto`).
9 from __future__ import print_function
20 from urllib2 import HTTPError
22 import urllib.request as urllib2
23 from urllib.error import HTTPError
29 # List of people to ping when the status of a tool or a book changed.
30 # These should be collaborators of the rust-lang/rust repository (with at least
31 # read privileges on it). CI will fail otherwise.
33 'miri': {'oli-obk', 'RalfJung', 'eddyb'},
35 'rustfmt': {'topecongiro', 'calebcartwright'},
36 'book': {'carols10cents', 'steveklabnik'},
37 'nomicon': {'frewsxcv', 'Gankra', 'JohnTitor'},
38 'reference': {'steveklabnik', 'Havvy', 'matthewjasper', 'ehuss'},
39 'rust-by-example': {'steveklabnik', 'marioidival'},
40 'embedded-book': {'adamgreig', 'andre-richter', 'jamesmunns', 'therealprof'},
41 'edition-guide': {'ehuss', 'steveklabnik'},
42 'rustc-dev-guide': {'spastorino', 'amanjeev', 'JohnTitor'},
46 'miri': ['A-miri', 'C-bug'],
47 'rls': ['A-rls', 'C-bug'],
48 'rustfmt': ['A-rustfmt', 'C-bug'],
51 'reference': ['C-bug'],
52 'rust-by-example': ['C-bug'],
53 'embedded-book': ['C-bug'],
54 'edition-guide': ['C-bug'],
55 'rustc-dev-guide': ['C-bug'],
59 'miri': 'https://github.com/rust-lang/miri',
60 'rls': 'https://github.com/rust-lang/rls',
61 'rustfmt': 'https://github.com/rust-lang/rustfmt',
62 'book': 'https://github.com/rust-lang/book',
63 'nomicon': 'https://github.com/rust-lang/nomicon',
64 'reference': 'https://github.com/rust-lang/reference',
65 'rust-by-example': 'https://github.com/rust-lang/rust-by-example',
66 'embedded-book': 'https://github.com/rust-embedded/book',
67 'edition-guide': 'https://github.com/rust-lang/edition-guide',
68 'rustc-dev-guide': 'https://github.com/rust-lang/rustc-dev-guide',
71 def load_json_from_response(resp):
72 # type: (typing.Any) -> typing.Any
74 if isinstance(content, bytes):
75 content_str = content.decode('utf-8')
77 print("Refusing to decode " + str(type(content)) + " to str")
78 return json.loads(content_str)
80 def validate_maintainers(repo, github_token):
81 # type: (str, str) -> None
82 '''Ensure all maintainers are assignable on a GitHub repo'''
83 next_link_re = re.compile(r'<([^>]+)>; rel="next"')
85 # Load the list of assignable people in the GitHub repo
86 assignable = [] # type: typing.List[str]
87 url = 'https://api.github.com/repos/' \
88 + '%s/collaborators?per_page=100' % repo # type: typing.Optional[str]
89 while url is not None:
90 response = urllib2.urlopen(urllib2.Request(url, headers={
91 'Authorization': 'token ' + github_token,
92 # Properly load nested teams.
93 'Accept': 'application/vnd.github.hellcat-preview+json',
95 assignable.extend(user['login'] for user in load_json_from_response(response))
96 # Load the next page if available
98 link_header = response.headers.get('Link')
100 matches = next_link_re.match(link_header)
101 if matches is not None:
102 url = matches.group(1)
105 for tool, maintainers in MAINTAINERS.items():
106 for maintainer in maintainers:
107 if maintainer not in assignable:
110 "error: %s maintainer @%s is not assignable in the %s repo"
111 % (tool, maintainer, repo),
116 print(" To be assignable, a person needs to be explicitly listed as a")
117 print(" collaborator in the repository settings. The simple way to")
118 print(" fix this is to ask someone with 'admin' privileges on the repo")
119 print(" to add the person or whole team as a collaborator with 'read'")
120 print(" privileges. Those privileges don't grant any extra permissions")
121 print(" so it's safe to apply them.")
123 print("The build will fail due to this.")
127 def read_current_status(current_commit, path):
128 # type: (str, str) -> typing.Mapping[str, typing.Any]
129 '''Reads build status of `current_commit` from content of `history/*.tsv`
131 with open(path, 'r') as f:
133 (commit, status) = line.split('\t', 1)
134 if commit == current_commit:
135 return json.loads(status)
141 return os.environ['TOOLSTATE_ISSUES_API_URL']
144 def maybe_delink(message):
146 if os.environ.get('TOOLSTATE_SKIP_MENTIONS') is not None:
147 return message.replace("@", "")
160 # type: (str, str, typing.Iterable[str], str, str, typing.List[str], str) -> None
161 '''Open an issue about the toolstate failure.'''
162 if status == 'test-fail':
163 status_description = 'has failing tests'
165 status_description = 'no longer builds'
166 request = json.dumps({
167 'body': maybe_delink(textwrap.dedent('''\
168 Hello, this is your friendly neighborhood mergebot.
169 After merging PR {}, I observed that the tool {} {}.
170 A follow-up PR to the repository {} is needed to fix the fallout.
172 cc @{}, do you think you would have time to do the follow-up work?
173 If so, that would be great!
175 relevant_pr_number, tool, status_description,
176 REPOS.get(tool), relevant_pr_user
178 'title': '`{}` no longer builds after {}'.format(tool, relevant_pr_number),
179 'assignees': list(assignees),
182 print("Creating issue:\n{}".format(request))
183 response = urllib2.urlopen(urllib2.Request(
187 'Authorization': 'token ' + github_token,
188 'Content-Type': 'application/json',
203 # type: (str, str, str, str, str, str, str) -> str
204 '''Updates `_data/latest.json` to match build result of the given commit.
206 with open('_data/latest.json', 'r+') as f:
207 latest = json.load(f, object_pairs_hook=collections.OrderedDict)
210 os: read_current_status(current_commit, 'history/' + os + '.tsv')
211 for os in ['windows', 'linux']
214 slug = 'rust-lang/rust'
215 message = textwrap.dedent('''\
216 📣 Toolstate changed by {}!
218 Tested on commit {}@{}.
219 Direct link to PR: <{}>
221 ''').format(relevant_pr_number, slug, current_commit, relevant_pr_url)
222 anything_changed = False
223 for status in latest:
224 tool = status['tool']
226 create_issue_for_status = None # set to the status that caused the issue
228 for os, s in current_status.items():
230 new = s.get(tool, old)
232 maintainers = ' '.join('@'+name for name in MAINTAINERS.get(tool, ()))
233 # comparing the strings, but they are ordered appropriately:
234 # "test-pass" > "test-fail" > "build-fail"
236 # things got fixed or at least the status quo improved
238 message += '🎉 {} on {}: {} → {} (cc {}).\n' \
239 .format(tool, os, old, new, maintainers)
241 # tests or builds are failing and were not failing before
243 title = '💔 {} on {}: {} → {}' \
244 .format(tool, os, old, new)
245 message += '{} (cc {}).\n' \
246 .format(title, maintainers)
247 # See if we need to create an issue.
249 # Create issue if tests used to pass before. Don't open a *second*
250 # issue when we regress from "test-fail" to "build-fail".
251 if old == 'test-pass':
252 create_issue_for_status = new
254 # Create issue if things no longer build.
255 # (No issue for mere test failures to avoid spurious issues.)
256 if new == 'build-fail':
257 create_issue_for_status = new
259 if create_issue_for_status is not None:
262 tool, create_issue_for_status, MAINTAINERS.get(tool, ()),
263 relevant_pr_number, relevant_pr_user, LABELS.get(tool, []),
266 except HTTPError as e:
267 # network errors will simply end up not creating an issue, but that's better
268 # than failing the entire build job
269 print("HTTPError when creating issue for status regression: {0}\n{1!r}"
270 .format(e, e.read()))
272 print("I/O error when creating issue for status regression: {0}".format(e))
274 print("Unexpected error when creating issue for status regression: {0}"
275 .format(sys.exc_info()[0]))
279 status['commit'] = current_commit
280 status['datetime'] = current_datetime
281 anything_changed = True
283 if not anything_changed:
288 json.dump(latest, f, indent=4, separators=(',', ': '))
292 # Warning: Do not try to add a function containing the body of this try block.
293 # There are variables declared within that are implicitly global; it is unknown
294 # which ones precisely but at least this is true for `github_token`.
296 if __name__ != '__main__':
298 repo = os.environ.get('TOOLSTATE_VALIDATE_MAINTAINERS_REPO')
300 github_token = os.environ.get('TOOLSTATE_REPO_ACCESS_TOKEN')
302 validate_maintainers(repo, github_token)
304 print('skipping toolstate maintainers validation since no GitHub token is present')
305 # When validating maintainers don't run the full script.
308 cur_commit = sys.argv[1]
309 cur_datetime = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
310 cur_commit_msg = sys.argv[2]
311 save_message_to_path = sys.argv[3]
312 github_token = sys.argv[4]
314 # assume that PR authors are also owners of the repo where the branch lives
315 relevant_pr_match = re.search(
316 r'Auto merge of #([0-9]+) - ([^:]+):[^,]+, r=(\S+)',
319 if relevant_pr_match:
320 number = relevant_pr_match.group(1)
321 relevant_pr_user = relevant_pr_match.group(2)
322 relevant_pr_number = 'rust-lang/rust#' + number
323 relevant_pr_url = 'https://github.com/rust-lang/rust/pull/' + number
324 pr_reviewer = relevant_pr_match.group(3)
327 relevant_pr_user = 'ghost'
328 relevant_pr_number = '<unknown PR>'
329 relevant_pr_url = '<unknown>'
330 pr_reviewer = 'ghost'
332 message = update_latest(
342 print('<Nothing changed>')
348 print('Dry run only, not committing anything')
351 with open(save_message_to_path, 'w') as f:
354 # Write the toolstate comment on the PR as well.
355 issue_url = gh_url() + '/{}/comments'.format(number)
356 response = urllib2.urlopen(urllib2.Request(
358 json.dumps({'body': maybe_delink(message)}).encode(),
360 'Authorization': 'token ' + github_token,
361 'Content-Type': 'application/json',
365 except HTTPError as e:
366 print("HTTPError: %s\n%r" % (e, e.read()))