2 # -*- coding: utf-8 -*-
4 # This script computes the new "current" toolstate for the toolstate repo (not to be
5 # confused with publishing the test results, which happens in `src/bootstrap/toolstate.rs`).
6 # It gets called from `src/ci/publish_toolstate.sh` when a new commit lands on `master`
7 # (i.e., after it passed all checks on `auto`).
9 from __future__ import print_function
20 from urllib2 import HTTPError
22 import urllib.request as urllib2
23 from urllib.error import HTTPError
29 # List of people to ping when the status of a tool or a book changed.
30 # These should be collaborators of the rust-lang/rust repository (with at least
31 # read privileges on it). CI will fail otherwise.
33 'miri': {'oli-obk', 'RalfJung', 'eddyb'},
35 'rustfmt': {'topecongiro', 'calebcartwright'},
36 'book': {'carols10cents', 'steveklabnik'},
37 'nomicon': {'frewsxcv', 'Gankra', 'JohnTitor'},
38 'reference': {'steveklabnik', 'Havvy', 'matthewjasper', 'ehuss'},
39 'rust-by-example': {'steveklabnik', 'marioidival'},
40 'embedded-book': {'adamgreig', 'andre-richter', 'jamesmunns', 'therealprof'},
41 'edition-guide': {'ehuss', 'steveklabnik'},
42 'rustc-dev-guide': {'spastorino', 'amanjeev', 'JohnTitor'},
46 'miri': ['A-miri', 'C-bug'],
47 'rls': ['A-rls', 'C-bug'],
48 'rustfmt': ['A-rustfmt', 'C-bug'],
51 'reference': ['C-bug'],
52 'rust-by-example': ['C-bug'],
53 'embedded-book': ['C-bug'],
54 'edition-guide': ['C-bug'],
55 'rustc-dev-guide': ['C-bug'],
59 'miri': 'https://github.com/rust-lang/miri',
60 'rls': 'https://github.com/rust-lang/rls',
61 'rustfmt': 'https://github.com/rust-lang/rustfmt',
62 'book': 'https://github.com/rust-lang/book',
63 'nomicon': 'https://github.com/rust-lang/nomicon',
64 'reference': 'https://github.com/rust-lang/reference',
65 'rust-by-example': 'https://github.com/rust-lang/rust-by-example',
66 'embedded-book': 'https://github.com/rust-embedded/book',
67 'edition-guide': 'https://github.com/rust-lang/edition-guide',
68 'rustc-dev-guide': 'https://github.com/rust-lang/rustc-dev-guide',
71 def load_json_from_response(resp):
72 # type: (typing.Any) -> typing.Any
74 if isinstance(content, bytes):
75 content_str = content.decode('utf-8')
77 print("Refusing to decode " + str(type(content)) + " to str")
78 return json.loads(content_str)
80 def validate_maintainers(repo, github_token):
81 # type: (str, str) -> None
82 '''Ensure all maintainers are assignable on a GitHub repo'''
83 next_link_re = re.compile(r'<([^>]+)>; rel="next"')
85 # Load the list of assignable people in the GitHub repo
86 assignable = [] # type: typing.List[str]
87 url = 'https://api.github.com/repos/' \
88 + '%s/collaborators?per_page=100' % repo # type: typing.Optional[str]
89 while url is not None:
90 response = urllib2.urlopen(urllib2.Request(url, headers={
91 'Authorization': 'token ' + github_token,
92 # Properly load nested teams.
93 'Accept': 'application/vnd.github.hellcat-preview+json',
95 assignable.extend(user['login'] for user in load_json_from_response(response))
96 # Load the next page if available
98 link_header = response.headers.get('Link')
100 matches = next_link_re.match(link_header)
101 if matches is not None:
102 url = matches.group(1)
105 for tool, maintainers in MAINTAINERS.items():
106 for maintainer in maintainers:
107 if maintainer not in assignable:
110 "error: %s maintainer @%s is not assignable in the %s repo"
111 % (tool, maintainer, repo),
116 print(" To be assignable, a person needs to be explicitly listed as a")
117 print(" collaborator in the repository settings. The simple way to")
118 print(" fix this is to ask someone with 'admin' privileges on the repo")
119 print(" to add the person or whole team as a collaborator with 'read'")
120 print(" privileges. Those privileges don't grant any extra permissions")
121 print(" so it's safe to apply them.")
123 print("The build will fail due to this.")
127 def read_current_status(current_commit, path):
128 # type: (str, str) -> typing.Mapping[str, typing.Any]
129 '''Reads build status of `current_commit` from content of `history/*.tsv`
131 with open(path, 'r') as f:
133 (commit, status) = line.split('\t', 1)
134 if commit == current_commit:
135 return json.loads(status)
141 return os.environ['TOOLSTATE_ISSUES_API_URL']
144 def maybe_delink(message):
146 if os.environ.get('TOOLSTATE_SKIP_MENTIONS') is not None:
147 return message.replace("@", "")
160 # type: (str, str, typing.Iterable[str], str, str, typing.List[str], str) -> None
161 '''Open an issue about the toolstate failure.'''
162 if status == 'test-fail':
163 status_description = 'has failing tests'
165 status_description = 'no longer builds'
166 request = json.dumps({
167 'body': maybe_delink(textwrap.dedent('''\
168 Hello, this is your friendly neighborhood mergebot.
169 After merging PR {}, I observed that the tool {} {}.
170 A follow-up PR to the repository {} is needed to fix the fallout.
172 cc @{}, do you think you would have time to do the follow-up work?
173 If so, that would be great!
175 relevant_pr_number, tool, status_description,
176 REPOS.get(tool), relevant_pr_user
178 'title': '`{}` no longer builds after {}'.format(tool, relevant_pr_number),
179 'assignees': list(assignees),
182 print("Creating issue:\n{}".format(request))
183 response = urllib2.urlopen(urllib2.Request(
187 'Authorization': 'token ' + github_token,
188 'Content-Type': 'application/json',
203 # type: (str, str, str, str, str, str, str) -> str
204 '''Updates `_data/latest.json` to match build result of the given commit.
206 with open('_data/latest.json', 'r+') as f:
207 latest = json.load(f, object_pairs_hook=collections.OrderedDict)
210 os: read_current_status(current_commit, 'history/' + os + '.tsv')
211 for os in ['windows', 'linux']
214 slug = 'rust-lang/rust'
215 message = textwrap.dedent('''\
216 📣 Toolstate changed by {}!
218 Tested on commit {}@{}.
219 Direct link to PR: <{}>
221 ''').format(relevant_pr_number, slug, current_commit, relevant_pr_url)
222 anything_changed = False
223 for status in latest:
224 tool = status['tool']
226 create_issue_for_status = None # set to the status that caused the issue
228 for os, s in current_status.items():
230 new = s.get(tool, old)
232 maintainers = ' '.join('@'+name for name in MAINTAINERS.get(tool, ()))
233 # comparing the strings, but they are ordered appropriately:
234 # "test-pass" > "test-fail" > "build-fail"
236 # things got fixed or at least the status quo improved
238 message += '🎉 {} on {}: {} → {} (cc {}).\n' \
239 .format(tool, os, old, new, maintainers)
241 # tests or builds are failing and were not failing before
243 title = '💔 {} on {}: {} → {}' \
244 .format(tool, os, old, new)
245 message += '{} (cc {}).\n' \
246 .format(title, maintainers)
247 # See if we need to create an issue.
249 # Create issue if tests used to pass before. Don't open a *second*
250 # issue when we regress from "test-fail" to "build-fail".
251 if old == 'test-pass':
252 create_issue_for_status = new
254 # Create issue if things no longer build.
255 # (No issue for mere test failures to avoid spurious issues.)
256 if new == 'build-fail':
257 create_issue_for_status = new
259 if create_issue_for_status is not None:
262 tool, create_issue_for_status, MAINTAINERS.get(tool, ()),
263 relevant_pr_number, relevant_pr_user, LABELS.get(tool, []),
266 except HTTPError as e:
267 # network errors will simply end up not creating an issue, but that's better
268 # than failing the entire build job
269 print("HTTPError when creating issue for status regression: {0}\n{1!r}"
270 .format(e, e.read()))
272 print("I/O error when creating issue for status regression: {0}".format(e))
274 print("Unexpected error when creating issue for status regression: {0}"
275 .format(sys.exc_info()[0]))
279 status['commit'] = current_commit
280 status['datetime'] = current_datetime
281 anything_changed = True
283 if not anything_changed:
288 json.dump(latest, f, indent=4, separators=(',', ': '))
292 # Warning: Do not try to add a function containing the body of this try block.
293 # There are variables declared within that are implicitly global; it is unknown
294 # which ones precisely but at least this is true for `github_token`.
296 if __name__ != '__main__':
298 repo = os.environ.get('TOOLSTATE_VALIDATE_MAINTAINERS_REPO')
300 github_token = os.environ.get('TOOLSTATE_REPO_ACCESS_TOKEN')
302 # FIXME: This is currently broken. Starting on 2021-09-15, GitHub
303 # seems to have changed it so that to list the collaborators
304 # requires admin permissions. I think this will probably just need
305 # to be removed since we are probably not going to use an admin
306 # token, and I don't see another way to do this.
307 print('maintainer validation disabled')
308 # validate_maintainers(repo, github_token)
310 print('skipping toolstate maintainers validation since no GitHub token is present')
311 # When validating maintainers don't run the full script.
314 cur_commit = sys.argv[1]
315 cur_datetime = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
316 cur_commit_msg = sys.argv[2]
317 save_message_to_path = sys.argv[3]
318 github_token = sys.argv[4]
320 # assume that PR authors are also owners of the repo where the branch lives
321 relevant_pr_match = re.search(
322 r'Auto merge of #([0-9]+) - ([^:]+):[^,]+, r=(\S+)',
325 if relevant_pr_match:
326 number = relevant_pr_match.group(1)
327 relevant_pr_user = relevant_pr_match.group(2)
328 relevant_pr_number = 'rust-lang/rust#' + number
329 relevant_pr_url = 'https://github.com/rust-lang/rust/pull/' + number
330 pr_reviewer = relevant_pr_match.group(3)
333 relevant_pr_user = 'ghost'
334 relevant_pr_number = '<unknown PR>'
335 relevant_pr_url = '<unknown>'
336 pr_reviewer = 'ghost'
338 message = update_latest(
348 print('<Nothing changed>')
354 print('Dry run only, not committing anything')
357 with open(save_message_to_path, 'w') as f:
360 # Write the toolstate comment on the PR as well.
361 issue_url = gh_url() + '/{}/comments'.format(number)
362 response = urllib2.urlopen(urllib2.Request(
364 json.dumps({'body': maybe_delink(message)}).encode(),
366 'Authorization': 'token ' + github_token,
367 'Content-Type': 'application/json',
371 except HTTPError as e:
372 print("HTTPError: %s\n%r" % (e, e.read()))