2 # -*- coding: utf-8 -*-
4 # This script computes the new "current" toolstate for the toolstate repo (not to be
5 # confused with publishing the test results, which happens in `src/bootstrap/toolstate.rs`).
6 # It gets called from `src/ci/publish_toolstate.sh` when a new commit lands on `master`
7 # (i.e., after it passed all checks on `auto`).
9 from __future__ import print_function
20 from urllib2 import HTTPError
22 import urllib.request as urllib2
23 from urllib.error import HTTPError
29 # List of people to ping when the status of a tool or a book changed.
30 # These should be collaborators of the rust-lang/rust repository (with at least
31 # read privileges on it). CI will fail otherwise.
33 'miri': {'oli-obk', 'RalfJung'},
34 'book': {'carols10cents'},
35 'nomicon': {'frewsxcv', 'Gankra', 'JohnTitor'},
36 'reference': {'Havvy', 'matthewjasper', 'ehuss'},
37 'rust-by-example': {'marioidival'},
38 'embedded-book': {'adamgreig', 'andre-richter', 'jamesmunns', 'therealprof'},
39 'edition-guide': {'ehuss'},
40 'rustc-dev-guide': {'spastorino', 'amanjeev', 'JohnTitor'},
44 'miri': ['A-miri', 'C-bug'],
47 'reference': ['C-bug'],
48 'rust-by-example': ['C-bug'],
49 'embedded-book': ['C-bug'],
50 'edition-guide': ['C-bug'],
51 'rustc-dev-guide': ['C-bug'],
55 'miri': 'https://github.com/rust-lang/miri',
56 'book': 'https://github.com/rust-lang/book',
57 'nomicon': 'https://github.com/rust-lang/nomicon',
58 'reference': 'https://github.com/rust-lang/reference',
59 'rust-by-example': 'https://github.com/rust-lang/rust-by-example',
60 'embedded-book': 'https://github.com/rust-embedded/book',
61 'edition-guide': 'https://github.com/rust-lang/edition-guide',
62 'rustc-dev-guide': 'https://github.com/rust-lang/rustc-dev-guide',
65 def load_json_from_response(resp):
66 # type: (typing.Any) -> typing.Any
68 if isinstance(content, bytes):
69 content_str = content.decode('utf-8')
71 print("Refusing to decode " + str(type(content)) + " to str")
72 return json.loads(content_str)
74 def validate_maintainers(repo, github_token):
75 # type: (str, str) -> None
76 '''Ensure all maintainers are assignable on a GitHub repo'''
77 next_link_re = re.compile(r'<([^>]+)>; rel="next"')
79 # Load the list of assignable people in the GitHub repo
80 assignable = [] # type: typing.List[str]
81 url = 'https://api.github.com/repos/' \
82 + '%s/collaborators?per_page=100' % repo # type: typing.Optional[str]
83 while url is not None:
84 response = urllib2.urlopen(urllib2.Request(url, headers={
85 'Authorization': 'token ' + github_token,
86 # Properly load nested teams.
87 'Accept': 'application/vnd.github.hellcat-preview+json',
89 assignable.extend(user['login'] for user in load_json_from_response(response))
90 # Load the next page if available
92 link_header = response.headers.get('Link')
94 matches = next_link_re.match(link_header)
95 if matches is not None:
96 url = matches.group(1)
99 for tool, maintainers in MAINTAINERS.items():
100 for maintainer in maintainers:
101 if maintainer not in assignable:
104 "error: %s maintainer @%s is not assignable in the %s repo"
105 % (tool, maintainer, repo),
110 print(" To be assignable, a person needs to be explicitly listed as a")
111 print(" collaborator in the repository settings. The simple way to")
112 print(" fix this is to ask someone with 'admin' privileges on the repo")
113 print(" to add the person or whole team as a collaborator with 'read'")
114 print(" privileges. Those privileges don't grant any extra permissions")
115 print(" so it's safe to apply them.")
117 print("The build will fail due to this.")
121 def read_current_status(current_commit, path):
122 # type: (str, str) -> typing.Mapping[str, typing.Any]
123 '''Reads build status of `current_commit` from content of `history/*.tsv`
125 with open(path, 'r') as f:
127 (commit, status) = line.split('\t', 1)
128 if commit == current_commit:
129 return json.loads(status)
135 return os.environ['TOOLSTATE_ISSUES_API_URL']
138 def maybe_delink(message):
140 if os.environ.get('TOOLSTATE_SKIP_MENTIONS') is not None:
141 return message.replace("@", "")
154 # type: (str, str, typing.Iterable[str], str, str, typing.List[str], str) -> None
155 '''Open an issue about the toolstate failure.'''
156 if status == 'test-fail':
157 status_description = 'has failing tests'
159 status_description = 'no longer builds'
160 request = json.dumps({
161 'body': maybe_delink(textwrap.dedent('''\
162 Hello, this is your friendly neighborhood mergebot.
163 After merging PR {}, I observed that the tool {} {}.
164 A follow-up PR to the repository {} is needed to fix the fallout.
166 cc @{}, do you think you would have time to do the follow-up work?
167 If so, that would be great!
169 relevant_pr_number, tool, status_description,
170 REPOS.get(tool), relevant_pr_user
172 'title': '`{}` no longer builds after {}'.format(tool, relevant_pr_number),
173 'assignees': list(assignees),
176 print("Creating issue:\n{}".format(request))
177 response = urllib2.urlopen(urllib2.Request(
181 'Authorization': 'token ' + github_token,
182 'Content-Type': 'application/json',
197 # type: (str, str, str, str, str, str, str) -> str
198 '''Updates `_data/latest.json` to match build result of the given commit.
200 with open('_data/latest.json', 'r+') as f:
201 latest = json.load(f, object_pairs_hook=collections.OrderedDict)
204 os: read_current_status(current_commit, 'history/' + os + '.tsv')
205 for os in ['windows', 'linux']
208 slug = 'rust-lang/rust'
209 message = textwrap.dedent('''\
210 📣 Toolstate changed by {}!
212 Tested on commit {}@{}.
213 Direct link to PR: <{}>
215 ''').format(relevant_pr_number, slug, current_commit, relevant_pr_url)
216 anything_changed = False
217 for status in latest:
218 tool = status['tool']
220 create_issue_for_status = None # set to the status that caused the issue
222 for os, s in current_status.items():
224 new = s.get(tool, old)
226 maintainers = ' '.join('@'+name for name in MAINTAINERS.get(tool, ()))
227 # comparing the strings, but they are ordered appropriately:
228 # "test-pass" > "test-fail" > "build-fail"
230 # things got fixed or at least the status quo improved
232 message += '🎉 {} on {}: {} → {} (cc {}).\n' \
233 .format(tool, os, old, new, maintainers)
235 # tests or builds are failing and were not failing before
237 title = '💔 {} on {}: {} → {}' \
238 .format(tool, os, old, new)
239 message += '{} (cc {}).\n' \
240 .format(title, maintainers)
241 # See if we need to create an issue.
243 # Create issue if tests used to pass before. Don't open a *second*
244 # issue when we regress from "test-fail" to "build-fail".
245 if old == 'test-pass':
246 create_issue_for_status = new
248 # Create issue if things no longer build.
249 # (No issue for mere test failures to avoid spurious issues.)
250 if new == 'build-fail':
251 create_issue_for_status = new
253 if create_issue_for_status is not None:
256 tool, create_issue_for_status, MAINTAINERS.get(tool, ()),
257 relevant_pr_number, relevant_pr_user, LABELS.get(tool, []),
260 except HTTPError as e:
261 # network errors will simply end up not creating an issue, but that's better
262 # than failing the entire build job
263 print("HTTPError when creating issue for status regression: {0}\n{1!r}"
264 .format(e, e.read()))
266 print("I/O error when creating issue for status regression: {0}".format(e))
268 print("Unexpected error when creating issue for status regression: {0}"
269 .format(sys.exc_info()[0]))
273 status['commit'] = current_commit
274 status['datetime'] = current_datetime
275 anything_changed = True
277 if not anything_changed:
282 json.dump(latest, f, indent=4, separators=(',', ': '))
286 # Warning: Do not try to add a function containing the body of this try block.
287 # There are variables declared within that are implicitly global; it is unknown
288 # which ones precisely but at least this is true for `github_token`.
290 if __name__ != '__main__':
292 repo = os.environ.get('TOOLSTATE_VALIDATE_MAINTAINERS_REPO')
294 github_token = os.environ.get('TOOLSTATE_REPO_ACCESS_TOKEN')
296 # FIXME: This is currently broken. Starting on 2021-09-15, GitHub
297 # seems to have changed it so that to list the collaborators
298 # requires admin permissions. I think this will probably just need
299 # to be removed since we are probably not going to use an admin
300 # token, and I don't see another way to do this.
301 print('maintainer validation disabled')
302 # validate_maintainers(repo, github_token)
304 print('skipping toolstate maintainers validation since no GitHub token is present')
305 # When validating maintainers don't run the full script.
308 cur_commit = sys.argv[1]
309 cur_datetime = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
310 cur_commit_msg = sys.argv[2]
311 save_message_to_path = sys.argv[3]
312 github_token = sys.argv[4]
314 # assume that PR authors are also owners of the repo where the branch lives
315 relevant_pr_match = re.search(
316 r'Auto merge of #([0-9]+) - ([^:]+):[^,]+, r=(\S+)',
319 if relevant_pr_match:
320 number = relevant_pr_match.group(1)
321 relevant_pr_user = relevant_pr_match.group(2)
322 relevant_pr_number = 'rust-lang/rust#' + number
323 relevant_pr_url = 'https://github.com/rust-lang/rust/pull/' + number
324 pr_reviewer = relevant_pr_match.group(3)
327 relevant_pr_user = 'ghost'
328 relevant_pr_number = '<unknown PR>'
329 relevant_pr_url = '<unknown>'
330 pr_reviewer = 'ghost'
332 message = update_latest(
342 print('<Nothing changed>')
348 print('Dry run only, not committing anything')
351 with open(save_message_to_path, 'w') as f:
354 # Write the toolstate comment on the PR as well.
355 issue_url = gh_url() + '/{}/comments'.format(number)
356 response = urllib2.urlopen(urllib2.Request(
358 json.dumps({'body': maybe_delink(message)}).encode(),
360 'Authorization': 'token ' + github_token,
361 'Content-Type': 'application/json',
365 except HTTPError as e:
366 print("HTTPError: %s\n%r" % (e, e.read()))