2 # -*- coding: utf-8 -*-
4 # This script computes the new "current" toolstate for the toolstate repo (not to be
5 # confused with publishing the test results, which happens in `src/bootstrap/toolstate.rs`).
6 # It gets called from `src/ci/publish_toolstate.sh` when a new commit lands on `master`
7 # (i.e., after it passed all checks on `auto`).
9 from __future__ import print_function
20 from urllib2 import HTTPError
22 import urllib.request as urllib2
23 from urllib.error import HTTPError
29 # List of people to ping when the status of a tool or a book changed.
30 # These should be collaborators of the rust-lang/rust repository (with at least
31 # read privileges on it). CI will fail otherwise.
33 'miri': {'oli-obk', 'RalfJung'},
35 'book': {'carols10cents'},
36 'nomicon': {'frewsxcv', 'Gankra', 'JohnTitor'},
37 'reference': {'Havvy', 'matthewjasper', 'ehuss'},
38 'rust-by-example': {'marioidival'},
39 'embedded-book': {'adamgreig', 'andre-richter', 'jamesmunns', 'therealprof'},
40 'edition-guide': {'ehuss'},
41 'rustc-dev-guide': {'spastorino', 'amanjeev', 'JohnTitor'},
45 'miri': ['A-miri', 'C-bug'],
46 'rls': ['A-rls', 'C-bug'],
49 'reference': ['C-bug'],
50 'rust-by-example': ['C-bug'],
51 'embedded-book': ['C-bug'],
52 'edition-guide': ['C-bug'],
53 'rustc-dev-guide': ['C-bug'],
57 'miri': 'https://github.com/rust-lang/miri',
58 'rls': 'https://github.com/rust-lang/rls',
59 'book': 'https://github.com/rust-lang/book',
60 'nomicon': 'https://github.com/rust-lang/nomicon',
61 'reference': 'https://github.com/rust-lang/reference',
62 'rust-by-example': 'https://github.com/rust-lang/rust-by-example',
63 'embedded-book': 'https://github.com/rust-embedded/book',
64 'edition-guide': 'https://github.com/rust-lang/edition-guide',
65 'rustc-dev-guide': 'https://github.com/rust-lang/rustc-dev-guide',
68 def load_json_from_response(resp):
69 # type: (typing.Any) -> typing.Any
71 if isinstance(content, bytes):
72 content_str = content.decode('utf-8')
74 print("Refusing to decode " + str(type(content)) + " to str")
75 return json.loads(content_str)
77 def validate_maintainers(repo, github_token):
78 # type: (str, str) -> None
79 '''Ensure all maintainers are assignable on a GitHub repo'''
80 next_link_re = re.compile(r'<([^>]+)>; rel="next"')
82 # Load the list of assignable people in the GitHub repo
83 assignable = [] # type: typing.List[str]
84 url = 'https://api.github.com/repos/' \
85 + '%s/collaborators?per_page=100' % repo # type: typing.Optional[str]
86 while url is not None:
87 response = urllib2.urlopen(urllib2.Request(url, headers={
88 'Authorization': 'token ' + github_token,
89 # Properly load nested teams.
90 'Accept': 'application/vnd.github.hellcat-preview+json',
92 assignable.extend(user['login'] for user in load_json_from_response(response))
93 # Load the next page if available
95 link_header = response.headers.get('Link')
97 matches = next_link_re.match(link_header)
98 if matches is not None:
99 url = matches.group(1)
102 for tool, maintainers in MAINTAINERS.items():
103 for maintainer in maintainers:
104 if maintainer not in assignable:
107 "error: %s maintainer @%s is not assignable in the %s repo"
108 % (tool, maintainer, repo),
113 print(" To be assignable, a person needs to be explicitly listed as a")
114 print(" collaborator in the repository settings. The simple way to")
115 print(" fix this is to ask someone with 'admin' privileges on the repo")
116 print(" to add the person or whole team as a collaborator with 'read'")
117 print(" privileges. Those privileges don't grant any extra permissions")
118 print(" so it's safe to apply them.")
120 print("The build will fail due to this.")
124 def read_current_status(current_commit, path):
125 # type: (str, str) -> typing.Mapping[str, typing.Any]
126 '''Reads build status of `current_commit` from content of `history/*.tsv`
128 with open(path, 'r') as f:
130 (commit, status) = line.split('\t', 1)
131 if commit == current_commit:
132 return json.loads(status)
138 return os.environ['TOOLSTATE_ISSUES_API_URL']
141 def maybe_delink(message):
143 if os.environ.get('TOOLSTATE_SKIP_MENTIONS') is not None:
144 return message.replace("@", "")
157 # type: (str, str, typing.Iterable[str], str, str, typing.List[str], str) -> None
158 '''Open an issue about the toolstate failure.'''
159 if status == 'test-fail':
160 status_description = 'has failing tests'
162 status_description = 'no longer builds'
163 request = json.dumps({
164 'body': maybe_delink(textwrap.dedent('''\
165 Hello, this is your friendly neighborhood mergebot.
166 After merging PR {}, I observed that the tool {} {}.
167 A follow-up PR to the repository {} is needed to fix the fallout.
169 cc @{}, do you think you would have time to do the follow-up work?
170 If so, that would be great!
172 relevant_pr_number, tool, status_description,
173 REPOS.get(tool), relevant_pr_user
175 'title': '`{}` no longer builds after {}'.format(tool, relevant_pr_number),
176 'assignees': list(assignees),
179 print("Creating issue:\n{}".format(request))
180 response = urllib2.urlopen(urllib2.Request(
184 'Authorization': 'token ' + github_token,
185 'Content-Type': 'application/json',
200 # type: (str, str, str, str, str, str, str) -> str
201 '''Updates `_data/latest.json` to match build result of the given commit.
203 with open('_data/latest.json', 'r+') as f:
204 latest = json.load(f, object_pairs_hook=collections.OrderedDict)
207 os: read_current_status(current_commit, 'history/' + os + '.tsv')
208 for os in ['windows', 'linux']
211 slug = 'rust-lang/rust'
212 message = textwrap.dedent('''\
213 📣 Toolstate changed by {}!
215 Tested on commit {}@{}.
216 Direct link to PR: <{}>
218 ''').format(relevant_pr_number, slug, current_commit, relevant_pr_url)
219 anything_changed = False
220 for status in latest:
221 tool = status['tool']
223 create_issue_for_status = None # set to the status that caused the issue
225 for os, s in current_status.items():
227 new = s.get(tool, old)
229 maintainers = ' '.join('@'+name for name in MAINTAINERS.get(tool, ()))
230 # comparing the strings, but they are ordered appropriately:
231 # "test-pass" > "test-fail" > "build-fail"
233 # things got fixed or at least the status quo improved
235 message += '🎉 {} on {}: {} → {} (cc {}).\n' \
236 .format(tool, os, old, new, maintainers)
238 # tests or builds are failing and were not failing before
240 title = '💔 {} on {}: {} → {}' \
241 .format(tool, os, old, new)
242 message += '{} (cc {}).\n' \
243 .format(title, maintainers)
244 # See if we need to create an issue.
246 # Create issue if tests used to pass before. Don't open a *second*
247 # issue when we regress from "test-fail" to "build-fail".
248 if old == 'test-pass':
249 create_issue_for_status = new
251 # Create issue if things no longer build.
252 # (No issue for mere test failures to avoid spurious issues.)
253 if new == 'build-fail':
254 create_issue_for_status = new
256 if create_issue_for_status is not None:
259 tool, create_issue_for_status, MAINTAINERS.get(tool, ()),
260 relevant_pr_number, relevant_pr_user, LABELS.get(tool, []),
263 except HTTPError as e:
264 # network errors will simply end up not creating an issue, but that's better
265 # than failing the entire build job
266 print("HTTPError when creating issue for status regression: {0}\n{1!r}"
267 .format(e, e.read()))
269 print("I/O error when creating issue for status regression: {0}".format(e))
271 print("Unexpected error when creating issue for status regression: {0}"
272 .format(sys.exc_info()[0]))
276 status['commit'] = current_commit
277 status['datetime'] = current_datetime
278 anything_changed = True
280 if not anything_changed:
285 json.dump(latest, f, indent=4, separators=(',', ': '))
289 # Warning: Do not try to add a function containing the body of this try block.
290 # There are variables declared within that are implicitly global; it is unknown
291 # which ones precisely but at least this is true for `github_token`.
293 if __name__ != '__main__':
295 repo = os.environ.get('TOOLSTATE_VALIDATE_MAINTAINERS_REPO')
297 github_token = os.environ.get('TOOLSTATE_REPO_ACCESS_TOKEN')
299 # FIXME: This is currently broken. Starting on 2021-09-15, GitHub
300 # seems to have changed it so that to list the collaborators
301 # requires admin permissions. I think this will probably just need
302 # to be removed since we are probably not going to use an admin
303 # token, and I don't see another way to do this.
304 print('maintainer validation disabled')
305 # validate_maintainers(repo, github_token)
307 print('skipping toolstate maintainers validation since no GitHub token is present')
308 # When validating maintainers don't run the full script.
311 cur_commit = sys.argv[1]
312 cur_datetime = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
313 cur_commit_msg = sys.argv[2]
314 save_message_to_path = sys.argv[3]
315 github_token = sys.argv[4]
317 # assume that PR authors are also owners of the repo where the branch lives
318 relevant_pr_match = re.search(
319 r'Auto merge of #([0-9]+) - ([^:]+):[^,]+, r=(\S+)',
322 if relevant_pr_match:
323 number = relevant_pr_match.group(1)
324 relevant_pr_user = relevant_pr_match.group(2)
325 relevant_pr_number = 'rust-lang/rust#' + number
326 relevant_pr_url = 'https://github.com/rust-lang/rust/pull/' + number
327 pr_reviewer = relevant_pr_match.group(3)
330 relevant_pr_user = 'ghost'
331 relevant_pr_number = '<unknown PR>'
332 relevant_pr_url = '<unknown>'
333 pr_reviewer = 'ghost'
335 message = update_latest(
345 print('<Nothing changed>')
351 print('Dry run only, not committing anything')
354 with open(save_message_to_path, 'w') as f:
357 # Write the toolstate comment on the PR as well.
358 issue_url = gh_url() + '/{}/comments'.format(number)
359 response = urllib2.urlopen(urllib2.Request(
361 json.dumps({'body': maybe_delink(message)}).encode(),
363 'Authorization': 'token ' + github_token,
364 'Content-Type': 'application/json',
368 except HTTPError as e:
369 print("HTTPError: %s\n%r" % (e, e.read()))