2 # -*- coding: utf-8 -*-
4 # This script computes the new "current" toolstate for the toolstate repo (not to be
5 # confused with publishing the test results, which happens in `src/bootstrap/toolstate.rs`).
6 # It gets called from `src/ci/publish_toolstate.sh` when a new commit lands on `master`
7 # (i.e., after it passed all checks on `auto`).
9 from __future__ import print_function
20 from urllib2 import HTTPError
22 import urllib.request as urllib2
23 from urllib.error import HTTPError
29 # List of people to ping when the status of a tool or a book changed.
30 # These should be collaborators of the rust-lang/rust repository (with at least
31 # read privileges on it). CI will fail otherwise.
33 'book': {'carols10cents'},
34 'nomicon': {'frewsxcv', 'Gankra', 'JohnTitor'},
35 'reference': {'Havvy', 'matthewjasper', 'ehuss'},
36 'rust-by-example': {'marioidival'},
37 'embedded-book': {'adamgreig', 'andre-richter', 'jamesmunns', 'therealprof'},
38 'edition-guide': {'ehuss'},
39 'rustc-dev-guide': {'spastorino', 'amanjeev', 'JohnTitor'},
45 'reference': ['C-bug'],
46 'rust-by-example': ['C-bug'],
47 'embedded-book': ['C-bug'],
48 'edition-guide': ['C-bug'],
49 'rustc-dev-guide': ['C-bug'],
53 'book': 'https://github.com/rust-lang/book',
54 'nomicon': 'https://github.com/rust-lang/nomicon',
55 'reference': 'https://github.com/rust-lang/reference',
56 'rust-by-example': 'https://github.com/rust-lang/rust-by-example',
57 'embedded-book': 'https://github.com/rust-embedded/book',
58 'edition-guide': 'https://github.com/rust-lang/edition-guide',
59 'rustc-dev-guide': 'https://github.com/rust-lang/rustc-dev-guide',
62 def load_json_from_response(resp):
63 # type: (typing.Any) -> typing.Any
65 if isinstance(content, bytes):
66 content_str = content.decode('utf-8')
68 print("Refusing to decode " + str(type(content)) + " to str")
69 return json.loads(content_str)
71 def validate_maintainers(repo, github_token):
72 # type: (str, str) -> None
73 '''Ensure all maintainers are assignable on a GitHub repo'''
74 next_link_re = re.compile(r'<([^>]+)>; rel="next"')
76 # Load the list of assignable people in the GitHub repo
77 assignable = [] # type: typing.List[str]
78 url = 'https://api.github.com/repos/' \
79 + '%s/collaborators?per_page=100' % repo # type: typing.Optional[str]
80 while url is not None:
81 response = urllib2.urlopen(urllib2.Request(url, headers={
82 'Authorization': 'token ' + github_token,
83 # Properly load nested teams.
84 'Accept': 'application/vnd.github.hellcat-preview+json',
86 assignable.extend(user['login'] for user in load_json_from_response(response))
87 # Load the next page if available
89 link_header = response.headers.get('Link')
91 matches = next_link_re.match(link_header)
92 if matches is not None:
93 url = matches.group(1)
96 for tool, maintainers in MAINTAINERS.items():
97 for maintainer in maintainers:
98 if maintainer not in assignable:
101 "error: %s maintainer @%s is not assignable in the %s repo"
102 % (tool, maintainer, repo),
107 print(" To be assignable, a person needs to be explicitly listed as a")
108 print(" collaborator in the repository settings. The simple way to")
109 print(" fix this is to ask someone with 'admin' privileges on the repo")
110 print(" to add the person or whole team as a collaborator with 'read'")
111 print(" privileges. Those privileges don't grant any extra permissions")
112 print(" so it's safe to apply them.")
114 print("The build will fail due to this.")
118 def read_current_status(current_commit, path):
119 # type: (str, str) -> typing.Mapping[str, typing.Any]
120 '''Reads build status of `current_commit` from content of `history/*.tsv`
122 with open(path, 'r') as f:
124 (commit, status) = line.split('\t', 1)
125 if commit == current_commit:
126 return json.loads(status)
132 return os.environ['TOOLSTATE_ISSUES_API_URL']
135 def maybe_delink(message):
137 if os.environ.get('TOOLSTATE_SKIP_MENTIONS') is not None:
138 return message.replace("@", "")
151 # type: (str, str, typing.Iterable[str], str, str, typing.List[str], str) -> None
152 '''Open an issue about the toolstate failure.'''
153 if status == 'test-fail':
154 status_description = 'has failing tests'
156 status_description = 'no longer builds'
157 request = json.dumps({
158 'body': maybe_delink(textwrap.dedent('''\
159 Hello, this is your friendly neighborhood mergebot.
160 After merging PR {}, I observed that the tool {} {}.
161 A follow-up PR to the repository {} is needed to fix the fallout.
163 cc @{}, do you think you would have time to do the follow-up work?
164 If so, that would be great!
166 relevant_pr_number, tool, status_description,
167 REPOS.get(tool), relevant_pr_user
169 'title': '`{}` no longer builds after {}'.format(tool, relevant_pr_number),
170 'assignees': list(assignees),
173 print("Creating issue:\n{}".format(request))
174 response = urllib2.urlopen(urllib2.Request(
178 'Authorization': 'token ' + github_token,
179 'Content-Type': 'application/json',
194 # type: (str, str, str, str, str, str, str) -> str
195 '''Updates `_data/latest.json` to match build result of the given commit.
197 with open('_data/latest.json', 'r+') as f:
198 latest = json.load(f, object_pairs_hook=collections.OrderedDict)
201 os: read_current_status(current_commit, 'history/' + os + '.tsv')
202 for os in ['windows', 'linux']
205 slug = 'rust-lang/rust'
206 message = textwrap.dedent('''\
207 📣 Toolstate changed by {}!
209 Tested on commit {}@{}.
210 Direct link to PR: <{}>
212 ''').format(relevant_pr_number, slug, current_commit, relevant_pr_url)
213 anything_changed = False
214 for status in latest:
215 tool = status['tool']
217 create_issue_for_status = None # set to the status that caused the issue
219 for os, s in current_status.items():
221 new = s.get(tool, old)
223 maintainers = ' '.join('@'+name for name in MAINTAINERS.get(tool, ()))
224 # comparing the strings, but they are ordered appropriately:
225 # "test-pass" > "test-fail" > "build-fail"
227 # things got fixed or at least the status quo improved
229 message += '🎉 {} on {}: {} → {} (cc {}).\n' \
230 .format(tool, os, old, new, maintainers)
232 # tests or builds are failing and were not failing before
234 title = '💔 {} on {}: {} → {}' \
235 .format(tool, os, old, new)
236 message += '{} (cc {}).\n' \
237 .format(title, maintainers)
238 # See if we need to create an issue.
239 # Create issue if things no longer build.
240 # (No issue for mere test failures to avoid spurious issues.)
241 if new == 'build-fail':
242 create_issue_for_status = new
244 if create_issue_for_status is not None:
247 tool, create_issue_for_status, MAINTAINERS.get(tool, ()),
248 relevant_pr_number, relevant_pr_user, LABELS.get(tool, []),
251 except HTTPError as e:
252 # network errors will simply end up not creating an issue, but that's better
253 # than failing the entire build job
254 print("HTTPError when creating issue for status regression: {0}\n{1!r}"
255 .format(e, e.read()))
257 print("I/O error when creating issue for status regression: {0}".format(e))
259 print("Unexpected error when creating issue for status regression: {0}"
260 .format(sys.exc_info()[0]))
264 status['commit'] = current_commit
265 status['datetime'] = current_datetime
266 anything_changed = True
268 if not anything_changed:
273 json.dump(latest, f, indent=4, separators=(',', ': '))
277 # Warning: Do not try to add a function containing the body of this try block.
278 # There are variables declared within that are implicitly global; it is unknown
279 # which ones precisely but at least this is true for `github_token`.
281 if __name__ != '__main__':
283 repo = os.environ.get('TOOLSTATE_VALIDATE_MAINTAINERS_REPO')
285 github_token = os.environ.get('TOOLSTATE_REPO_ACCESS_TOKEN')
287 # FIXME: This is currently broken. Starting on 2021-09-15, GitHub
288 # seems to have changed it so that to list the collaborators
289 # requires admin permissions. I think this will probably just need
290 # to be removed since we are probably not going to use an admin
291 # token, and I don't see another way to do this.
292 print('maintainer validation disabled')
293 # validate_maintainers(repo, github_token)
295 print('skipping toolstate maintainers validation since no GitHub token is present')
296 # When validating maintainers don't run the full script.
299 cur_commit = sys.argv[1]
300 cur_datetime = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
301 cur_commit_msg = sys.argv[2]
302 save_message_to_path = sys.argv[3]
303 github_token = sys.argv[4]
305 # assume that PR authors are also owners of the repo where the branch lives
306 relevant_pr_match = re.search(
307 r'Auto merge of #([0-9]+) - ([^:]+):[^,]+, r=(\S+)',
310 if relevant_pr_match:
311 number = relevant_pr_match.group(1)
312 relevant_pr_user = relevant_pr_match.group(2)
313 relevant_pr_number = 'rust-lang/rust#' + number
314 relevant_pr_url = 'https://github.com/rust-lang/rust/pull/' + number
315 pr_reviewer = relevant_pr_match.group(3)
318 relevant_pr_user = 'ghost'
319 relevant_pr_number = '<unknown PR>'
320 relevant_pr_url = '<unknown>'
321 pr_reviewer = 'ghost'
323 message = update_latest(
333 print('<Nothing changed>')
339 print('Dry run only, not committing anything')
342 with open(save_message_to_path, 'w') as f:
345 # Write the toolstate comment on the PR as well.
346 issue_url = gh_url() + '/{}/comments'.format(number)
347 response = urllib2.urlopen(urllib2.Request(
349 json.dumps({'body': maybe_delink(message)}).encode(),
351 'Authorization': 'token ' + github_token,
352 'Content-Type': 'application/json',
356 except HTTPError as e:
357 print("HTTPError: %s\n%r" % (e, e.read()))