2 # ignore-tidy-linelength
4 # This is a small script that we use on CI to collect CPU usage statistics of
5 # our builders. By seeing graphs of CPU usage over time we hope to correlate
6 # that with possible improvements to Rust's own build system, ideally diagnosing
7 # that either builders are always fully using their CPU resources or they're
8 # idle for long stretches of time.
10 # This script is relatively simple, but it's platform specific. Each platform
11 # (OSX/Windows/Linux) has a different way of calculating the current state of
12 # CPU at a point in time. We then compare two captured states to determine the
13 # percentage of time spent in one state versus another. The state capturing is
14 # all platform-specific but the loop at the bottom is the cross platform part
15 # that executes everywhere.
17 # # Viewing statistics
19 # All builders will upload their CPU statistics as CSV files to our S3 buckets.
20 # These URLS look like:
22 # https://$bucket.s3.amazonaws.com/rustc-builds/$commit/cpu-$builder.csv
26 # https://rust-lang-ci2.s3.amazonaws.com/rustc-builds/68baada19cd5340f05f0db15a3e16d6671609bcc/cpu-x86_64-apple.csv
28 # Each CSV file has two columns. The first is the timestamp of the measurement
29 # and the second column is the % of idle cpu time in that time slice. Ideally
30 # the second column is always zero.
32 # Once you've downloaded a file there's various ways to plot it and visualize
33 # it. For command line usage you use the `src/etc/cpu-usage-over-time-plot.sh`
34 # script in this repository.
40 if sys.platform == 'linux2':
43 with open('/proc/stat', 'r') as file:
44 data = file.readline().split()
46 raise Exception('did not start with "cpu"')
47 self.user = int(data[1])
48 self.nice = int(data[2])
49 self.system = int(data[3])
50 self.idle = int(data[4])
51 self.iowait = int(data[5])
52 self.irq = int(data[6])
53 self.softirq = int(data[7])
54 self.steal = int(data[8])
55 self.guest = int(data[9])
56 self.guest_nice = int(data[10])
58 def idle_since(self, prev):
59 user = self.user - prev.user
60 nice = self.nice - prev.nice
61 system = self.system - prev.system
62 idle = self.idle - prev.idle
63 iowait = self.iowait - prev.iowait
64 irq = self.irq - prev.irq
65 softirq = self.softirq - prev.softirq
66 steal = self.steal - prev.steal
67 guest = self.guest - prev.guest
68 guest_nice = self.guest_nice - prev.guest_nice
69 total = user + nice + system + idle + iowait + irq + softirq + steal + guest + guest_nice
70 return float(idle) / float(total) * 100
72 elif sys.platform == 'win32':
73 from ctypes.wintypes import DWORD
74 from ctypes import Structure, windll, WinError, GetLastError, byref
76 class FILETIME(Structure):
78 ("dwLowDateTime", DWORD),
79 ("dwHighDateTime", DWORD),
84 idle, kernel, user = FILETIME(), FILETIME(), FILETIME()
86 success = windll.kernel32.GetSystemTimes(
92 assert success, WinError(GetLastError())[1]
94 self.idle = (idle.dwHighDateTime << 32) | idle.dwLowDateTime
95 self.kernel = (kernel.dwHighDateTime << 32) | kernel.dwLowDateTime
96 self.user = (user.dwHighDateTime << 32) | user.dwLowDateTime
98 def idle_since(self, prev):
99 idle = self.idle - prev.idle
100 user = self.user - prev.user
101 kernel = self.kernel - prev.kernel
102 return float(idle) / float(user + kernel) * 100
104 elif sys.platform == 'darwin':
106 libc = cdll.LoadLibrary('/usr/lib/libc.dylib')
108 PROESSOR_CPU_LOAD_INFO = c_int(2)
113 c_int_p = POINTER(c_int)
117 num_cpus_u = c_uint(0)
119 cpu_info_cnt = c_int(0)
120 err = libc.host_processor_info(
121 libc.mach_host_self(),
122 PROESSOR_CPU_LOAD_INFO,
133 while cur < cpu_info_cnt.value:
134 self.user += cpu_info[cur + CPU_STATE_USER]
135 self.system += cpu_info[cur + CPU_STATE_SYSTEM]
136 self.idle += cpu_info[cur + CPU_STATE_IDLE]
137 self.nice += cpu_info[cur + CPU_STATE_NICE]
138 cur += num_cpus_u.value
140 def idle_since(self, prev):
141 user = self.user - prev.user
142 system = self.system - prev.system
143 idle = self.idle - prev.idle
144 nice = self.nice - prev.nice
145 return float(idle) / float(user + system + idle + nice) * 100.0
148 print('unknown platform', sys.platform)
155 next_state = State();
156 now = datetime.datetime.utcnow().isoformat()
157 idle = next_state.idle_since(cur_state)
158 print("%s,%s" % (now, idle))
160 cur_state = next_state