src/ci/cpu-usage-over-time.py

   1 #!/usr/bin/env python
   2 # ignore-tidy-linelength
   3
   4 # This is a small script that we use on CI to collect CPU usage statistics of
   5 # our builders. By seeing graphs of CPU usage over time we hope to correlate
   6 # that with possible improvements to Rust's own build system, ideally diagnosing
   7 # that either builders are always fully using their CPU resources or they're
   8 # idle for long stretches of time.
   9 #
  10 # This script is relatively simple, but it's platform specific. Each platform
  11 # (OSX/Windows/Linux) has a different way of calculating the current state of
  12 # CPU at a point in time. We then compare two captured states to determine the
  13 # percentage of time spent in one state versus another. The state capturing is
  14 # all platform-specific but the loop at the bottom is the cross platform part
  15 # that executes everywhere.
  16 #
  17 # # Viewing statistics
  18 #
  19 # All builders will upload their CPU statistics as CSV files to our S3 buckets.
  20 # These URLS look like:
  21 #
  22 #   https://$bucket.s3.amazonaws.com/rustc-builds/$commit/cpu-$builder.csv
  23 #
  24 # for example
  25 #
  26 #   https://rust-lang-ci2.s3.amazonaws.com/rustc-builds/68baada19cd5340f05f0db15a3e16d6671609bcc/cpu-x86_64-apple.csv
  27 #
  28 # Each CSV file has two columns. The first is the timestamp of the measurement
  29 # and the second column is the % of idle cpu time in that time slice. Ideally
  30 # the second column is always zero.
  31 #
  32 # Once you've downloaded a file there's various ways to plot it and visualize
  33 # it. For command line usage you use the `src/etc/cpu-usage-over-time-plot.sh`
  34 # script in this repository.
  35
  36 import datetime
  37 import sys
  38 import time
  39
  40 if sys.platform == 'linux2':
  41     class State:
  42         def __init__(self):
  43             with open('/proc/stat', 'r') as file:
  44                 data = file.readline().split()
  45             if data[0] != 'cpu':
  46                 raise Exception('did not start with "cpu"')
  47             self.user = int(data[1])
  48             self.nice = int(data[2])
  49             self.system = int(data[3])
  50             self.idle = int(data[4])
  51             self.iowait = int(data[5])
  52             self.irq = int(data[6])
  53             self.softirq = int(data[7])
  54             self.steal = int(data[8])
  55             self.guest = int(data[9])
  56             self.guest_nice = int(data[10])
  57
  58         def idle_since(self, prev):
  59             user = self.user - prev.user
  60             nice = self.nice - prev.nice
  61             system = self.system - prev.system
  62             idle = self.idle - prev.idle
  63             iowait = self.iowait - prev.iowait
  64             irq = self.irq - prev.irq
  65             softirq = self.softirq - prev.softirq
  66             steal = self.steal - prev.steal
  67             guest = self.guest - prev.guest
  68             guest_nice = self.guest_nice - prev.guest_nice
  69             total = user + nice + system + idle + iowait + irq + softirq + steal + guest + guest_nice
  70             return float(idle) / float(total) * 100
  71
  72 elif sys.platform == 'win32':
  73     from ctypes.wintypes import DWORD
  74     from ctypes import Structure, windll, WinError, GetLastError, byref
  75
  76     class FILETIME(Structure):
  77         _fields_ = [
  78             ("dwLowDateTime", DWORD),
  79             ("dwHighDateTime", DWORD),
  80         ]
  81
  82     class State:
  83         def __init__(self):
  84             idle, kernel, user = FILETIME(), FILETIME(), FILETIME()
  85
  86             success = windll.kernel32.GetSystemTimes(
  87                 byref(idle),
  88                 byref(kernel),
  89                 byref(user),
  90             )
  91
  92             assert success, WinError(GetLastError())[1]
  93
  94             self.idle = (idle.dwHighDateTime << 32) | idle.dwLowDateTime
  95             self.kernel = (kernel.dwHighDateTime << 32) | kernel.dwLowDateTime
  96             self.user = (user.dwHighDateTime << 32) | user.dwLowDateTime
  97
  98         def idle_since(self, prev):
  99             idle = self.idle - prev.idle
 100             user = self.user - prev.user
 101             kernel = self.kernel - prev.kernel
 102             return float(idle) / float(user + kernel) * 100
 103
 104 elif sys.platform == 'darwin':
 105     from ctypes import *
 106     libc = cdll.LoadLibrary('/usr/lib/libc.dylib')
 107
 108     PROESSOR_CPU_LOAD_INFO = c_int(2)
 109     CPU_STATE_USER = 0
 110     CPU_STATE_SYSTEM = 1
 111     CPU_STATE_IDLE = 2
 112     CPU_STATE_NICE = 3
 113     c_int_p = POINTER(c_int)
 114
 115     class State:
 116         def __init__(self):
 117             num_cpus_u = c_uint(0)
 118             cpu_info = c_int_p()
 119             cpu_info_cnt = c_int(0)
 120             err = libc.host_processor_info(
 121                 libc.mach_host_self(),
 122                 PROESSOR_CPU_LOAD_INFO,
 123                 byref(num_cpus_u),
 124                 byref(cpu_info),
 125                 byref(cpu_info_cnt),
 126             )
 127             assert err == 0
 128             self.user = 0
 129             self.system = 0
 130             self.idle = 0
 131             self.nice = 0
 132             cur = 0
 133             while cur < cpu_info_cnt.value:
 134                 self.user += cpu_info[cur + CPU_STATE_USER]
 135                 self.system += cpu_info[cur + CPU_STATE_SYSTEM]
 136                 self.idle += cpu_info[cur + CPU_STATE_IDLE]
 137                 self.nice += cpu_info[cur + CPU_STATE_NICE]
 138                 cur += num_cpus_u.value
 139
 140         def idle_since(self, prev):
 141             user = self.user - prev.user
 142             system = self.system - prev.system
 143             idle = self.idle - prev.idle
 144             nice = self.nice - prev.nice
 145             return float(idle) / float(user + system + idle + nice) * 100.0
 146
 147 else:
 148     print('unknown platform', sys.platform)
 149     sys.exit(1)
 150
 151 cur_state = State()
 152 print("Time,Idle")
 153 while True:
 154     time.sleep(1)
 155     next_state = State()
 156     now = datetime.datetime.utcnow().isoformat()
 157     idle = next_state.idle_since(cur_state)
 158     print("%s,%s" % (now, idle))
 159     sys.stdout.flush()
 160     cur_state = next_state