1 //! This module is responsible for collecting metrics profiling information for the current build
2 //! and dumping it to disk as JSON, to aid investigations on build and CI performance.
4 //! As this module requires additional dependencies not present during local builds, it's cfg'd
5 //! away whenever the `build.metrics` config option is not set to `true`.
7 use crate::builder::Step;
10 use serde::{Deserialize, Serialize};
11 use std::cell::RefCell;
13 use std::io::BufWriter;
14 use std::time::{Duration, Instant};
15 use sysinfo::{CpuExt, System, SystemExt};
17 pub(crate) struct BuildMetrics {
18 state: RefCell<MetricsState>,
22 pub(crate) fn init() -> Self {
23 let state = RefCell::new(MetricsState {
24 finished_steps: Vec::new(),
25 running_steps: Vec::new(),
27 system_info: System::new(),
29 invocation_timer_start: Instant::now(),
32 BuildMetrics { state }
35 pub(crate) fn enter_step<S: Step>(&self, step: &S) {
36 let mut state = self.state.borrow_mut();
38 // Consider all the stats gathered so far as the parent's.
39 if !state.running_steps.is_empty() {
40 self.collect_stats(&mut *state);
43 state.system_info.refresh_cpu();
44 state.timer_start = Some(Instant::now());
46 state.running_steps.push(StepMetrics {
47 type_: std::any::type_name::<S>().into(),
48 debug_repr: format!("{step:?}"),
50 cpu_usage_time_sec: 0.0,
51 duration_excluding_children_sec: Duration::ZERO,
57 pub(crate) fn exit_step(&self) {
58 let mut state = self.state.borrow_mut();
60 self.collect_stats(&mut *state);
62 let step = state.running_steps.pop().unwrap();
63 if state.running_steps.is_empty() {
64 state.finished_steps.push(step);
65 state.timer_start = None;
67 state.running_steps.last_mut().unwrap().children.push(step);
69 // Start collecting again for the parent step.
70 state.system_info.refresh_cpu();
71 state.timer_start = Some(Instant::now());
75 fn collect_stats(&self, state: &mut MetricsState) {
76 let step = state.running_steps.last_mut().unwrap();
78 let elapsed = state.timer_start.unwrap().elapsed();
79 step.duration_excluding_children_sec += elapsed;
81 state.system_info.refresh_cpu();
82 let cpu = state.system_info.cpus().iter().map(|p| p.cpu_usage()).sum::<f32>();
83 step.cpu_usage_time_sec += cpu as f64 / 100.0 * elapsed.as_secs_f64();
86 pub(crate) fn persist(&self, build: &Build) {
87 let mut state = self.state.borrow_mut();
88 assert!(state.running_steps.is_empty(), "steps are still executing");
90 let dest = build.out.join("metrics.json");
92 let mut system = System::new();
94 system.refresh_memory();
96 let system_stats = JsonInvocationSystemStats {
97 cpu_threads_count: system.cpus().len(),
98 cpu_model: system.cpus()[0].brand().into(),
100 memory_total_bytes: system.total_memory() * 1024,
102 let steps = std::mem::take(&mut state.finished_steps);
104 // Some of our CI builds consist of multiple independent CI invocations. Ensure all the
105 // previous invocations are still present in the resulting file.
106 let mut invocations = match std::fs::read(&dest) {
107 Ok(contents) => t!(serde_json::from_slice::<JsonRoot>(&contents)).invocations,
109 if err.kind() != std::io::ErrorKind::NotFound {
110 panic!("failed to open existing metrics file at {}: {err}", dest.display());
115 invocations.push(JsonInvocation {
116 duration_including_children_sec: state.invocation_timer_start.elapsed().as_secs_f64(),
117 children: steps.into_iter().map(|step| self.prepare_json_step(step)).collect(),
120 let json = JsonRoot { system_stats, invocations };
122 t!(std::fs::create_dir_all(dest.parent().unwrap()));
123 let mut file = BufWriter::new(t!(File::create(&dest)));
124 t!(serde_json::to_writer(&mut file, &json));
127 fn prepare_json_step(&self, step: StepMetrics) -> JsonNode {
128 JsonNode::RustbuildStep {
130 debug_repr: step.debug_repr,
132 duration_excluding_children_sec: step.duration_excluding_children_sec.as_secs_f64(),
133 system_stats: JsonStepSystemStats {
134 cpu_utilization_percent: step.cpu_usage_time_sec * 100.0
135 / step.duration_excluding_children_sec.as_secs_f64(),
141 .map(|child| self.prepare_json_step(child))
147 struct MetricsState {
148 finished_steps: Vec<StepMetrics>,
149 running_steps: Vec<StepMetrics>,
152 timer_start: Option<Instant>,
153 invocation_timer_start: Instant,
160 cpu_usage_time_sec: f64,
161 duration_excluding_children_sec: Duration,
163 children: Vec<StepMetrics>,
166 #[derive(Serialize, Deserialize)]
167 #[serde(rename_all = "snake_case")]
169 system_stats: JsonInvocationSystemStats,
170 invocations: Vec<JsonInvocation>,
173 #[derive(Serialize, Deserialize)]
174 #[serde(rename_all = "snake_case")]
175 struct JsonInvocation {
176 duration_including_children_sec: f64,
177 children: Vec<JsonNode>,
180 #[derive(Serialize, Deserialize)]
181 #[serde(tag = "kind", rename_all = "snake_case")]
184 #[serde(rename = "type")]
188 duration_excluding_children_sec: f64,
189 system_stats: JsonStepSystemStats,
191 children: Vec<JsonNode>,
195 #[derive(Serialize, Deserialize)]
196 #[serde(rename_all = "snake_case")]
197 struct JsonInvocationSystemStats {
198 cpu_threads_count: usize,
201 memory_total_bytes: u64,
204 #[derive(Serialize, Deserialize)]
205 #[serde(rename_all = "snake_case")]
206 struct JsonStepSystemStats {
207 cpu_utilization_percent: f64,