1 // FIXME: This is a complete copy of `cargo/src/cargo/util/read2.rs`
2 // Consider unify the read2() in libstd, cargo and this to prevent further code duplication.
7 pub use self::imp::read2;
8 use std::io::{self, Write};
10 use std::process::{Child, Output};
12 pub fn read2_abbreviated(mut child: Child, filter_paths_from_len: &[String]) -> io::Result<Output> {
13 let mut stdout = ProcOutput::new();
14 let mut stderr = ProcOutput::new();
16 drop(child.stdin.take());
18 child.stdout.take().unwrap(),
19 child.stderr.take().unwrap(),
20 &mut |is_stdout, data, _| {
21 if is_stdout { &mut stdout } else { &mut stderr }.extend(data, filter_paths_from_len);
25 let status = child.wait()?;
27 Ok(Output { status, stdout: stdout.into_bytes(), stderr: stderr.into_bytes() })
30 const HEAD_LEN: usize = 160 * 1024;
31 const TAIL_LEN: usize = 256 * 1024;
33 // Whenever a path is filtered when counting the length of the output, we need to add some
34 // placeholder length to ensure a compiler emitting only filtered paths doesn't cause a OOM.
36 // 32 was chosen semi-arbitrarily: it was the highest power of two that still allowed the test
37 // suite to pass at the moment of implementing path filtering.
38 const FILTERED_PATHS_PLACEHOLDER_LEN: usize = 32;
41 Full { bytes: Vec<u8>, filtered_len: usize },
42 Abbreviated { head: Vec<u8>, skipped: usize, tail: Box<[u8]> },
47 ProcOutput::Full { bytes: Vec::new(), filtered_len: 0 }
50 fn extend(&mut self, data: &[u8], filter_paths_from_len: &[String]) {
51 let new_self = match *self {
52 ProcOutput::Full { ref mut bytes, ref mut filtered_len } => {
53 let old_len = bytes.len();
54 bytes.extend_from_slice(data);
55 *filtered_len += data.len();
57 // We had problems in the past with tests failing only in some environments,
58 // due to the length of the base path pushing the output size over the limit.
60 // To make those failures deterministic across all environments we ignore known
61 // paths when calculating the string length, while still including the full
62 // path in the output. This could result in some output being larger than the
63 // threshold, but it's better than having nondeterministic failures.
65 // The compiler emitting only excluded strings is addressed by adding a
66 // placeholder size for each excluded segment, which will eventually reach
67 // the configured threshold.
68 for path in filter_paths_from_len {
69 let path_bytes = path.as_bytes();
70 // We start matching `path_bytes - 1` into the previously loaded data,
71 // to account for the fact a path_bytes might be included across multiple
72 // `extend` calls. Starting from `- 1` avoids double-counting paths.
73 let matches = (&bytes[(old_len.saturating_sub(path_bytes.len() - 1))..])
74 .windows(path_bytes.len())
75 .filter(|window| window == &path_bytes)
77 *filtered_len -= matches * path_bytes.len();
79 // We can't just remove the length of the filtered path from the output lenght,
80 // otherwise a compiler emitting only filtered paths would OOM compiletest. Add
81 // a fixed placeholder length for each path to prevent that.
82 *filtered_len += matches * FILTERED_PATHS_PLACEHOLDER_LEN;
85 let new_len = bytes.len();
86 if *filtered_len <= HEAD_LEN + TAIL_LEN {
90 let mut head = replace(bytes, Vec::new());
91 let mut middle = head.split_off(HEAD_LEN);
92 let tail = middle.split_off(middle.len() - TAIL_LEN).into_boxed_slice();
93 let skipped = new_len - HEAD_LEN - TAIL_LEN;
94 ProcOutput::Abbreviated { head, skipped, tail }
96 ProcOutput::Abbreviated { ref mut skipped, ref mut tail, .. } => {
97 *skipped += data.len();
98 if data.len() <= TAIL_LEN {
99 tail[..data.len()].copy_from_slice(data);
100 tail.rotate_left(data.len());
102 tail.copy_from_slice(&data[(data.len() - TAIL_LEN)..]);
110 fn into_bytes(self) -> Vec<u8> {
112 ProcOutput::Full { bytes, .. } => bytes,
113 ProcOutput::Abbreviated { mut head, mut skipped, tail } => {
114 let mut tail = &*tail;
116 // Skip over '{' at the start of the tail, so we don't later wrongfully consider this as json.
117 // See <https://rust-lang.zulipchat.com/#narrow/stream/182449-t-compiler.2Fhelp/topic/Weird.20CI.20failure/near/321797811>
118 while tail.get(0) == Some(&b'{') {
123 write!(&mut head, "\n\n<<<<<< SKIPPED {} BYTES >>>>>>\n\n", skipped).unwrap();
124 head.extend_from_slice(tail);
131 #[cfg(not(any(unix, windows)))]
133 use std::io::{self, Read};
134 use std::process::{ChildStderr, ChildStdout};
137 out_pipe: ChildStdout,
138 err_pipe: ChildStderr,
139 data: &mut dyn FnMut(bool, &mut Vec<u8>, bool),
140 ) -> io::Result<()> {
141 let mut buffer = Vec::new();
142 out_pipe.read_to_end(&mut buffer)?;
143 data(true, &mut buffer, true);
145 err_pipe.read_to_end(&mut buffer)?;
146 data(false, &mut buffer, true);
154 use std::io::prelude::*;
156 use std::os::unix::prelude::*;
157 use std::process::{ChildStderr, ChildStdout};
160 mut out_pipe: ChildStdout,
161 mut err_pipe: ChildStderr,
162 data: &mut dyn FnMut(bool, &mut Vec<u8>, bool),
163 ) -> io::Result<()> {
165 libc::fcntl(out_pipe.as_raw_fd(), libc::F_SETFL, libc::O_NONBLOCK);
166 libc::fcntl(err_pipe.as_raw_fd(), libc::F_SETFL, libc::O_NONBLOCK);
169 let mut out_done = false;
170 let mut err_done = false;
171 let mut out = Vec::new();
172 let mut err = Vec::new();
174 let mut fds: [libc::pollfd; 2] = unsafe { mem::zeroed() };
175 fds[0].fd = out_pipe.as_raw_fd();
176 fds[0].events = libc::POLLIN;
177 fds[1].fd = err_pipe.as_raw_fd();
178 fds[1].events = libc::POLLIN;
183 // wait for either pipe to become readable using `select`
184 let r = unsafe { libc::poll(fds.as_mut_ptr(), nfds, -1) };
186 let err = io::Error::last_os_error();
187 if err.kind() == io::ErrorKind::Interrupted {
193 // Read as much as we can from each pipe, ignoring EWOULDBLOCK or
194 // EAGAIN. If we hit EOF, then this will happen because the underlying
195 // reader will return Ok(0), in which case we'll see `Ok` ourselves. In
196 // this case we flip the other fd back into blocking mode and read
197 // whatever's leftover on that file descriptor.
198 let handle = |res: io::Result<_>| match res {
201 if e.kind() == io::ErrorKind::WouldBlock {
208 if !err_done && fds[errfd].revents != 0 && handle(err_pipe.read_to_end(&mut err))? {
212 data(false, &mut err, err_done);
213 if !out_done && fds[0].revents != 0 && handle(out_pipe.read_to_end(&mut out))? {
215 fds[0].fd = err_pipe.as_raw_fd();
219 data(true, &mut out, out_done);
228 use std::os::windows::prelude::*;
229 use std::process::{ChildStderr, ChildStdout};
232 use miow::iocp::{CompletionPort, CompletionStatus};
233 use miow::pipe::NamedPipe;
234 use miow::Overlapped;
235 use winapi::shared::winerror::ERROR_BROKEN_PIPE;
238 dst: &'a mut Vec<u8>,
239 overlapped: Overlapped,
245 out_pipe: ChildStdout,
246 err_pipe: ChildStderr,
247 data: &mut dyn FnMut(bool, &mut Vec<u8>, bool),
248 ) -> io::Result<()> {
249 let mut out = Vec::new();
250 let mut err = Vec::new();
252 let port = CompletionPort::new(1)?;
253 port.add_handle(0, &out_pipe)?;
254 port.add_handle(1, &err_pipe)?;
257 let mut out_pipe = Pipe::new(out_pipe, &mut out);
258 let mut err_pipe = Pipe::new(err_pipe, &mut err);
263 let mut status = [CompletionStatus::zero(), CompletionStatus::zero()];
265 while !out_pipe.done || !err_pipe.done {
266 for status in port.get_many(&mut status, None)? {
267 if status.token() == 0 {
268 out_pipe.complete(status);
269 data(true, out_pipe.dst, out_pipe.done);
272 err_pipe.complete(status);
273 data(false, err_pipe.dst, err_pipe.done);
284 unsafe fn new<P: IntoRawHandle>(p: P, dst: &'a mut Vec<u8>) -> Pipe<'a> {
287 pipe: NamedPipe::from_raw_handle(p.into_raw_handle()),
288 overlapped: Overlapped::zero(),
293 unsafe fn read(&mut self) -> io::Result<()> {
294 let dst = slice_to_end(self.dst);
295 match self.pipe.read_overlapped(dst, self.overlapped.raw()) {
298 if e.raw_os_error() == Some(ERROR_BROKEN_PIPE as i32) {
308 unsafe fn complete(&mut self, status: &CompletionStatus) {
309 let prev = self.dst.len();
310 self.dst.set_len(prev + status.bytes_transferred() as usize);
311 if status.bytes_transferred() == 0 {
317 unsafe fn slice_to_end(v: &mut Vec<u8>) -> &mut [u8] {
318 if v.capacity() == 0 {
321 if v.capacity() == v.len() {
324 slice::from_raw_parts_mut(v.as_mut_ptr().offset(v.len() as isize), v.capacity() - v.len())