1 // Copyright 2013 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 // ignore-lexer-test FIXME #15883
13 //! Buffering wrappers for I/O traits
17 use io::{Reader, Writer, Stream, Buffer, DEFAULT_BUF_SIZE, IoResult};
18 use iter::{IteratorExt, ExactSizeIterator, repeat};
21 use option::Option::{Some, None};
22 use result::Result::Ok;
23 use slice::{SliceExt};
27 /// Wraps a Reader and buffers input from it
29 /// It can be excessively inefficient to work directly with a `Reader`. For
30 /// example, every call to `read` on `TcpStream` results in a system call. A
31 /// `BufferedReader` performs large, infrequent reads on the underlying
32 /// `Reader` and maintains an in-memory buffer of the results.
37 /// use std::io::{BufferedReader, File};
39 /// let file = File::open(&Path::new("message.txt"));
40 /// let mut reader = BufferedReader::new(file);
42 /// let mut buf = [0; 100];
43 /// match reader.read(&mut buf) {
44 /// Ok(nread) => println!("Read {} bytes", nread),
45 /// Err(e) => println!("error reading: {}", e)
48 pub struct BufferedReader<R> {
55 impl<R> fmt::Show for BufferedReader<R> where R: fmt::Show {
56 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
57 write!(fmt, "BufferedReader {{ reader: {:?}, buffer: {}/{} }}",
58 self.inner, self.cap - self.pos, self.buf.len())
62 impl<R: Reader> BufferedReader<R> {
63 /// Creates a new `BufferedReader` with the specified buffer capacity
64 pub fn with_capacity(cap: uint, inner: R) -> BufferedReader<R> {
67 // We can't use the same trick here as we do for BufferedWriter,
68 // since this memory is visible to the inner Reader.
69 buf: repeat(0).take(cap).collect(),
75 /// Creates a new `BufferedReader` with a default buffer capacity
76 pub fn new(inner: R) -> BufferedReader<R> {
77 BufferedReader::with_capacity(DEFAULT_BUF_SIZE, inner)
80 /// Gets a reference to the underlying reader.
81 pub fn get_ref<'a>(&self) -> &R { &self.inner }
83 /// Gets a mutable reference to the underlying reader.
87 /// It is inadvisable to directly read from the underlying reader.
88 pub fn get_mut(&mut self) -> &mut R { &mut self.inner }
90 /// Unwraps this `BufferedReader`, returning the underlying reader.
92 /// Note that any leftover data in the internal buffer is lost.
93 pub fn into_inner(self) -> R { self.inner }
96 impl<R: Reader> Buffer for BufferedReader<R> {
97 fn fill_buf<'a>(&'a mut self) -> IoResult<&'a [u8]> {
98 if self.pos == self.cap {
99 self.cap = try!(self.inner.read(self.buf.as_mut_slice()));
102 Ok(&self.buf[self.pos..self.cap])
105 fn consume(&mut self, amt: uint) {
107 assert!(self.pos <= self.cap);
111 impl<R: Reader> Reader for BufferedReader<R> {
112 fn read(&mut self, buf: &mut [u8]) -> IoResult<uint> {
113 if self.pos == self.cap && buf.len() >= self.buf.capacity() {
114 return self.inner.read(buf);
117 let available = try!(self.fill_buf());
118 let nread = cmp::min(available.len(), buf.len());
119 slice::bytes::copy_memory(buf, &available[0..nread]);
127 /// Wraps a Writer and buffers output to it
129 /// It can be excessively inefficient to work directly with a `Writer`. For
130 /// example, every call to `write` on `TcpStream` results in a system call. A
131 /// `BufferedWriter` keeps an in memory buffer of data and writes it to the
132 /// underlying `Writer` in large, infrequent batches.
134 /// This writer will be flushed when it is dropped.
139 /// use std::io::{BufferedWriter, File};
141 /// let file = File::create(&Path::new("message.txt")).unwrap();
142 /// let mut writer = BufferedWriter::new(file);
144 /// writer.write_str("hello, world").unwrap();
145 /// writer.flush().unwrap();
147 pub struct BufferedWriter<W> {
153 impl<W> fmt::Show for BufferedWriter<W> where W: fmt::Show {
154 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
155 write!(fmt, "BufferedWriter {{ writer: {:?}, buffer: {}/{} }}",
156 self.inner.as_ref().unwrap(), self.pos, self.buf.len())
160 impl<W: Writer> BufferedWriter<W> {
161 /// Creates a new `BufferedWriter` with the specified buffer capacity
162 pub fn with_capacity(cap: uint, inner: W) -> BufferedWriter<W> {
163 // It's *much* faster to create an uninitialized buffer than it is to
164 // fill everything in with 0. This buffer is entirely an implementation
165 // detail and is never exposed, so we're safe to not initialize
166 // everything up-front. This allows creation of BufferedWriter instances
167 // to be very cheap (large mallocs are not nearly as expensive as large
169 let mut buf = Vec::with_capacity(cap);
170 unsafe { buf.set_len(cap); }
178 /// Creates a new `BufferedWriter` with a default buffer capacity
179 pub fn new(inner: W) -> BufferedWriter<W> {
180 BufferedWriter::with_capacity(DEFAULT_BUF_SIZE, inner)
183 fn flush_buf(&mut self) -> IoResult<()> {
185 let ret = self.inner.as_mut().unwrap().write(&self.buf[0..self.pos]);
193 /// Gets a reference to the underlying writer.
194 pub fn get_ref(&self) -> &W { self.inner.as_ref().unwrap() }
196 /// Gets a mutable reference to the underlying write.
200 /// It is inadvisable to directly read from the underlying writer.
201 pub fn get_mut(&mut self) -> &mut W { self.inner.as_mut().unwrap() }
203 /// Unwraps this `BufferedWriter`, returning the underlying writer.
205 /// The buffer is flushed before returning the writer.
206 pub fn into_inner(mut self) -> W {
207 // FIXME(#12628): is panicking the right thing to do if flushing panicks?
208 self.flush_buf().unwrap();
209 self.inner.take().unwrap()
213 impl<W: Writer> Writer for BufferedWriter<W> {
214 fn write(&mut self, buf: &[u8]) -> IoResult<()> {
215 if self.pos + buf.len() > self.buf.len() {
216 try!(self.flush_buf());
219 if buf.len() > self.buf.len() {
220 self.inner.as_mut().unwrap().write(buf)
222 let dst = self.buf.slice_from_mut(self.pos);
223 slice::bytes::copy_memory(dst, buf);
224 self.pos += buf.len();
229 fn flush(&mut self) -> IoResult<()> {
230 self.flush_buf().and_then(|()| self.inner.as_mut().unwrap().flush())
235 impl<W: Writer> Drop for BufferedWriter<W> {
237 if self.inner.is_some() {
238 // dtors should not panic, so we ignore a panicked flush
239 let _ = self.flush_buf();
244 /// Wraps a Writer and buffers output to it, flushing whenever a newline (`0x0a`,
245 /// `'\n'`) is detected.
247 /// This writer will be flushed when it is dropped.
248 pub struct LineBufferedWriter<W> {
249 inner: BufferedWriter<W>,
252 impl<W> fmt::Show for LineBufferedWriter<W> where W: fmt::Show {
253 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
254 write!(fmt, "LineBufferedWriter {{ writer: {:?}, buffer: {}/{} }}",
255 self.inner.inner, self.inner.pos, self.inner.buf.len())
259 impl<W: Writer> LineBufferedWriter<W> {
260 /// Creates a new `LineBufferedWriter`
261 pub fn new(inner: W) -> LineBufferedWriter<W> {
262 // Lines typically aren't that long, don't use a giant buffer
264 inner: BufferedWriter::with_capacity(1024, inner)
268 /// Gets a reference to the underlying writer.
270 /// This type does not expose the ability to get a mutable reference to the
271 /// underlying reader because that could possibly corrupt the buffer.
272 pub fn get_ref<'a>(&'a self) -> &'a W { self.inner.get_ref() }
274 /// Unwraps this `LineBufferedWriter`, returning the underlying writer.
276 /// The internal buffer is flushed before returning the writer.
277 pub fn into_inner(self) -> W { self.inner.into_inner() }
280 impl<W: Writer> Writer for LineBufferedWriter<W> {
281 fn write(&mut self, buf: &[u8]) -> IoResult<()> {
282 match buf.iter().rposition(|&b| b == b'\n') {
284 try!(self.inner.write(&buf[0..(i + 1)]));
285 try!(self.inner.flush());
286 try!(self.inner.write(&buf[(i + 1)..]));
289 None => self.inner.write(buf),
293 fn flush(&mut self) -> IoResult<()> { self.inner.flush() }
296 struct InternalBufferedWriter<W>(BufferedWriter<W>);
298 impl<W> InternalBufferedWriter<W> {
299 fn get_mut<'a>(&'a mut self) -> &'a mut BufferedWriter<W> {
300 let InternalBufferedWriter(ref mut w) = *self;
305 impl<W: Reader> Reader for InternalBufferedWriter<W> {
306 fn read(&mut self, buf: &mut [u8]) -> IoResult<uint> {
307 self.get_mut().inner.as_mut().unwrap().read(buf)
311 /// Wraps a Stream and buffers input and output to and from it.
313 /// It can be excessively inefficient to work directly with a `Stream`. For
314 /// example, every call to `read` or `write` on `TcpStream` results in a system
315 /// call. A `BufferedStream` keeps in memory buffers of data, making large,
316 /// infrequent calls to `read` and `write` on the underlying `Stream`.
318 /// The output half will be flushed when this stream is dropped.
323 /// # #![allow(unused_must_use)]
324 /// use std::io::{BufferedStream, File};
326 /// let file = File::open(&Path::new("message.txt"));
327 /// let mut stream = BufferedStream::new(file);
329 /// stream.write("hello, world".as_bytes());
332 /// let mut buf = [0; 100];
333 /// match stream.read(&mut buf) {
334 /// Ok(nread) => println!("Read {} bytes", nread),
335 /// Err(e) => println!("error reading: {}", e)
338 pub struct BufferedStream<S> {
339 inner: BufferedReader<InternalBufferedWriter<S>>
342 impl<S> fmt::Show for BufferedStream<S> where S: fmt::Show {
343 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
344 let reader = &self.inner;
345 let writer = &self.inner.inner.0;
346 write!(fmt, "BufferedStream {{ stream: {:?}, write_buffer: {}/{}, read_buffer: {}/{} }}",
348 writer.pos, writer.buf.len(),
349 reader.cap - reader.pos, reader.buf.len())
353 impl<S: Stream> BufferedStream<S> {
354 /// Creates a new buffered stream with explicitly listed capacities for the
355 /// reader/writer buffer.
356 pub fn with_capacities(reader_cap: uint, writer_cap: uint, inner: S)
357 -> BufferedStream<S> {
358 let writer = BufferedWriter::with_capacity(writer_cap, inner);
359 let internal_writer = InternalBufferedWriter(writer);
360 let reader = BufferedReader::with_capacity(reader_cap,
362 BufferedStream { inner: reader }
365 /// Creates a new buffered stream with the default reader/writer buffer
367 pub fn new(inner: S) -> BufferedStream<S> {
368 BufferedStream::with_capacities(DEFAULT_BUF_SIZE, DEFAULT_BUF_SIZE,
372 /// Gets a reference to the underlying stream.
373 pub fn get_ref(&self) -> &S {
374 let InternalBufferedWriter(ref w) = self.inner.inner;
378 /// Gets a mutable reference to the underlying stream.
382 /// It is inadvisable to read directly from or write directly to the
383 /// underlying stream.
384 pub fn get_mut(&mut self) -> &mut S {
385 let InternalBufferedWriter(ref mut w) = self.inner.inner;
389 /// Unwraps this `BufferedStream`, returning the underlying stream.
391 /// The internal buffer is flushed before returning the stream. Any leftover
392 /// data in the read buffer is lost.
393 pub fn into_inner(self) -> S {
394 let InternalBufferedWriter(w) = self.inner.inner;
399 impl<S: Stream> Buffer for BufferedStream<S> {
400 fn fill_buf<'a>(&'a mut self) -> IoResult<&'a [u8]> { self.inner.fill_buf() }
401 fn consume(&mut self, amt: uint) { self.inner.consume(amt) }
404 impl<S: Stream> Reader for BufferedStream<S> {
405 fn read(&mut self, buf: &mut [u8]) -> IoResult<uint> {
410 impl<S: Stream> Writer for BufferedStream<S> {
411 fn write(&mut self, buf: &[u8]) -> IoResult<()> {
412 self.inner.inner.get_mut().write(buf)
414 fn flush(&mut self) -> IoResult<()> {
415 self.inner.inner.get_mut().flush()
425 use super::super::{IoResult, EndOfFile};
426 use super::super::mem::MemReader;
427 use self::test::Bencher;
429 /// A type, free to create, primarily intended for benchmarking creation of
430 /// wrappers that, just for construction, don't need a Reader/Writer that
431 /// does anything useful. Is equivalent to `/dev/null` in semantics.
432 #[derive(Clone,PartialEq,PartialOrd)]
433 pub struct NullStream;
435 impl Reader for NullStream {
436 fn read(&mut self, _: &mut [u8]) -> io::IoResult<uint> {
437 Err(io::standard_error(io::EndOfFile))
441 impl Writer for NullStream {
442 fn write(&mut self, _: &[u8]) -> io::IoResult<()> { Ok(()) }
445 /// A dummy reader intended at testing short-reads propagation.
446 pub struct ShortReader {
450 impl Reader for ShortReader {
451 fn read(&mut self, _: &mut [u8]) -> io::IoResult<uint> {
452 if self.lengths.is_empty() {
453 Err(io::standard_error(io::EndOfFile))
455 Ok(self.lengths.remove(0))
461 fn test_buffered_reader() {
462 let inner = MemReader::new(vec!(5, 6, 7, 0, 1, 2, 3, 4));
463 let mut reader = BufferedReader::with_capacity(2, inner);
465 let mut buf = [0, 0, 0];
466 let nread = reader.read(&mut buf);
467 assert_eq!(Ok(3), nread);
468 let b: &[_] = &[5, 6, 7];
471 let mut buf = [0, 0];
472 let nread = reader.read(&mut buf);
473 assert_eq!(Ok(2), nread);
474 let b: &[_] = &[0, 1];
478 let nread = reader.read(&mut buf);
479 assert_eq!(Ok(1), nread);
483 let mut buf = [0, 0, 0];
484 let nread = reader.read(&mut buf);
485 assert_eq!(Ok(1), nread);
486 let b: &[_] = &[3, 0, 0];
489 let nread = reader.read(&mut buf);
490 assert_eq!(Ok(1), nread);
491 let b: &[_] = &[4, 0, 0];
494 assert!(reader.read(&mut buf).is_err());
498 fn test_buffered_writer() {
499 let inner = Vec::new();
500 let mut writer = BufferedWriter::with_capacity(2, inner);
502 writer.write(&[0, 1]).unwrap();
504 assert_eq!(&writer.get_ref()[], b);
506 writer.write(&[2]).unwrap();
507 let b: &[_] = &[0, 1];
508 assert_eq!(&writer.get_ref()[], b);
510 writer.write(&[3]).unwrap();
511 assert_eq!(&writer.get_ref()[], b);
513 writer.flush().unwrap();
514 let a: &[_] = &[0, 1, 2, 3];
515 assert_eq!(a, &writer.get_ref()[]);
517 writer.write(&[4]).unwrap();
518 writer.write(&[5]).unwrap();
519 assert_eq!(a, &writer.get_ref()[]);
521 writer.write(&[6]).unwrap();
522 let a: &[_] = &[0, 1, 2, 3, 4, 5];
523 assert_eq!(a, &writer.get_ref()[]);
525 writer.write(&[7, 8]).unwrap();
526 let a: &[_] = &[0, 1, 2, 3, 4, 5, 6];
527 assert_eq!(a, &writer.get_ref()[]);
529 writer.write(&[9, 10, 11]).unwrap();
530 let a: &[_] = &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11];
531 assert_eq!(a, &writer.get_ref()[]);
533 writer.flush().unwrap();
534 assert_eq!(a, &writer.get_ref()[]);
538 fn test_buffered_writer_inner_flushes() {
539 let mut w = BufferedWriter::with_capacity(3, Vec::new());
540 w.write(&[0, 1]).unwrap();
542 assert_eq!(a, &w.get_ref()[]);
543 let w = w.into_inner();
544 let a: &[_] = &[0, 1];
548 // This is just here to make sure that we don't infinite loop in the
549 // newtype struct autoderef weirdness
551 fn test_buffered_stream() {
554 impl io::Writer for S {
555 fn write(&mut self, _: &[u8]) -> io::IoResult<()> { Ok(()) }
558 impl io::Reader for S {
559 fn read(&mut self, _: &mut [u8]) -> io::IoResult<uint> {
560 Err(io::standard_error(io::EndOfFile))
564 let mut stream = BufferedStream::new(S);
566 assert!(stream.read(&mut buf).is_err());
567 stream.write(&buf).unwrap();
568 stream.flush().unwrap();
572 fn test_read_until() {
573 let inner = MemReader::new(vec!(0, 1, 2, 1, 0));
574 let mut reader = BufferedReader::with_capacity(2, inner);
575 assert_eq!(reader.read_until(0), Ok(vec!(0)));
576 assert_eq!(reader.read_until(2), Ok(vec!(1, 2)));
577 assert_eq!(reader.read_until(1), Ok(vec!(1)));
578 assert_eq!(reader.read_until(8), Ok(vec!(0)));
579 assert!(reader.read_until(9).is_err());
583 fn test_line_buffer() {
584 let mut writer = LineBufferedWriter::new(Vec::new());
585 writer.write(&[0]).unwrap();
587 assert_eq!(&writer.get_ref()[], b);
588 writer.write(&[1]).unwrap();
589 assert_eq!(&writer.get_ref()[], b);
590 writer.flush().unwrap();
591 let b: &[_] = &[0, 1];
592 assert_eq!(&writer.get_ref()[], b);
593 writer.write(&[0, b'\n', 1, b'\n', 2]).unwrap();
594 let b: &[_] = &[0, 1, 0, b'\n', 1, b'\n'];
595 assert_eq!(&writer.get_ref()[], b);
596 writer.flush().unwrap();
597 let b: &[_] = &[0, 1, 0, b'\n', 1, b'\n', 2];
598 assert_eq!(&writer.get_ref()[], b);
599 writer.write(&[3, b'\n']).unwrap();
600 let b: &[_] = &[0, 1, 0, b'\n', 1, b'\n', 2, 3, b'\n'];
601 assert_eq!(&writer.get_ref()[], b);
605 fn test_read_line() {
606 let in_buf = MemReader::new(b"a\nb\nc".to_vec());
607 let mut reader = BufferedReader::with_capacity(2, in_buf);
608 assert_eq!(reader.read_line(), Ok("a\n".to_string()));
609 assert_eq!(reader.read_line(), Ok("b\n".to_string()));
610 assert_eq!(reader.read_line(), Ok("c".to_string()));
611 assert!(reader.read_line().is_err());
616 let in_buf = MemReader::new(b"a\nb\nc".to_vec());
617 let mut reader = BufferedReader::with_capacity(2, in_buf);
618 let mut it = reader.lines();
619 assert_eq!(it.next(), Some(Ok("a\n".to_string())));
620 assert_eq!(it.next(), Some(Ok("b\n".to_string())));
621 assert_eq!(it.next(), Some(Ok("c".to_string())));
622 assert_eq!(it.next(), None);
626 fn test_short_reads() {
627 let inner = ShortReader{lengths: vec![0, 1, 2, 0, 1, 0]};
628 let mut reader = BufferedReader::new(inner);
629 let mut buf = [0, 0];
630 assert_eq!(reader.read(&mut buf), Ok(0));
631 assert_eq!(reader.read(&mut buf), Ok(1));
632 assert_eq!(reader.read(&mut buf), Ok(2));
633 assert_eq!(reader.read(&mut buf), Ok(0));
634 assert_eq!(reader.read(&mut buf), Ok(1));
635 assert_eq!(reader.read(&mut buf), Ok(0));
636 assert!(reader.read(&mut buf).is_err());
640 fn read_char_buffered() {
641 let buf = [195u8, 159u8];
642 let mut reader = BufferedReader::with_capacity(1, &buf[]);
643 assert_eq!(reader.read_char(), Ok('ß'));
648 let buf = [195u8, 159u8, b'a'];
649 let mut reader = BufferedReader::with_capacity(1, &buf[]);
650 let mut it = reader.chars();
651 assert_eq!(it.next(), Some(Ok('ß')));
652 assert_eq!(it.next(), Some(Ok('a')));
653 assert_eq!(it.next(), None);
658 fn dont_panic_in_drop_on_panicked_flush() {
659 struct FailFlushWriter;
661 impl Writer for FailFlushWriter {
662 fn write(&mut self, _buf: &[u8]) -> IoResult<()> { Ok(()) }
663 fn flush(&mut self) -> IoResult<()> { Err(io::standard_error(EndOfFile)) }
666 let writer = FailFlushWriter;
667 let _writer = BufferedWriter::new(writer);
669 // If writer panics *again* due to the flush error then the process will abort.
674 fn bench_buffered_reader(b: &mut Bencher) {
676 BufferedReader::new(NullStream)
681 fn bench_buffered_writer(b: &mut Bencher) {
683 BufferedWriter::new(NullStream)
688 fn bench_buffered_stream(b: &mut Bencher) {
690 BufferedStream::new(NullStream);