serialize: base64: allow LF in addition to CRLF and optimize slightly

It is useful to have configurable newlines in base64 as the standard
leaves that for the implementation to decide.  GNU `base64` apparently
uses LF, which meant in `uutils` we had to manually convert the CRLF to
LF.  This made the program very slow for large inputs.

[breaking-change]

diff --git a/src/libserialize/base64.rs b/src/libserialize/base64.rs
index dd5039c9b828317a20d20365422022e982497803..1cb8fdd025d38c049a61538c424b55e353820f8f 100644 (file)
--- a/src/libserialize/base64.rs
+++ b/src/libserialize/base64.rs
@@ -1,4 +1,4 @@
-// Copyright 2012-2013 The Rust Project Developers. See the COPYRIGHT
+// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
 // file at the top-level directory of this distribution and at
 // http://rust-lang.org/COPYRIGHT.
 //
@@ -14,6 +14,7 @@
 
 pub use self::FromBase64Error::*;
 pub use self::CharacterSet::*;
+pub use self::Newline::*;
 
 use std::fmt;
 use std::error;
@@ -28,10 +29,22 @@ pub enum CharacterSet {
 
 impl Copy for CharacterSet {}
 
+/// Available newline types
+pub enum Newline {
+    /// A linefeed (i.e. Unix-style newline)
+    LF,
+    /// A carriage return and a linefeed (i.e. Windows-style newline)
+    CRLF
+}
+
+impl Copy for Newline {}
+
 /// Contains configuration parameters for `to_base64`.
 pub struct Config {
     /// Character set to use
     pub char_set: CharacterSet,
+    /// Newline to use
+    pub newline: Newline,
     /// True to pad output with `=` characters
     pub pad: bool,
     /// `Some(len)` to wrap lines at `len`, `None` to disable line wrapping
@@ -42,15 +55,15 @@ impl Copy for Config {}
 
 /// Configuration for RFC 4648 standard base64 encoding
 pub static STANDARD: Config =
-    Config {char_set: Standard, pad: true, line_length: None};
+    Config {char_set: Standard, newline: CRLF, pad: true, line_length: None};
 
 /// Configuration for RFC 4648 base64url encoding
 pub static URL_SAFE: Config =
-    Config {char_set: UrlSafe, pad: false, line_length: None};
+    Config {char_set: UrlSafe, newline: CRLF, pad: false, line_length: None};
 
 /// Configuration for RFC 2045 MIME base64 encoding
 pub static MIME: Config =
-    Config {char_set: Standard, pad: true, line_length: Some(76)};
+    Config {char_set: Standard, newline: CRLF, pad: true, line_length: Some(76)};
 
 static STANDARD_CHARS: &'static[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ\
                                         abcdefghijklmnopqrstuvwxyz\
@@ -87,24 +100,29 @@ fn to_base64(&self, config: Config) -> String {
             UrlSafe => URLSAFE_CHARS
         };
 
-        let mut v = Vec::new();
+        // In general, this Vec only needs (4/3) * self.len() memory, but
+        // addition is faster than multiplication and division.
+        let mut v = Vec::with_capacity(self.len() + self.len());
         let mut i = 0;
         let mut cur_length = 0;
         let len = self.len();
-        while i < len - (len % 3) {
-            match config.line_length {
-                Some(line_length) =>
-                    if cur_length >= line_length {
-                        v.push(b'\r');
-                        v.push(b'\n');
-                        cur_length = 0;
-                    },
-                None => ()
+        let mod_len = len % 3;
+        let cond_len = len - mod_len;
+        while i < cond_len {
+            let (first, second, third) = (self[i], self[i + 1], self[i + 2]);
+            if let Some(line_length) = config.line_length {
+                if cur_length >= line_length {
+                    v.push_all(match config.newline {
+                        LF => b"\n",
+                        CRLF => b"\r\n"
+                    });
+                    cur_length = 0;
+                }
             }
 
-            let n = (self[i] as u32) << 16 |
-                    (self[i + 1] as u32) << 8 |
-                    (self[i + 2] as u32);
+            let n = (first  as u32) << 16 |
+                    (second as u32) << 8 |
+                    (third  as u32);
 
             // This 24-bit number gets separated into four 6-bit numbers.
             v.push(bytes[((n >> 18) & 63) as uint]);
@@ -116,20 +134,20 @@ fn to_base64(&self, config: Config) -> String {
             i += 3;
         }
 
-        if len % 3 != 0 {
-            match config.line_length {
-                Some(line_length) =>
-                    if cur_length >= line_length {
-                        v.push(b'\r');
-                        v.push(b'\n');
-                    },
-                None => ()
+        if mod_len != 0 {
+            if let Some(line_length) = config.line_length {
+                if cur_length >= line_length {
+                    v.push_all(match config.newline {
+                        LF => b"\n",
+                        CRLF => b"\r\n"
+                    });
+                }
             }
         }
 
         // Heh, would be cool if we knew this was exhaustive
         // (the dream of bounded integer types)
-        match len % 3 {
+        match mod_len {
             0 => (),
             1 => {
                 let n = (self[i] as u32) << 16;
@@ -232,7 +250,7 @@ fn from_base64(&self) -> Result<Vec<u8>, FromBase64Error> {
 
 impl FromBase64 for [u8] {
     fn from_base64(&self) -> Result<Vec<u8>, FromBase64Error> {
-        let mut r = Vec::new();
+        let mut r = Vec::with_capacity(self.len());
         let mut buf: u32 = 0;
         let mut modulus = 0i;
 
@@ -288,7 +306,7 @@ fn from_base64(&self) -> Result<Vec<u8>, FromBase64Error> {
 mod tests {
     extern crate test;
     use self::test::Bencher;
-    use base64::{Config, FromBase64, ToBase64, STANDARD, URL_SAFE};
+    use base64::{Config, FromBase64, ToBase64, STANDARD, URL_SAFE, LF};
 
     #[test]
     fn test_to_base64_basic() {
@@ -302,7 +320,7 @@ fn test_to_base64_basic() {
     }
 
     #[test]
-    fn test_to_base64_line_break() {
+    fn test_to_base64_crlf_line_break() {
         assert!(![0u8, ..1000].to_base64(Config {line_length: None, ..STANDARD})
                               .contains("\r\n"));
         assert_eq!("foobar".as_bytes().to_base64(Config {line_length: Some(4),
@@ -310,6 +328,18 @@ fn test_to_base64_line_break() {
                    "Zm9v\r\nYmFy");
     }
 
+    #[test]
+    fn test_to_base64_lf_line_break() {
+        assert!(![0u8, ..1000].to_base64(Config {line_length: None, newline: LF,
+                                                 ..STANDARD})
+                              .as_slice()
+                              .contains("\n"));
+        assert_eq!("foobar".as_bytes().to_base64(Config {line_length: Some(4),
+                                                         newline: LF,
+                                                         ..STANDARD}),
+                   "Zm9v\nYmFy".to_string());
+    }
+
     #[test]
     fn test_to_base64_padding() {
         assert_eq!("f".as_bytes().to_base64(Config {pad: false, ..STANDARD}), "Zg");
@@ -344,6 +374,10 @@ fn test_from_base64_newlines() {
                    b"foobar");
         assert_eq!("Zm9vYg==\r\n".from_base64().unwrap(),
                    b"foob");
+        assert_eq!("Zm9v\nYmFy".from_base64().unwrap(),
+                   b"foobar");
+        assert_eq!("Zm9vYg==\n".from_base64().unwrap(),
+                   b"foob");
     }
 
     #[test]

diff --git a/src/libserialize/lib.rs b/src/libserialize/lib.rs
index 9711d5c7209be7c123c5910bbaea781adf9fdabc..1cff4c334e7430d1d8827e6e89281e0c862dc934 100644 (file)
--- a/src/libserialize/lib.rs
+++ b/src/libserialize/lib.rs
@@ -23,7 +23,7 @@
        html_root_url = "http://doc.rust-lang.org/nightly/",
        html_playground_url = "http://play.rust-lang.org/")]
 #![allow(unknown_features)]
-#![feature(macro_rules, default_type_params, phase, slicing_syntax, globs)]
+#![feature(macro_rules, default_type_params, phase, slicing_syntax, globs, if_let)]
 
 // test harness access
 #[cfg(test)]