Add Utf16Encoder. Generalize Utf16CodeUnits for any Iterator<char>.

This allows encoding to UTF-16 something that is not in UTF-8, e.g. a
`[char]` UTF-32 string.

This might help with https://github.com/servo/servo/issues/4023

diff --git a/src/libcollections/str.rs b/src/libcollections/str.rs
index aaa7da312f29a2f37420b04eedb9cedff69165af..a8bfc29151f33f6e611782f563e8784fe644f26c 100644 (file)
--- a/src/libcollections/str.rs
+++ b/src/libcollections/str.rs
@@ -74,7 +74,8 @@
 pub use core::str::{from_utf8, CharEq, Chars, CharOffsets};
 pub use core::str::{Bytes, CharSplits};
 pub use core::str::{CharSplitsN, AnyLines, MatchIndices, StrSplits};
-pub use core::str::{Utf16CodeUnits, eq_slice, is_utf8, is_utf16, Utf16Items};
+pub use core::str::{Utf16Encoder, Utf16CodeUnits};
+pub use core::str::{eq_slice, is_utf8, is_utf16, Utf16Items};
 pub use core::str::{Utf16Item, ScalarValue, LoneSurrogate, utf16_items};
 pub use core::str::{truncate_utf16_at_nul, utf8_char_width, CharRange};
 pub use core::str::{FromStr, from_str};

diff --git a/src/libcore/str.rs b/src/libcore/str.rs
index 24f26b15f27ac97d4ee9c6b1b2c09034c9234902..68e490ecb19c44e2b233667c0cf39bfd4d4ccb72 100644 (file)
--- a/src/libcore/str.rs
+++ b/src/libcore/str.rs
@@ -762,11 +762,33 @@ fn next(&mut self) -> Option<&'a str> {
 /// Use with the `std::iter` module.
 #[deriving(Clone)]
 pub struct Utf16CodeUnits<'a> {
-    chars: Chars<'a>,
-    extra: u16
+    encoder: Utf16Encoder<Chars<'a>>
 }
 
 impl<'a> Iterator<u16> for Utf16CodeUnits<'a> {
+    #[inline]
+    fn next(&mut self) -> Option<u16> { self.encoder.next() }
+
+    #[inline]
+    fn size_hint(&self) -> (uint, Option<uint>) { self.encoder.size_hint() }
+}
+
+
+/// Iterator adaptor for encoding `char`s to UTF-16.
+#[deriving(Clone)]
+pub struct Utf16Encoder<I> {
+    chars: I,
+    extra: u16
+}
+
+impl<I> Utf16Encoder<I> {
+    /// Create an UTF-16 encoder from any `char` iterator.
+    pub fn new(chars: I) -> Utf16Encoder<I> where I: Iterator<char> {
+        Utf16Encoder { chars: chars, extra: 0 }
+    }
+}
+
+impl<I> Iterator<u16> for Utf16Encoder<I> where I: Iterator<char> {
     #[inline]
     fn next(&mut self) -> Option<u16> {
         if self.extra != 0 {
@@ -2225,7 +2247,7 @@ fn as_ptr(&self) -> *const u8 {
 
     #[inline]
     fn utf16_units(&self) -> Utf16CodeUnits {
-        Utf16CodeUnits{ chars: self.chars(), extra: 0}
+        Utf16CodeUnits { encoder: Utf16Encoder::new(self.chars()) }
     }
 
     #[inline]

diff --git a/src/libcoretest/str.rs b/src/libcoretest/str.rs
index 5f44fd807ccae726d3ecf852129b9f777565dbc6..9574aeb3762de230f7e2244809796a90aae6a170 100644 (file)
--- a/src/libcoretest/str.rs
+++ b/src/libcoretest/str.rs
@@ -114,3 +114,10 @@ fn test_rev_split_char_iterator_no_trailing() {
     split.reverse();
     assert_eq!(split, vec!["", "Märy häd ä little lämb", "Little lämb"]);
 }
+
+#[test]
+fn test_utf16_code_units() {
+    use core::str::Utf16Encoder;
+    assert_eq!(Utf16Encoder::new(vec!['é', '\U0001F4A9'].into_iter()).collect::<Vec<u16>>(),
+               vec![0xE9, 0xD83D, 0xDCA9])
+}