1 //! UTF-8 and UTF-16 decoding iterators
5 use super::from_u32_unchecked;
7 /// An iterator that decodes UTF-16 encoded code points from an iterator of `u16`s.
8 #[stable(feature = "decode_utf16", since = "1.9.0")]
9 #[derive(Clone, Debug)]
10 pub struct DecodeUtf16<I>
12 I: Iterator<Item = u16>,
18 /// An error that can be returned when decoding UTF-16 code points.
19 #[stable(feature = "decode_utf16", since = "1.9.0")]
20 #[derive(Debug, Clone, Eq, PartialEq)]
21 pub struct DecodeUtf16Error {
25 /// Creates an iterator over the UTF-16 encoded code points in `iter`,
26 /// returning unpaired surrogates as `Err`s.
33 /// use std::char::decode_utf16;
35 /// // 𝄞mus<invalid>ic<invalid>
37 /// 0xD834, 0xDD1E, 0x006d, 0x0075, 0x0073, 0xDD1E, 0x0069, 0x0063, 0xD834,
41 /// decode_utf16(v.iter().cloned())
42 /// .map(|r| r.map_err(|e| e.unpaired_surrogate()))
43 /// .collect::<Vec<_>>(),
46 /// Ok('m'), Ok('u'), Ok('s'),
54 /// A lossy decoder can be obtained by replacing `Err` results with the replacement character:
57 /// use std::char::{decode_utf16, REPLACEMENT_CHARACTER};
59 /// // 𝄞mus<invalid>ic<invalid>
61 /// 0xD834, 0xDD1E, 0x006d, 0x0075, 0x0073, 0xDD1E, 0x0069, 0x0063, 0xD834,
65 /// decode_utf16(v.iter().cloned())
66 /// .map(|r| r.unwrap_or(REPLACEMENT_CHARACTER))
67 /// .collect::<String>(),
71 #[stable(feature = "decode_utf16", since = "1.9.0")]
73 pub fn decode_utf16<I: IntoIterator<Item = u16>>(iter: I) -> DecodeUtf16<I::IntoIter> {
74 DecodeUtf16 { iter: iter.into_iter(), buf: None }
77 #[stable(feature = "decode_utf16", since = "1.9.0")]
78 impl<I: Iterator<Item = u16>> Iterator for DecodeUtf16<I> {
79 type Item = Result<char, DecodeUtf16Error>;
81 fn next(&mut self) -> Option<Result<char, DecodeUtf16Error>> {
82 let u = match self.buf.take() {
84 None => self.iter.next()?,
87 if u < 0xD800 || 0xDFFF < u {
88 // SAFETY: not a surrogate
89 Some(Ok(unsafe { from_u32_unchecked(u as u32) }))
90 } else if u >= 0xDC00 {
91 // a trailing surrogate
92 Some(Err(DecodeUtf16Error { code: u }))
94 let u2 = match self.iter.next() {
97 None => return Some(Err(DecodeUtf16Error { code: u })),
99 if u2 < 0xDC00 || u2 > 0xDFFF {
100 // not a trailing surrogate so we're not a valid
101 // surrogate pair, so rewind to redecode u2 next time.
103 return Some(Err(DecodeUtf16Error { code: u }));
106 // all ok, so lets decode it.
107 let c = (((u - 0xD800) as u32) << 10 | (u2 - 0xDC00) as u32) + 0x1_0000;
108 // SAFETY: we checked that it's a legal unicode value
109 Some(Ok(unsafe { from_u32_unchecked(c) }))
114 fn size_hint(&self) -> (usize, Option<usize>) {
115 let (low, high) = self.iter.size_hint();
116 // we could be entirely valid surrogates (2 elements per
117 // char), or entirely non-surrogates (1 element per char)
122 impl DecodeUtf16Error {
123 /// Returns the unpaired surrogate which caused this error.
124 #[stable(feature = "decode_utf16", since = "1.9.0")]
125 pub fn unpaired_surrogate(&self) -> u16 {
130 #[stable(feature = "decode_utf16", since = "1.9.0")]
131 impl fmt::Display for DecodeUtf16Error {
132 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
133 write!(f, "unpaired surrogate found: {:x}", self.code)