1 //! UTF-8 and UTF-16 decoding iterators
4 use crate::error::Error;
7 use super::from_u32_unchecked;
9 /// An iterator that decodes UTF-16 encoded code points from an iterator of `u16`s.
11 /// This `struct` is created by the [`decode_utf16`] method on [`char`]. See its
12 /// documentation for more.
14 /// [`decode_utf16`]: char::decode_utf16
15 #[stable(feature = "decode_utf16", since = "1.9.0")]
16 #[derive(Clone, Debug)]
17 pub struct DecodeUtf16<I>
19 I: Iterator<Item = u16>,
25 /// An error that can be returned when decoding UTF-16 code points.
27 /// This `struct` is created when using the [`DecodeUtf16`] type.
28 #[stable(feature = "decode_utf16", since = "1.9.0")]
29 #[derive(Debug, Clone, Eq, PartialEq)]
30 pub struct DecodeUtf16Error {
34 /// Creates an iterator over the UTF-16 encoded code points in `iter`,
35 /// returning unpaired surrogates as `Err`s. See [`char::decode_utf16`].
37 pub(super) fn decode_utf16<I: IntoIterator<Item = u16>>(iter: I) -> DecodeUtf16<I::IntoIter> {
38 DecodeUtf16 { iter: iter.into_iter(), buf: None }
41 #[stable(feature = "decode_utf16", since = "1.9.0")]
42 impl<I: Iterator<Item = u16>> Iterator for DecodeUtf16<I> {
43 type Item = Result<char, DecodeUtf16Error>;
45 fn next(&mut self) -> Option<Result<char, DecodeUtf16Error>> {
46 let u = match self.buf.take() {
48 None => self.iter.next()?,
51 if !u.is_utf16_surrogate() {
52 // SAFETY: not a surrogate
53 Some(Ok(unsafe { from_u32_unchecked(u as u32) }))
54 } else if u >= 0xDC00 {
55 // a trailing surrogate
56 Some(Err(DecodeUtf16Error { code: u }))
58 let u2 = match self.iter.next() {
61 None => return Some(Err(DecodeUtf16Error { code: u })),
63 if u2 < 0xDC00 || u2 > 0xDFFF {
64 // not a trailing surrogate so we're not a valid
65 // surrogate pair, so rewind to redecode u2 next time.
67 return Some(Err(DecodeUtf16Error { code: u }));
70 // all ok, so lets decode it.
71 let c = (((u - 0xD800) as u32) << 10 | (u2 - 0xDC00) as u32) + 0x1_0000;
72 // SAFETY: we checked that it's a legal unicode value
73 Some(Ok(unsafe { from_u32_unchecked(c) }))
78 fn size_hint(&self) -> (usize, Option<usize>) {
79 let (low, high) = self.iter.size_hint();
81 let (low_buf, high_buf) = match self.buf {
82 // buf is empty, no additional elements from it.
84 // `u` is a non surrogate, so it's always an additional character.
85 Some(u) if !u.is_utf16_surrogate() => (1, 1),
86 // `u` is a leading surrogate (it can never be a trailing surrogate and
87 // it's a surrogate due to the previous branch) and `self.iter` is empty.
89 // `u` can't be paired, since the `self.iter` is empty,
90 // so it will always become an additional element (error).
91 Some(_u) if high == Some(0) => (1, 1),
92 // `u` is a leading surrogate and `iter` may be non-empty.
94 // `u` can either pair with a trailing surrogate, in which case no additional elements
95 // are produced, or it can become an error, in which case it's an additional character (error).
99 // `self.iter` could contain entirely valid surrogates (2 elements per
100 // char), or entirely non-surrogates (1 element per char).
102 // On odd lower bound, at least one element must stay unpaired
103 // (with other elements from `self.iter`), so we round up.
104 let low = low.div_ceil(2) + low_buf;
105 let high = high.and_then(|h| h.checked_add(high_buf));
111 impl DecodeUtf16Error {
112 /// Returns the unpaired surrogate which caused this error.
114 #[stable(feature = "decode_utf16", since = "1.9.0")]
115 pub fn unpaired_surrogate(&self) -> u16 {
120 #[stable(feature = "decode_utf16", since = "1.9.0")]
121 impl fmt::Display for DecodeUtf16Error {
122 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
123 write!(f, "unpaired surrogate found: {:x}", self.code)
127 #[cfg(not(bootstrap))]
128 #[stable(feature = "decode_utf16", since = "1.9.0")]
129 impl Error for DecodeUtf16Error {
131 fn description(&self) -> &str {
132 "unpaired surrogate found"