1 // Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
2 // file at the top-level directory of this distribution and at
3 // http://rust-lang.org/COPYRIGHT.
5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8 // option. This file may not be copied, modified, or distributed
9 // except according to those terms.
11 //! Character manipulation.
13 //! For more details, see ::std_unicode::char (a.k.a. std::char)
15 #![allow(non_snake_case)]
16 #![stable(feature = "core_char", since = "1.2.0")]
18 use char_private::is_printable;
20 use fmt::{self, Write};
22 use str::{from_utf8_unchecked_mut, FromStr};
23 use iter::FusedIterator;
26 // UTF-8 ranges and tags for encoding characters
27 const TAG_CONT: u8 = 0b1000_0000;
28 const TAG_TWO_B: u8 = 0b1100_0000;
29 const TAG_THREE_B: u8 = 0b1110_0000;
30 const TAG_FOUR_B: u8 = 0b1111_0000;
31 const MAX_ONE_B: u32 = 0x80;
32 const MAX_TWO_B: u32 = 0x800;
33 const MAX_THREE_B: u32 = 0x10000;
36 Lu Uppercase_Letter an uppercase letter
37 Ll Lowercase_Letter a lowercase letter
38 Lt Titlecase_Letter a digraphic character, with first part uppercase
39 Lm Modifier_Letter a modifier letter
40 Lo Other_Letter other letters, including syllables and ideographs
41 Mn Nonspacing_Mark a nonspacing combining mark (zero advance width)
42 Mc Spacing_Mark a spacing combining mark (positive advance width)
43 Me Enclosing_Mark an enclosing combining mark
44 Nd Decimal_Number a decimal digit
45 Nl Letter_Number a letterlike numeric character
46 No Other_Number a numeric character of other type
47 Pc Connector_Punctuation a connecting punctuation mark, like a tie
48 Pd Dash_Punctuation a dash or hyphen punctuation mark
49 Ps Open_Punctuation an opening punctuation mark (of a pair)
50 Pe Close_Punctuation a closing punctuation mark (of a pair)
51 Pi Initial_Punctuation an initial quotation mark
52 Pf Final_Punctuation a final quotation mark
53 Po Other_Punctuation a punctuation mark of other type
54 Sm Math_Symbol a symbol of primarily mathematical use
55 Sc Currency_Symbol a currency sign
56 Sk Modifier_Symbol a non-letterlike modifier symbol
57 So Other_Symbol a symbol of other type
58 Zs Space_Separator a space character (of various non-zero widths)
59 Zl Line_Separator U+2028 LINE SEPARATOR only
60 Zp Paragraph_Separator U+2029 PARAGRAPH SEPARATOR only
61 Cc Control a C0 or C1 control code
62 Cf Format a format control character
63 Cs Surrogate a surrogate code point
64 Co Private_Use a private-use character
65 Cn Unassigned a reserved unassigned code point or a noncharacter
68 /// The highest valid code point a `char` can have.
70 /// A [`char`] is a [Unicode Scalar Value], which means that it is a [Code
71 /// Point], but only ones within a certain range. `MAX` is the highest valid
72 /// code point that's a valid [Unicode Scalar Value].
74 /// [`char`]: ../../std/primitive.char.html
75 /// [Unicode Scalar Value]: http://www.unicode.org/glossary/#unicode_scalar_value
76 /// [Code Point]: http://www.unicode.org/glossary/#code_point
77 #[stable(feature = "rust1", since = "1.0.0")]
78 pub const MAX: char = '\u{10ffff}';
80 /// Converts a `u32` to a `char`.
82 /// Note that all [`char`]s are valid [`u32`]s, and can be casted to one with
89 /// assert_eq!(128175, i);
92 /// However, the reverse is not true: not all valid [`u32`]s are valid
93 /// [`char`]s. `from_u32()` will return `None` if the input is not a valid value
96 /// [`char`]: ../../std/primitive.char.html
97 /// [`u32`]: ../../std/primitive.u32.html
98 /// [`as`]: ../../book/first-edition/casting-between-types.html#as
100 /// For an unsafe version of this function which ignores these checks, see
101 /// [`from_u32_unchecked`].
103 /// [`from_u32_unchecked`]: fn.from_u32_unchecked.html
112 /// let c = char::from_u32(0x2764);
114 /// assert_eq!(Some('❤'), c);
117 /// Returning `None` when the input is not a valid [`char`]:
122 /// let c = char::from_u32(0x110000);
124 /// assert_eq!(None, c);
127 #[stable(feature = "rust1", since = "1.0.0")]
128 pub fn from_u32(i: u32) -> Option<char> {
129 char::try_from(i).ok()
132 /// Converts a `u32` to a `char`, ignoring validity.
134 /// Note that all [`char`]s are valid [`u32`]s, and can be casted to one with
139 /// let i = c as u32;
141 /// assert_eq!(128175, i);
144 /// However, the reverse is not true: not all valid [`u32`]s are valid
145 /// [`char`]s. `from_u32_unchecked()` will ignore this, and blindly cast to
146 /// [`char`], possibly creating an invalid one.
148 /// [`char`]: ../../std/primitive.char.html
149 /// [`u32`]: ../../std/primitive.u32.html
150 /// [`as`]: ../../book/first-edition/casting-between-types.html#as
154 /// This function is unsafe, as it may construct invalid `char` values.
156 /// For a safe version of this function, see the [`from_u32`] function.
158 /// [`from_u32`]: fn.from_u32.html
167 /// let c = unsafe { char::from_u32_unchecked(0x2764) };
169 /// assert_eq!('❤', c);
172 #[stable(feature = "char_from_unchecked", since = "1.5.0")]
173 pub unsafe fn from_u32_unchecked(i: u32) -> char {
177 #[stable(feature = "char_convert", since = "1.13.0")]
178 impl From<char> for u32 {
180 fn from(c: char) -> Self {
185 /// Maps a byte in 0x00...0xFF to a `char` whose code point has the same value, in U+0000 to U+00FF.
187 /// Unicode is designed such that this effectively decodes bytes
188 /// with the character encoding that IANA calls ISO-8859-1.
189 /// This encoding is compatible with ASCII.
191 /// Note that this is different from ISO/IEC 8859-1 a.k.a. ISO 8859-1 (with one less hyphen),
192 /// which leaves some "blanks", byte values that are not assigned to any character.
193 /// ISO-8859-1 (the IANA one) assigns them to the C0 and C1 control codes.
195 /// Note that this is *also* different from Windows-1252 a.k.a. code page 1252,
196 /// which is a superset ISO/IEC 8859-1 that assigns some (not all!) blanks
197 /// to punctuation and various Latin characters.
199 /// To confuse things further, [on the Web](https://encoding.spec.whatwg.org/)
200 /// `ascii`, `iso-8859-1`, and `windows-1252` are all aliases
201 /// for a superset of Windows-1252 that fills the remaining blanks with corresponding
202 /// C0 and C1 control codes.
203 #[stable(feature = "char_convert", since = "1.13.0")]
204 impl From<u8> for char {
206 fn from(i: u8) -> Self {
212 /// An error which can be returned when parsing a char.
213 #[stable(feature = "char_from_str", since = "1.20.0")]
214 #[derive(Clone, Debug)]
215 pub struct ParseCharError {
219 impl ParseCharError {
220 #[unstable(feature = "char_error_internals",
221 reason = "this method should not be available publicly",
224 pub fn __description(&self) -> &str {
226 CharErrorKind::EmptyString => {
227 "cannot parse char from empty string"
229 CharErrorKind::TooManyChars => "too many characters in string"
234 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
240 #[stable(feature = "char_from_str", since = "1.20.0")]
241 impl fmt::Display for ParseCharError {
242 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
243 self.__description().fmt(f)
248 #[stable(feature = "char_from_str", since = "1.20.0")]
249 impl FromStr for char {
250 type Err = ParseCharError;
253 fn from_str(s: &str) -> Result<Self, Self::Err> {
254 let mut chars = s.chars();
255 match (chars.next(), chars.next()) {
257 Err(ParseCharError { kind: CharErrorKind::EmptyString })
259 (Some(c), None) => Ok(c),
261 Err(ParseCharError { kind: CharErrorKind::TooManyChars })
268 #[unstable(feature = "try_from", issue = "33417")]
269 impl TryFrom<u32> for char {
270 type Error = CharTryFromError;
273 fn try_from(i: u32) -> Result<Self, Self::Error> {
274 if (i > MAX as u32) || (i >= 0xD800 && i <= 0xDFFF) {
275 Err(CharTryFromError(()))
277 Ok(unsafe { from_u32_unchecked(i) })
282 /// The error type returned when a conversion from u32 to char fails.
283 #[unstable(feature = "try_from", issue = "33417")]
284 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
285 pub struct CharTryFromError(());
287 #[unstable(feature = "try_from", issue = "33417")]
288 impl fmt::Display for CharTryFromError {
289 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
290 "converted integer out of range for `char`".fmt(f)
294 /// Converts a digit in the given radix to a `char`.
296 /// A 'radix' here is sometimes also called a 'base'. A radix of two
297 /// indicates a binary number, a radix of ten, decimal, and a radix of
298 /// sixteen, hexadecimal, to give some common values. Arbitrary
299 /// radices are supported.
301 /// `from_digit()` will return `None` if the input is not a digit in
306 /// Panics if given a radix larger than 36.
315 /// let c = char::from_digit(4, 10);
317 /// assert_eq!(Some('4'), c);
319 /// // Decimal 11 is a single digit in base 16
320 /// let c = char::from_digit(11, 16);
322 /// assert_eq!(Some('b'), c);
325 /// Returning `None` when the input is not a digit:
330 /// let c = char::from_digit(20, 10);
332 /// assert_eq!(None, c);
335 /// Passing a large radix, causing a panic:
341 /// let result = thread::spawn(|| {
343 /// let c = char::from_digit(1, 37);
346 /// assert!(result.is_err());
349 #[stable(feature = "rust1", since = "1.0.0")]
350 pub fn from_digit(num: u32, radix: u32) -> Option<char> {
352 panic!("from_digit: radix is too high (maximum 36)");
357 Some((b'0' + num) as char)
359 Some((b'a' + num - 10) as char)
366 // NB: the stabilization and documentation for this trait is in
367 // unicode/char.rs, not here
368 #[allow(missing_docs)] // docs in libunicode/u_char.rs
370 #[unstable(feature = "core_char_ext",
371 reason = "the stable interface is `impl char` in later crate",
374 #[stable(feature = "core", since = "1.6.0")]
375 fn is_digit(self, radix: u32) -> bool;
376 #[stable(feature = "core", since = "1.6.0")]
377 fn to_digit(self, radix: u32) -> Option<u32>;
378 #[stable(feature = "core", since = "1.6.0")]
379 fn escape_unicode(self) -> EscapeUnicode;
380 #[stable(feature = "core", since = "1.6.0")]
381 fn escape_default(self) -> EscapeDefault;
382 #[stable(feature = "char_escape_debug", since = "1.20.0")]
383 fn escape_debug(self) -> EscapeDebug;
384 #[stable(feature = "core", since = "1.6.0")]
385 fn len_utf8(self) -> usize;
386 #[stable(feature = "core", since = "1.6.0")]
387 fn len_utf16(self) -> usize;
388 #[stable(feature = "unicode_encode_char", since = "1.15.0")]
389 fn encode_utf8(self, dst: &mut [u8]) -> &mut str;
390 #[stable(feature = "unicode_encode_char", since = "1.15.0")]
391 fn encode_utf16(self, dst: &mut [u16]) -> &mut [u16];
394 #[stable(feature = "core", since = "1.6.0")]
395 impl CharExt for char {
397 fn is_digit(self, radix: u32) -> bool {
398 self.to_digit(radix).is_some()
402 fn to_digit(self, radix: u32) -> Option<u32> {
404 panic!("to_digit: radix is too high (maximum 36)");
406 let val = match self {
407 '0' ... '9' => self as u32 - '0' as u32,
408 'a' ... 'z' => self as u32 - 'a' as u32 + 10,
409 'A' ... 'Z' => self as u32 - 'A' as u32 + 10,
412 if val < radix { Some(val) }
417 fn escape_unicode(self) -> EscapeUnicode {
420 // or-ing 1 ensures that for c==0 the code computes that one
421 // digit should be printed and (which is the same) avoids the
422 // (31 - 32) underflow
423 let msb = 31 - (c | 1).leading_zeros();
425 // the index of the most significant hex digit
426 let ms_hex_digit = msb / 4;
429 state: EscapeUnicodeState::Backslash,
430 hex_digit_idx: ms_hex_digit as usize,
435 fn escape_default(self) -> EscapeDefault {
436 let init_state = match self {
437 '\t' => EscapeDefaultState::Backslash('t'),
438 '\r' => EscapeDefaultState::Backslash('r'),
439 '\n' => EscapeDefaultState::Backslash('n'),
440 '\\' | '\'' | '"' => EscapeDefaultState::Backslash(self),
441 '\x20' ... '\x7e' => EscapeDefaultState::Char(self),
442 _ => EscapeDefaultState::Unicode(self.escape_unicode())
444 EscapeDefault { state: init_state }
448 fn escape_debug(self) -> EscapeDebug {
449 let init_state = match self {
450 '\t' => EscapeDefaultState::Backslash('t'),
451 '\r' => EscapeDefaultState::Backslash('r'),
452 '\n' => EscapeDefaultState::Backslash('n'),
453 '\\' | '\'' | '"' => EscapeDefaultState::Backslash(self),
454 c if is_printable(c) => EscapeDefaultState::Char(c),
455 c => EscapeDefaultState::Unicode(c.escape_unicode()),
457 EscapeDebug(EscapeDefault { state: init_state })
461 fn len_utf8(self) -> usize {
462 let code = self as u32;
463 if code < MAX_ONE_B {
465 } else if code < MAX_TWO_B {
467 } else if code < MAX_THREE_B {
475 fn len_utf16(self) -> usize {
476 let ch = self as u32;
477 if (ch & 0xFFFF) == ch { 1 } else { 2 }
481 fn encode_utf8(self, dst: &mut [u8]) -> &mut str {
482 let code = self as u32;
485 if code < MAX_ONE_B && !dst.is_empty() {
486 *dst.get_unchecked_mut(0) = code as u8;
488 } else if code < MAX_TWO_B && dst.len() >= 2 {
489 *dst.get_unchecked_mut(0) = (code >> 6 & 0x1F) as u8 | TAG_TWO_B;
490 *dst.get_unchecked_mut(1) = (code & 0x3F) as u8 | TAG_CONT;
492 } else if code < MAX_THREE_B && dst.len() >= 3 {
493 *dst.get_unchecked_mut(0) = (code >> 12 & 0x0F) as u8 | TAG_THREE_B;
494 *dst.get_unchecked_mut(1) = (code >> 6 & 0x3F) as u8 | TAG_CONT;
495 *dst.get_unchecked_mut(2) = (code & 0x3F) as u8 | TAG_CONT;
497 } else if dst.len() >= 4 {
498 *dst.get_unchecked_mut(0) = (code >> 18 & 0x07) as u8 | TAG_FOUR_B;
499 *dst.get_unchecked_mut(1) = (code >> 12 & 0x3F) as u8 | TAG_CONT;
500 *dst.get_unchecked_mut(2) = (code >> 6 & 0x3F) as u8 | TAG_CONT;
501 *dst.get_unchecked_mut(3) = (code & 0x3F) as u8 | TAG_CONT;
504 panic!("encode_utf8: need {} bytes to encode U+{:X}, but the buffer has {}",
505 from_u32_unchecked(code).len_utf8(),
509 from_utf8_unchecked_mut(dst.get_unchecked_mut(..len))
514 fn encode_utf16(self, dst: &mut [u16]) -> &mut [u16] {
515 let mut code = self as u32;
517 if (code & 0xFFFF) == code && !dst.is_empty() {
518 // The BMP falls through (assuming non-surrogate, as it should)
519 *dst.get_unchecked_mut(0) = code as u16;
520 slice::from_raw_parts_mut(dst.as_mut_ptr(), 1)
521 } else if dst.len() >= 2 {
522 // Supplementary planes break into surrogates.
524 *dst.get_unchecked_mut(0) = 0xD800 | ((code >> 10) as u16);
525 *dst.get_unchecked_mut(1) = 0xDC00 | ((code as u16) & 0x3FF);
526 slice::from_raw_parts_mut(dst.as_mut_ptr(), 2)
528 panic!("encode_utf16: need {} units to encode U+{:X}, but the buffer has {}",
529 from_u32_unchecked(code).len_utf16(),
537 /// Returns an iterator that yields the hexadecimal Unicode escape of a
538 /// character, as `char`s.
540 /// This `struct` is created by the [`escape_unicode`] method on [`char`]. See
541 /// its documentation for more.
543 /// [`escape_unicode`]: ../../std/primitive.char.html#method.escape_unicode
544 /// [`char`]: ../../std/primitive.char.html
545 #[derive(Clone, Debug)]
546 #[stable(feature = "rust1", since = "1.0.0")]
547 pub struct EscapeUnicode {
549 state: EscapeUnicodeState,
551 // The index of the next hex digit to be printed (0 if none),
552 // i.e. the number of remaining hex digits to be printed;
553 // increasing from the least significant digit: 0x543210
554 hex_digit_idx: usize,
557 // The enum values are ordered so that their representation is the
558 // same as the remaining length (besides the hexadecimal digits). This
559 // likely makes `len()` a single load from memory) and inline-worth.
560 #[derive(Clone, Debug)]
561 enum EscapeUnicodeState {
570 #[stable(feature = "rust1", since = "1.0.0")]
571 impl Iterator for EscapeUnicode {
574 fn next(&mut self) -> Option<char> {
576 EscapeUnicodeState::Backslash => {
577 self.state = EscapeUnicodeState::Type;
580 EscapeUnicodeState::Type => {
581 self.state = EscapeUnicodeState::LeftBrace;
584 EscapeUnicodeState::LeftBrace => {
585 self.state = EscapeUnicodeState::Value;
588 EscapeUnicodeState::Value => {
589 let hex_digit = ((self.c as u32) >> (self.hex_digit_idx * 4)) & 0xf;
590 let c = from_digit(hex_digit, 16).unwrap();
591 if self.hex_digit_idx == 0 {
592 self.state = EscapeUnicodeState::RightBrace;
594 self.hex_digit_idx -= 1;
598 EscapeUnicodeState::RightBrace => {
599 self.state = EscapeUnicodeState::Done;
602 EscapeUnicodeState::Done => None,
607 fn size_hint(&self) -> (usize, Option<usize>) {
613 fn count(self) -> usize {
617 fn last(self) -> Option<char> {
619 EscapeUnicodeState::Done => None,
621 EscapeUnicodeState::RightBrace |
622 EscapeUnicodeState::Value |
623 EscapeUnicodeState::LeftBrace |
624 EscapeUnicodeState::Type |
625 EscapeUnicodeState::Backslash => Some('}'),
630 #[stable(feature = "exact_size_escape", since = "1.11.0")]
631 impl ExactSizeIterator for EscapeUnicode {
633 fn len(&self) -> usize {
634 // The match is a single memory access with no branching
635 self.hex_digit_idx + match self.state {
636 EscapeUnicodeState::Done => 0,
637 EscapeUnicodeState::RightBrace => 1,
638 EscapeUnicodeState::Value => 2,
639 EscapeUnicodeState::LeftBrace => 3,
640 EscapeUnicodeState::Type => 4,
641 EscapeUnicodeState::Backslash => 5,
646 #[unstable(feature = "fused", issue = "35602")]
647 impl FusedIterator for EscapeUnicode {}
649 #[stable(feature = "char_struct_display", since = "1.16.0")]
650 impl fmt::Display for EscapeUnicode {
651 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
652 for c in self.clone() {
659 /// An iterator that yields the literal escape code of a `char`.
661 /// This `struct` is created by the [`escape_default`] method on [`char`]. See
662 /// its documentation for more.
664 /// [`escape_default`]: ../../std/primitive.char.html#method.escape_default
665 /// [`char`]: ../../std/primitive.char.html
666 #[derive(Clone, Debug)]
667 #[stable(feature = "rust1", since = "1.0.0")]
668 pub struct EscapeDefault {
669 state: EscapeDefaultState
672 #[derive(Clone, Debug)]
673 enum EscapeDefaultState {
677 Unicode(EscapeUnicode),
680 #[stable(feature = "rust1", since = "1.0.0")]
681 impl Iterator for EscapeDefault {
684 fn next(&mut self) -> Option<char> {
686 EscapeDefaultState::Backslash(c) => {
687 self.state = EscapeDefaultState::Char(c);
690 EscapeDefaultState::Char(c) => {
691 self.state = EscapeDefaultState::Done;
694 EscapeDefaultState::Done => None,
695 EscapeDefaultState::Unicode(ref mut iter) => iter.next(),
700 fn size_hint(&self) -> (usize, Option<usize>) {
706 fn count(self) -> usize {
710 fn nth(&mut self, n: usize) -> Option<char> {
712 EscapeDefaultState::Backslash(c) if n == 0 => {
713 self.state = EscapeDefaultState::Char(c);
716 EscapeDefaultState::Backslash(c) if n == 1 => {
717 self.state = EscapeDefaultState::Done;
720 EscapeDefaultState::Backslash(_) => {
721 self.state = EscapeDefaultState::Done;
724 EscapeDefaultState::Char(c) => {
725 self.state = EscapeDefaultState::Done;
733 EscapeDefaultState::Done => return None,
734 EscapeDefaultState::Unicode(ref mut i) => return i.nth(n),
738 fn last(self) -> Option<char> {
740 EscapeDefaultState::Unicode(iter) => iter.last(),
741 EscapeDefaultState::Done => None,
742 EscapeDefaultState::Backslash(c) | EscapeDefaultState::Char(c) => Some(c),
747 #[stable(feature = "exact_size_escape", since = "1.11.0")]
748 impl ExactSizeIterator for EscapeDefault {
749 fn len(&self) -> usize {
751 EscapeDefaultState::Done => 0,
752 EscapeDefaultState::Char(_) => 1,
753 EscapeDefaultState::Backslash(_) => 2,
754 EscapeDefaultState::Unicode(ref iter) => iter.len(),
759 #[unstable(feature = "fused", issue = "35602")]
760 impl FusedIterator for EscapeDefault {}
762 #[stable(feature = "char_struct_display", since = "1.16.0")]
763 impl fmt::Display for EscapeDefault {
764 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
765 for c in self.clone() {
772 /// An iterator that yields the literal escape code of a `char`.
774 /// This `struct` is created by the [`escape_debug`] method on [`char`]. See its
775 /// documentation for more.
777 /// [`escape_debug`]: ../../std/primitive.char.html#method.escape_debug
778 /// [`char`]: ../../std/primitive.char.html
779 #[stable(feature = "char_escape_debug", since = "1.20.0")]
780 #[derive(Clone, Debug)]
781 pub struct EscapeDebug(EscapeDefault);
783 #[stable(feature = "char_escape_debug", since = "1.20.0")]
784 impl Iterator for EscapeDebug {
786 fn next(&mut self) -> Option<char> { self.0.next() }
787 fn size_hint(&self) -> (usize, Option<usize>) { self.0.size_hint() }
790 #[stable(feature = "char_escape_debug", since = "1.20.0")]
791 impl ExactSizeIterator for EscapeDebug { }
793 #[unstable(feature = "fused", issue = "35602")]
794 impl FusedIterator for EscapeDebug {}
796 #[stable(feature = "char_escape_debug", since = "1.20.0")]
797 impl fmt::Display for EscapeDebug {
798 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
799 fmt::Display::fmt(&self.0, f)
805 /// An iterator over an iterator of bytes of the characters the bytes represent
807 #[unstable(feature = "decode_utf8", issue = "33906")]
808 #[derive(Clone, Debug)]
809 pub struct DecodeUtf8<I: Iterator<Item = u8>>(::iter::Peekable<I>);
811 /// Decodes an `Iterator` of bytes as UTF-8.
812 #[unstable(feature = "decode_utf8", issue = "33906")]
814 pub fn decode_utf8<I: IntoIterator<Item = u8>>(i: I) -> DecodeUtf8<I::IntoIter> {
815 DecodeUtf8(i.into_iter().peekable())
818 /// `<DecodeUtf8 as Iterator>::next` returns this for an invalid input sequence.
819 #[unstable(feature = "decode_utf8", issue = "33906")]
820 #[derive(PartialEq, Eq, Debug)]
821 pub struct InvalidSequence(());
823 #[unstable(feature = "decode_utf8", issue = "33906")]
824 impl<I: Iterator<Item = u8>> Iterator for DecodeUtf8<I> {
825 type Item = Result<char, InvalidSequence>;
828 fn next(&mut self) -> Option<Result<char, InvalidSequence>> {
829 self.0.next().map(|first_byte| {
830 // Emit InvalidSequence according to
831 // Unicode §5.22 Best Practice for U+FFFD Substitution
832 // http://www.unicode.org/versions/Unicode9.0.0/ch05.pdf#G40630
834 // Roughly: consume at least one byte,
835 // then validate one byte at a time and stop before the first unexpected byte
836 // (which might be the valid start of the next byte sequence).
839 macro_rules! first_byte {
841 code_point = u32::from(first_byte & $mask)
844 macro_rules! continuation_byte {
845 () => { continuation_byte!(0x80...0xBF) };
847 match self.0.peek() {
848 Some(&byte @ $range) => {
849 code_point = (code_point << 6) | u32::from(byte & 0b0011_1111);
852 _ => return Err(InvalidSequence(()))
859 first_byte!(0b1111_1111);
862 first_byte!(0b0001_1111);
863 continuation_byte!();
866 first_byte!(0b0000_1111);
867 continuation_byte!(0xA0...0xBF); // 0x80...0x9F here are overlong
868 continuation_byte!();
870 0xE1...0xEC | 0xEE...0xEF => {
871 first_byte!(0b0000_1111);
872 continuation_byte!();
873 continuation_byte!();
876 first_byte!(0b0000_1111);
877 continuation_byte!(0x80...0x9F); // 0xA0..0xBF here are surrogates
878 continuation_byte!();
881 first_byte!(0b0000_0111);
882 continuation_byte!(0x90...0xBF); // 0x80..0x8F here are overlong
883 continuation_byte!();
884 continuation_byte!();
887 first_byte!(0b0000_0111);
888 continuation_byte!();
889 continuation_byte!();
890 continuation_byte!();
893 first_byte!(0b0000_0111);
894 continuation_byte!(0x80...0x8F); // 0x90..0xBF here are beyond char::MAX
895 continuation_byte!();
896 continuation_byte!();
898 _ => return Err(InvalidSequence(())) // Illegal first byte, overlong, or beyond MAX
901 Ok(from_u32_unchecked(code_point))
907 #[unstable(feature = "fused", issue = "35602")]
908 impl<I: FusedIterator<Item = u8>> FusedIterator for DecodeUtf8<I> {}