2 use crate::fmt::{self, Write};
5 use super::from_utf8_unchecked;
6 use super::validations::utf8_char_width;
8 /// Lossy UTF-8 string.
9 #[unstable(feature = "str_internals", issue = "none")]
10 pub struct Utf8Lossy {
16 pub fn from_bytes(bytes: &[u8]) -> &Utf8Lossy {
17 // SAFETY: Both use the same memory layout, and UTF-8 correctness isn't required.
18 unsafe { mem::transmute(bytes) }
21 pub fn chunks(&self) -> Utf8LossyChunksIter<'_> {
22 Utf8LossyChunksIter { source: &self.bytes }
26 /// Iterator over lossy UTF-8 string
27 #[must_use = "iterators are lazy and do nothing unless consumed"]
28 #[unstable(feature = "str_internals", issue = "none")]
29 #[allow(missing_debug_implementations)]
30 pub struct Utf8LossyChunksIter<'a> {
34 #[unstable(feature = "str_internals", issue = "none")]
35 #[derive(PartialEq, Eq, Debug)]
36 pub struct Utf8LossyChunk<'a> {
37 /// Sequence of valid chars.
38 /// Can be empty between broken UTF-8 chars.
40 /// Single broken char, empty if none.
41 /// Empty iff iterator item is last.
45 impl<'a> Iterator for Utf8LossyChunksIter<'a> {
46 type Item = Utf8LossyChunk<'a>;
48 fn next(&mut self) -> Option<Utf8LossyChunk<'a>> {
49 if self.source.is_empty() {
53 const TAG_CONT_U8: u8 = 128;
54 fn safe_get(xs: &[u8], i: usize) -> u8 {
55 *xs.get(i).unwrap_or(&0)
59 let mut valid_up_to = 0;
60 while i < self.source.len() {
61 // SAFETY: `i < self.source.len()` per previous line.
62 // For some reason the following are both significantly slower:
63 // while let Some(&byte) = self.source.get(i) {
64 // while let Some(byte) = self.source.get(i).copied() {
65 let byte = unsafe { *self.source.get_unchecked(i) };
69 // This could be a `1 => ...` case in the match below, but for
70 // the common case of all-ASCII inputs, we bypass loading the
71 // sizeable UTF8_CHAR_WIDTH table into cache.
73 let w = utf8_char_width(byte);
77 if safe_get(self.source, i) & 192 != TAG_CONT_U8 {
83 match (byte, safe_get(self.source, i)) {
84 (0xE0, 0xA0..=0xBF) => (),
85 (0xE1..=0xEC, 0x80..=0xBF) => (),
86 (0xED, 0x80..=0x9F) => (),
87 (0xEE..=0xEF, 0x80..=0xBF) => (),
91 if safe_get(self.source, i) & 192 != TAG_CONT_U8 {
97 match (byte, safe_get(self.source, i)) {
98 (0xF0, 0x90..=0xBF) => (),
99 (0xF1..=0xF3, 0x80..=0xBF) => (),
100 (0xF4, 0x80..=0x8F) => (),
104 if safe_get(self.source, i) & 192 != TAG_CONT_U8 {
108 if safe_get(self.source, i) & 192 != TAG_CONT_U8 {
120 // SAFETY: `i <= self.source.len()` because it is only ever incremented
121 // via `i += 1` and in between every single one of those increments, `i`
122 // is compared against `self.source.len()`. That happens either
123 // literally by `i < self.source.len()` in the while-loop's condition,
124 // or indirectly by `safe_get(self.source, i) & 192 != TAG_CONT_U8`. The
125 // loop is terminated as soon as the latest `i += 1` has made `i` no
126 // longer less than `self.source.len()`, which means it'll be at most
127 // equal to `self.source.len()`.
128 let (inspected, remaining) = unsafe { self.source.split_at_unchecked(i) };
129 self.source = remaining;
131 // SAFETY: `valid_up_to <= i` because it is only ever assigned via
132 // `valid_up_to = i` and `i` only increases.
133 let (valid, broken) = unsafe { inspected.split_at_unchecked(valid_up_to) };
135 Some(Utf8LossyChunk {
136 // SAFETY: All bytes up to `valid_up_to` are valid UTF-8.
137 valid: unsafe { from_utf8_unchecked(valid) },
143 impl fmt::Display for Utf8Lossy {
144 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
145 // If we're the empty string then our iterator won't actually yield
146 // anything, so perform the formatting manually
147 if self.bytes.is_empty() {
151 for Utf8LossyChunk { valid, broken } in self.chunks() {
152 // If we successfully decoded the whole chunk as a valid string then
153 // we can return a direct formatting of the string which will also
154 // respect various formatting flags if possible.
155 if valid.len() == self.bytes.len() {
156 assert!(broken.is_empty());
161 if !broken.is_empty() {
162 f.write_char(char::REPLACEMENT_CHARACTER)?;
169 impl fmt::Debug for Utf8Lossy {
170 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
173 for Utf8LossyChunk { valid, broken } in self.chunks() {
175 // Here we partially parse UTF-8 again which is suboptimal.
178 for (i, c) in valid.char_indices() {
179 let esc = c.escape_debug();
180 // If char needs escaping, flush backlog so far and write, else skip
182 f.write_str(&valid[from..i])?;
186 from = i + c.len_utf8();
189 f.write_str(&valid[from..])?;
192 // Broken parts of string as hex escape.
194 write!(f, "\\x{:02x}", b)?;