src/libcore/str/lossy.rs

   1 use crate::char;
   2 use crate::fmt::{self, Write};
   3 use crate::mem;
   4 use crate::str as core_str;
   5
   6 // ignore-tidy-undocumented-unsafe
   7
   8 /// Lossy UTF-8 string.
   9 #[unstable(feature = "str_internals", issue = "none")]
  10 pub struct Utf8Lossy {
  11     bytes: [u8],
  12 }
  13
  14 impl Utf8Lossy {
  15     pub fn from_str(s: &str) -> &Utf8Lossy {
  16         Utf8Lossy::from_bytes(s.as_bytes())
  17     }
  18
  19     pub fn from_bytes(bytes: &[u8]) -> &Utf8Lossy {
  20         unsafe { mem::transmute(bytes) }
  21     }
  22
  23     pub fn chunks(&self) -> Utf8LossyChunksIter<'_> {
  24         Utf8LossyChunksIter { source: &self.bytes }
  25     }
  26 }
  27
  28 /// Iterator over lossy UTF-8 string
  29 #[unstable(feature = "str_internals", issue = "none")]
  30 #[allow(missing_debug_implementations)]
  31 pub struct Utf8LossyChunksIter<'a> {
  32     source: &'a [u8],
  33 }
  34
  35 #[unstable(feature = "str_internals", issue = "none")]
  36 #[derive(PartialEq, Eq, Debug)]
  37 pub struct Utf8LossyChunk<'a> {
  38     /// Sequence of valid chars.
  39     /// Can be empty between broken UTF-8 chars.
  40     pub valid: &'a str,
  41     /// Single broken char, empty if none.
  42     /// Empty iff iterator item is last.
  43     pub broken: &'a [u8],
  44 }
  45
  46 impl<'a> Iterator for Utf8LossyChunksIter<'a> {
  47     type Item = Utf8LossyChunk<'a>;
  48
  49     fn next(&mut self) -> Option<Utf8LossyChunk<'a>> {
  50         if self.source.is_empty() {
  51             return None;
  52         }
  53
  54         const TAG_CONT_U8: u8 = 128;
  55         fn safe_get(xs: &[u8], i: usize) -> u8 {
  56             *xs.get(i).unwrap_or(&0)
  57         }
  58
  59         let mut i = 0;
  60         while i < self.source.len() {
  61             let i_ = i;
  62
  63             let byte = unsafe { *self.source.get_unchecked(i) };
  64             i += 1;
  65
  66             if byte < 128 {
  67             } else {
  68                 let w = core_str::utf8_char_width(byte);
  69
  70                 macro_rules! error {
  71                     () => {{
  72                         unsafe {
  73                             let r = Utf8LossyChunk {
  74                                 valid: core_str::from_utf8_unchecked(&self.source[0..i_]),
  75                                 broken: &self.source[i_..i],
  76                             };
  77                             self.source = &self.source[i..];
  78                             return Some(r);
  79                         }
  80                     }};
  81                 }
  82
  83                 match w {
  84                     2 => {
  85                         if safe_get(self.source, i) & 192 != TAG_CONT_U8 {
  86                             error!();
  87                         }
  88                         i += 1;
  89                     }
  90                     3 => {
  91                         match (byte, safe_get(self.source, i)) {
  92                             (0xE0, 0xA0..=0xBF) => (),
  93                             (0xE1..=0xEC, 0x80..=0xBF) => (),
  94                             (0xED, 0x80..=0x9F) => (),
  95                             (0xEE..=0xEF, 0x80..=0xBF) => (),
  96                             _ => {
  97                                 error!();
  98                             }
  99                         }
 100                         i += 1;
 101                         if safe_get(self.source, i) & 192 != TAG_CONT_U8 {
 102                             error!();
 103                         }
 104                         i += 1;
 105                     }
 106                     4 => {
 107                         match (byte, safe_get(self.source, i)) {
 108                             (0xF0, 0x90..=0xBF) => (),
 109                             (0xF1..=0xF3, 0x80..=0xBF) => (),
 110                             (0xF4, 0x80..=0x8F) => (),
 111                             _ => {
 112                                 error!();
 113                             }
 114                         }
 115                         i += 1;
 116                         if safe_get(self.source, i) & 192 != TAG_CONT_U8 {
 117                             error!();
 118                         }
 119                         i += 1;
 120                         if safe_get(self.source, i) & 192 != TAG_CONT_U8 {
 121                             error!();
 122                         }
 123                         i += 1;
 124                     }
 125                     _ => {
 126                         error!();
 127                     }
 128                 }
 129             }
 130         }
 131
 132         let r = Utf8LossyChunk {
 133             valid: unsafe { core_str::from_utf8_unchecked(self.source) },
 134             broken: &[],
 135         };
 136         self.source = &[];
 137         Some(r)
 138     }
 139 }
 140
 141 impl fmt::Display for Utf8Lossy {
 142     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
 143         // If we're the empty string then our iterator won't actually yield
 144         // anything, so perform the formatting manually
 145         if self.bytes.is_empty() {
 146             return "".fmt(f);
 147         }
 148
 149         for Utf8LossyChunk { valid, broken } in self.chunks() {
 150             // If we successfully decoded the whole chunk as a valid string then
 151             // we can return a direct formatting of the string which will also
 152             // respect various formatting flags if possible.
 153             if valid.len() == self.bytes.len() {
 154                 assert!(broken.is_empty());
 155                 return valid.fmt(f);
 156             }
 157
 158             f.write_str(valid)?;
 159             if !broken.is_empty() {
 160                 f.write_char(char::REPLACEMENT_CHARACTER)?;
 161             }
 162         }
 163         Ok(())
 164     }
 165 }
 166
 167 impl fmt::Debug for Utf8Lossy {
 168     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
 169         f.write_char('"')?;
 170
 171         for Utf8LossyChunk { valid, broken } in self.chunks() {
 172             // Valid part.
 173             // Here we partially parse UTF-8 again which is suboptimal.
 174             {
 175                 let mut from = 0;
 176                 for (i, c) in valid.char_indices() {
 177                     let esc = c.escape_debug();
 178                     // If char needs escaping, flush backlog so far and write, else skip
 179                     if esc.len() != 1 {
 180                         f.write_str(&valid[from..i])?;
 181                         for c in esc {
 182                             f.write_char(c)?;
 183                         }
 184                         from = i + c.len_utf8();
 185                     }
 186                 }
 187                 f.write_str(&valid[from..])?;
 188             }
 189
 190             // Broken parts of string as hex escape.
 191             for &b in broken {
 192                 write!(f, "\\x{:02x}", b)?;
 193             }
 194         }
 195
 196         f.write_char('"')
 197     }
 198 }