Rollup merge of #70493 - 0xd4d:rustdoc-tab-size, r=GuillaumeGomez

Fix rustdoc.css CSS tab-size property

This fixes the CSS tab size property names which are called `tab-size` / `-moz-tab-size` and not `tab-width`

Old issue https://github.com/rust-lang/rust/issues/49155 and related PR https://github.com/rust-lang/rust/pull/50947

tab-size: https://developer.mozilla.org/en-US/docs/Web/CSS/tab-size

diff --git a/src/libcore/unicode/mod.rs b/src/libcore/unicode/mod.rs
index b6eaf06aa7f63be316f86ef76f26ffa0ba935def..94a2507e26ccb728dae520e836603ff861bf3585 100644 (file)
--- a/src/libcore/unicode/mod.rs
+++ b/src/libcore/unicode/mod.rs
@@ -32,28 +32,3 @@ pub mod derived_property {
 pub use unicode_data::n::lookup as N;
 pub use unicode_data::uppercase::lookup as Uppercase;
 pub use unicode_data::white_space::lookup as White_Space;
-
-#[inline(always)]
-fn range_search<const N: usize, const N1: usize, const N2: usize>(
-    needle: u32,
-    chunk_idx_map: &[u8; N],
-    (last_chunk_idx, last_chunk_mapping): (u16, u8),
-    bitset_chunk_idx: &[[u8; 16]; N1],
-    bitset: &[u64; N2],
-) -> bool {
-    let bucket_idx = (needle / 64) as usize;
-    let chunk_map_idx = bucket_idx / 16;
-    let chunk_piece = bucket_idx % 16;
-    let chunk_idx = if chunk_map_idx >= N {
-        if chunk_map_idx == last_chunk_idx as usize {
-            last_chunk_mapping
-        } else {
-            return false;
-        }
-    } else {
-        chunk_idx_map[chunk_map_idx]
-    };
-    let idx = bitset_chunk_idx[(chunk_idx as usize)][chunk_piece];
-    let word = bitset[(idx as usize)];
-    (word & (1 << (needle % 64) as u64)) != 0
-}

diff --git a/src/libcore/unicode/unicode_data.rs b/src/libcore/unicode/unicode_data.rs
index 3e90028613ce1c3df30850b712c38b259a579e1d..48caa21fb0aa143f51118a0dae139091411d9852 100644 (file)
--- a/src/libcore/unicode/unicode_data.rs
+++ b/src/libcore/unicode/unicode_data.rs
@@ -1,618 +1,547 @@
 ///! This file is generated by src/tools/unicode-table-generator; do not edit manually!
-use super::range_search;
+
+#[inline(always)]
+fn bitset_search<
+    const N: usize,
+    const CHUNK_SIZE: usize,
+    const N1: usize,
+    const CANONICAL: usize,
+    const CANONICALIZED: usize,
+>(
+    needle: u32,
+    chunk_idx_map: &[u8; N],
+    bitset_chunk_idx: &[[u8; CHUNK_SIZE]; N1],
+    bitset_canonical: &[u64; CANONICAL],
+    bitset_canonicalized: &[(u8, u8); CANONICALIZED],
+) -> bool {
+    let bucket_idx = (needle / 64) as usize;
+    let chunk_map_idx = bucket_idx / CHUNK_SIZE;
+    let chunk_piece = bucket_idx % CHUNK_SIZE;
+    let chunk_idx = if let Some(&v) = chunk_idx_map.get(chunk_map_idx) {
+        v
+    } else {
+        return false;
+    };
+    let idx = bitset_chunk_idx[chunk_idx as usize][chunk_piece] as usize;
+    let word = if let Some(word) = bitset_canonical.get(idx) {
+        *word
+    } else {
+        let (real_idx, mapping) = bitset_canonicalized[idx - bitset_canonical.len()];
+        let mut word = bitset_canonical[real_idx as usize];
+        let should_invert = mapping & (1 << 6) != 0;
+        if should_invert {
+            word = !word;
+        }
+        // Lower 6 bits
+        let quantity = mapping & ((1 << 6) - 1);
+        if mapping & (1 << 7) != 0 {
+            // shift
+            word >>= quantity as u64;
+        } else {
+            word = word.rotate_left(quantity as u32);
+        }
+        word
+    };
+    (word & (1 << (needle % 64) as u64)) != 0
+}
+
+fn decode_prefix_sum(short_offset_run_header: u32) -> u32 {
+    short_offset_run_header & ((1 << 21) - 1)
+}
+
+fn decode_length(short_offset_run_header: u32) -> usize {
+    (short_offset_run_header >> 21) as usize
+}
+
+#[inline(always)]
+fn skip_search<const SOR: usize, const OFFSETS: usize>(
+    needle: u32,
+    short_offset_runs: &[u32; SOR],
+    offsets: &[u8; OFFSETS],
+) -> bool {
+    // Note that this *cannot* be past the end of the array, as the last
+    // element is greater than std::char::MAX (the largest possible needle).
+    //
+    // So, we cannot have found it (i.e. Ok(idx) + 1 != length) and the correct
+    // location cannot be past it, so Err(idx) != length either.
+    //
+    // This means that we can avoid bounds checking for the accesses below, too.
+    let last_idx =
+        match short_offset_runs.binary_search_by_key(&(needle << 11), |header| header << 11) {
+            Ok(idx) => idx + 1,
+            Err(idx) => idx,
+        };
+
+    let mut offset_idx = decode_length(short_offset_runs[last_idx]);
+    let length = if let Some(next) = short_offset_runs.get(last_idx + 1) {
+        decode_length(*next) - offset_idx
+    } else {
+        offsets.len() - offset_idx
+    };
+    let prev =
+        last_idx.checked_sub(1).map(|prev| decode_prefix_sum(short_offset_runs[prev])).unwrap_or(0);
+
+    let total = needle - prev;
+    let mut prefix_sum = 0;
+    for _ in 0..(length - 1) {
+        let offset = offsets[offset_idx];
+        prefix_sum += offset as u32;
+        if prefix_sum > total {
+            break;
+        }
+        offset_idx += 1;
+    }
+    offset_idx % 2 == 1
+}
 
 pub const UNICODE_VERSION: (u32, u32, u32) = (13, 0, 0);
 
 #[rustfmt::skip]
 pub mod alphabetic {
-    static BITSET_LAST_CHUNK_MAP: (u16, u8) = (196, 44);
-    static BITSET_CHUNKS_MAP: [u8; 196] = [
-        6, 32, 10, 18, 19, 23, 21, 12, 7, 5, 0, 20, 14, 50, 50, 50, 50, 50, 50, 37, 50, 50, 50, 50,
-        50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 49, 50, 30, 8, 50, 50, 50, 50,
-        50, 50, 50, 50, 50, 50, 46, 0, 0, 0, 0, 0, 0, 0, 0, 4, 36, 17, 31, 16, 25, 24, 26, 13, 15,
-        45, 27, 0, 0, 50, 11, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 39, 1, 50, 50, 50, 50, 50, 48,
-        50, 34, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 28, 0, 0, 0, 0, 0, 29, 0, 0, 9, 0, 33, 2, 3, 0, 0,
-        0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50,
-        50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 42, 50, 50, 50,
-        43, 22, 50, 50, 50, 50, 41, 50, 50, 50, 50, 50, 50, 47, 0, 0, 0, 38, 0, 50, 50, 50, 50,
+    static SHORT_OFFSET_RUNS: [u32; 52] = [
+        706, 33559113, 868226669, 947920662, 1157637302, 1306536960, 1310732293, 1398813696,
+        1449151936, 1451270141, 1455465613, 1459660301, 1468061604, 1648425216, 1658911342,
+        1661009214, 1707147904, 1793132343, 1853951616, 1994464256, 2330009312, 2418090906,
+        2428579840, 2439066671, 2441167872, 2443265607, 2445371392, 2447469113, 2449567296,
+        2476836856, 2508295382, 2512498688, 2518790431, 2520888060, 2533473280, 2535576576,
+        2556548774, 2634145792, 2682380992, 2715936768, 2720132608, 2736910640, 2875326464,
+        2887952094, 2890053429, 2894253730, 2902649825, 2906847232, 2908944926, 2911043584,
+        2913145675, 2916356939,
     ];
-    static BITSET_INDEX_CHUNKS: [[u8; 16]; 51] = [
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 0, 254, 247, 39, 68],
-        [0, 0, 0, 0, 0, 0, 0, 0, 111, 135, 113, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 195, 205, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 254, 254, 254, 254, 254, 210, 254, 25, 136, 251, 71, 243],
-        [0, 0, 182, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 107, 103, 180, 254, 254, 254, 254, 254, 254, 254, 61, 0, 155, 222, 181],
-        [0, 148, 30, 0, 172, 226, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [48, 80, 254, 169, 206, 123, 189, 139, 95, 179, 145, 86, 211, 204, 254, 56],
-        [53, 0, 0, 0, 129, 17, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0],
-        [59, 54, 185, 203, 171, 191, 161, 117, 158, 87, 164, 118, 162, 67, 159, 23],
-        [62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [95, 131, 168, 105, 254, 254, 254, 82, 254, 254, 254, 254, 236, 130, 137, 120],
-        [101, 0, 225, 146, 151, 2, 217, 45, 144, 246, 32, 101, 0, 0, 0, 0],
-        [119, 253, 224, 175, 193, 254, 227, 195, 0, 0, 0, 0, 0, 0, 0, 0],
-        [143, 190, 91, 0, 153, 218, 24, 0, 0, 0, 0, 92, 0, 0, 66, 0],
-        [150, 94, 37, 85, 102, 0, 157, 0, 88, 122, 31, 46, 89, 74, 20, 0],
-        [154, 34, 254, 110, 0, 84, 0, 0, 0, 0, 233, 19, 216, 108, 237, 21],
-        [166, 42, 165, 72, 167, 177, 126, 76, 109, 16, 127, 38, 1, 192, 124, 0],
-        [176, 246, 234, 174, 254, 254, 254, 254, 254, 235, 140, 241, 240, 26, 228, 128],
-        [213, 239, 254, 77, 209, 64, 142, 238, 63, 0, 0, 0, 0, 0, 0, 0],
-        [225, 101, 207, 89, 98, 81, 208, 10, 232, 83, 147, 1, 188, 13, 178, 70],
-        [237, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254],
-        [253, 254, 254, 254, 254, 254, 254, 254, 254, 214, 231, 99, 79, 78, 183, 27],
-        [254, 6, 100, 50, 75, 90, 254, 28, 134, 0, 202, 51, 163, 43, 0, 0],
-        [254, 9, 75, 75, 49, 0, 0, 0, 0, 0, 69, 0, 199, 6, 195, 93],
-        [254, 41, 254, 8, 0, 0, 141, 33, 145, 4, 97, 0, 55, 0, 0, 0],
-        [254, 62, 254, 254, 254, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [254, 121, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [254, 242, 170, 252, 138, 245, 254, 254, 254, 254, 220, 173, 186, 212, 219, 14],
-        [254, 254, 15, 132, 254, 254, 254, 254, 57, 149, 254, 65, 223, 254, 249, 187],
-        [254, 254, 196, 114, 201, 44, 0, 0, 254, 254, 254, 254, 95, 47, 0, 0],
-        [254, 254, 250, 254, 194, 229, 156, 73, 230, 215, 254, 152, 246, 248, 71, 104],
-        [254, 254, 254, 5, 254, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [254, 254, 254, 22, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [254, 254, 254, 254, 37, 200, 254, 254, 254, 254, 254, 116, 0, 0, 0, 0],
-        [254, 254, 254, 254, 133, 246, 244, 112, 0, 184, 254, 125, 106, 221, 145, 29],
-        [254, 254, 254, 254, 254, 254, 254, 0, 254, 254, 254, 254, 254, 254, 254, 254],
-        [254, 254, 254, 254, 254, 254, 254, 254, 35, 0, 0, 0, 0, 0, 0, 0],
-        [254, 254, 254, 254, 254, 254, 254, 254, 101, 37, 0, 60, 65, 160, 18, 0],
-        [254, 254, 254, 254, 254, 254, 254, 254, 254, 7, 0, 0, 0, 0, 0, 0],
-        [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 197, 254, 254, 254, 254, 254],
-        [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 35, 254, 254, 254, 254],
-        [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 84, 254, 254, 254],
-        [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 11, 0, 0],
-        [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 25, 0],
-        [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 198, 115],
-        [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 40],
-        [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 96],
-        [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 125],
-        [254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254, 254],
+    static OFFSETS: [u8; 1391] = [
+        65, 26, 6, 26, 47, 1, 10, 1, 4, 1, 5, 23, 1, 31, 1, 0, 4, 12, 14, 5, 7, 1, 1, 1, 86, 1, 42,
+        5, 1, 2, 2, 4, 1, 1, 6, 1, 1, 3, 1, 1, 1, 20, 1, 83, 1, 139, 8, 166, 1, 38, 2, 1, 6, 41, 39,
+        14, 1, 1, 1, 2, 1, 2, 1, 1, 8, 27, 4, 4, 29, 11, 5, 56, 1, 7, 14, 102, 1, 8, 4, 8, 4, 3, 10,
+        3, 2, 1, 16, 48, 13, 101, 24, 33, 9, 2, 4, 1, 5, 24, 2, 19, 19, 25, 7, 11, 53, 21, 1, 18,
+        12, 12, 3, 7, 6, 76, 1, 16, 1, 3, 4, 15, 13, 19, 1, 8, 2, 2, 2, 22, 1, 7, 1, 1, 3, 4, 3, 8,
+        2, 2, 2, 2, 1, 1, 8, 1, 4, 2, 1, 5, 12, 2, 10, 1, 4, 3, 1, 6, 4, 2, 2, 22, 1, 7, 1, 2, 1, 2,
+        1, 2, 4, 5, 4, 2, 2, 2, 4, 1, 7, 4, 1, 1, 17, 6, 11, 3, 1, 9, 1, 3, 1, 22, 1, 7, 1, 2, 1, 5,
+        3, 9, 1, 3, 1, 2, 3, 1, 15, 4, 21, 4, 4, 3, 1, 8, 2, 2, 2, 22, 1, 7, 1, 2, 1, 5, 3, 8, 2, 2,
+        2, 2, 9, 2, 4, 2, 1, 5, 13, 1, 16, 2, 1, 6, 3, 3, 1, 4, 3, 2, 1, 1, 1, 2, 3, 2, 3, 3, 3, 12,
+        4, 5, 3, 3, 1, 3, 3, 1, 6, 1, 40, 4, 1, 8, 1, 3, 1, 23, 1, 16, 3, 8, 1, 3, 1, 3, 8, 2, 1, 3,
+        5, 4, 28, 4, 1, 8, 1, 3, 1, 23, 1, 10, 1, 5, 3, 8, 1, 3, 1, 3, 8, 2, 7, 1, 1, 4, 13, 2, 13,
+        13, 1, 3, 1, 41, 2, 8, 1, 3, 1, 3, 1, 1, 5, 4, 7, 5, 22, 6, 1, 3, 1, 18, 3, 24, 1, 9, 1, 1,
+        2, 7, 8, 6, 1, 1, 1, 8, 18, 2, 13, 58, 5, 7, 6, 1, 51, 2, 1, 1, 1, 5, 1, 24, 1, 1, 1, 19, 1,
+        3, 2, 5, 1, 1, 6, 1, 14, 4, 32, 1, 63, 8, 1, 36, 4, 17, 6, 16, 1, 36, 67, 55, 1, 1, 2, 5,
+        16, 64, 10, 4, 2, 38, 1, 1, 5, 1, 2, 43, 1, 0, 1, 4, 2, 7, 1, 1, 1, 4, 2, 41, 1, 4, 2, 33,
+        1, 4, 2, 7, 1, 1, 1, 4, 2, 15, 1, 57, 1, 4, 2, 67, 37, 16, 16, 86, 2, 6, 3, 0, 2, 17, 1, 26,
+        5, 75, 3, 11, 7, 13, 1, 6, 12, 20, 12, 20, 12, 13, 1, 3, 1, 2, 12, 52, 2, 19, 14, 1, 4, 1,
+        67, 89, 7, 43, 5, 70, 10, 31, 1, 12, 4, 9, 23, 30, 2, 5, 11, 44, 4, 26, 54, 28, 4, 63, 2,
+        20, 50, 1, 23, 2, 63, 52, 1, 15, 1, 7, 52, 42, 2, 4, 10, 44, 1, 11, 14, 55, 22, 3, 10, 36,
+        2, 9, 7, 43, 2, 3, 41, 4, 1, 6, 1, 2, 3, 1, 5, 192, 39, 14, 11, 0, 2, 6, 2, 38, 2, 6, 2, 8,
+        1, 1, 1, 1, 1, 1, 1, 31, 2, 53, 1, 7, 1, 1, 3, 3, 1, 7, 3, 4, 2, 6, 4, 13, 5, 3, 1, 7, 116,
+        1, 13, 1, 16, 13, 101, 1, 4, 1, 2, 10, 1, 1, 3, 5, 6, 1, 1, 1, 1, 1, 1, 4, 1, 11, 2, 4, 5,
+        5, 4, 1, 17, 41, 0, 52, 0, 47, 1, 47, 1, 133, 6, 4, 3, 2, 12, 38, 1, 1, 5, 1, 2, 56, 7, 1,
+        16, 23, 9, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 32, 47, 1, 0, 3, 25, 9, 7, 5, 2,
+        5, 4, 86, 6, 3, 1, 90, 1, 4, 5, 43, 1, 94, 17, 32, 48, 16, 0, 0, 64, 0, 3, 0, 67, 46, 2, 0,
+        3, 16, 10, 2, 20, 47, 5, 8, 3, 113, 39, 9, 2, 103, 2, 53, 2, 9, 42, 17, 1, 33, 24, 52, 12,
+        68, 1, 1, 44, 6, 3, 1, 1, 3, 10, 33, 5, 35, 13, 29, 3, 51, 1, 12, 15, 1, 16, 16, 10, 5, 1,
+        55, 9, 14, 18, 23, 3, 69, 1, 1, 1, 1, 24, 3, 2, 16, 2, 4, 11, 6, 2, 6, 2, 6, 9, 7, 1, 7, 1,
+        43, 1, 14, 6, 123, 21, 0, 12, 23, 4, 49, 0, 0, 2, 106, 38, 7, 12, 5, 5, 12, 1, 13, 1, 5, 1,
+        1, 1, 2, 1, 2, 1, 108, 33, 0, 18, 64, 2, 54, 40, 12, 116, 5, 1, 135, 36, 26, 6, 26, 11, 89,
+        3, 6, 2, 6, 2, 6, 2, 3, 35, 12, 1, 26, 1, 19, 1, 2, 1, 15, 2, 14, 34, 123, 69, 53, 0, 29, 3,
+        49, 47, 32, 13, 30, 5, 43, 5, 30, 2, 36, 4, 8, 1, 5, 42, 158, 18, 36, 4, 36, 4, 40, 8, 52,
+        156, 0, 9, 22, 10, 8, 152, 6, 2, 1, 1, 44, 1, 2, 3, 1, 2, 23, 10, 23, 9, 31, 65, 19, 1, 2,
+        10, 22, 10, 26, 70, 56, 6, 2, 64, 4, 1, 2, 5, 8, 1, 3, 1, 29, 42, 29, 3, 29, 35, 8, 1, 28,
+        27, 54, 10, 22, 10, 19, 13, 18, 110, 73, 55, 51, 13, 51, 13, 40, 0, 42, 1, 2, 3, 2, 78, 29,
+        10, 1, 8, 22, 106, 21, 27, 23, 9, 70, 60, 55, 23, 25, 23, 51, 17, 4, 8, 35, 3, 1, 9, 64, 1,
+        4, 9, 2, 10, 1, 1, 1, 35, 18, 1, 34, 2, 1, 6, 1, 65, 7, 1, 1, 1, 4, 1, 15, 1, 10, 7, 57, 23,
+        4, 1, 8, 2, 2, 2, 22, 1, 7, 1, 2, 1, 5, 3, 8, 2, 2, 2, 2, 3, 1, 6, 1, 5, 7, 156, 66, 1, 3,
+        1, 4, 20, 3, 30, 66, 2, 2, 1, 1, 184, 54, 2, 7, 25, 6, 34, 63, 1, 1, 3, 1, 59, 54, 2, 1, 71,
+        27, 2, 14, 213, 57, 103, 64, 31, 8, 2, 1, 2, 8, 1, 2, 1, 30, 1, 2, 2, 2, 2, 4, 93, 8, 2, 46,
+        2, 6, 1, 1, 1, 2, 27, 51, 2, 10, 17, 72, 5, 1, 34, 57, 0, 9, 1, 45, 1, 7, 1, 1, 49, 30, 2,
+        22, 1, 14, 73, 7, 1, 2, 1, 44, 3, 1, 1, 2, 1, 3, 1, 1, 2, 2, 24, 6, 1, 2, 1, 37, 1, 2, 1, 4,
+        1, 1, 0, 23, 185, 1, 79, 0, 102, 111, 17, 196, 0, 0, 0, 0, 0, 0, 7, 31, 113, 30, 18, 48, 16,
+        4, 31, 21, 5, 19, 0, 64, 128, 75, 4, 57, 7, 17, 64, 2, 1, 1, 12, 2, 14, 0, 8, 0, 42, 9, 0,
+        0, 49, 3, 17, 4, 8, 0, 0, 107, 5, 13, 3, 9, 7, 10, 4, 1, 0, 85, 1, 71, 1, 2, 2, 1, 2, 2, 2,
+        4, 1, 12, 1, 1, 1, 7, 1, 65, 1, 4, 2, 8, 1, 7, 1, 28, 1, 4, 1, 5, 1, 1, 3, 7, 1, 0, 2, 25,
+        1, 25, 1, 31, 1, 25, 1, 31, 1, 25, 1, 31, 1, 25, 1, 31, 1, 25, 1, 8, 0, 7, 1, 17, 2, 7, 1,
+        2, 1, 5, 213, 45, 10, 7, 16, 1, 0, 44, 0, 197, 59, 68, 3, 1, 3, 1, 0, 4, 1, 27, 1, 2, 1, 1,
+        2, 1, 1, 10, 1, 4, 1, 1, 1, 1, 6, 1, 4, 1, 1, 1, 1, 1, 1, 3, 1, 2, 1, 1, 2, 1, 1, 1, 1, 1,
+        1, 1, 1, 1, 1, 2, 1, 1, 2, 4, 1, 7, 1, 4, 1, 4, 1, 1, 1, 10, 1, 17, 5, 3, 1, 5, 1, 17, 0,
+        26, 6, 26, 6, 26, 0, 0, 34, 0, 11, 222, 2, 0, 14, 0, 0, 0, 0, 0, 0,
     ];
-    static BITSET: [u64; 255] = [
-        0, 1, 7, 15, 17, 31, 63, 127, 179, 511, 1023, 2047, 2191, 4079, 4087, 8191, 8319, 16384,
-        65535, 131071, 262143, 4128527, 4194303, 8461767, 24870911, 67108863, 134217727, 276824575,
-        335593502, 486341884, 536805376, 536870911, 553648127, 1056964608, 1073692671, 1073741823,
-        1140785663, 2147483647, 4026540127, 4294934783, 8589934591, 15032387515, 64548249055,
-        68191066527, 68719476735, 115913785343, 137438953215, 1095220854783, 1099511627711,
-        1099511627775, 2199023190016, 2199023255551, 4398046511103, 8641373536127, 8791831609343,
-        8795690369023, 8796093022207, 13198434443263, 17592186044415, 35184321757183,
-        70368744112128, 88094074470339, 140737488355327, 140737488355328, 141836999983103,
-        281474976710655, 281474976710656, 563017343310239, 844472174772224, 875211255709695,
-        1125625028935679, 1125899906842623, 1688915364814303, 2119858418286774, 2251795522912255,
-        2251799813685247, 3377704004976767, 3509778554814463, 3905461007941631, 4503595333443583,
-        4503599627370495, 8796093022142464, 9006649498927104, 9007192812290047, 9007199254740991,
-        15762594400829440, 17169970223906821, 17732925109967239, 18014398491652207,
-        18014398509481983, 20266198323101936, 36027697507139583, 36028792723996672,
-        36028792723996703, 36028792728190975, 36028797018963967, 72057594037927935,
-        90071992547409919, 143851303137705983, 144053615424700415, 144115188075855868,
-        144115188075855871, 288230371860938751, 297241973452963840, 301749971126844416,
-        319718190147960832, 576460743713488896, 576460743847706622, 576460752303359999,
-        576460752303423486, 576460752303423487, 790380184120328175, 1152640029630136575,
-        1152917029519358975, 1152921504591118335, 1152921504606845055, 1152921504606846975,
-        1153765996922689951, 2161727885562420159, 2251241253188403424, 2295745090394464220,
-        2305570330330005503, 2305843004918726656, 2305843004919250943, 2305843009196916483,
-        2305843009213693951, 3457638613854978030, 4323455298678290390, 4557642822898941951,
-        4575692405780512767, 4611686017001275199, 4611686018360336384, 4611686018427322368,
-        4611686018427387903, 4656722014700830719, 6843210385291930244, 6881498031078244479,
-        6908521828386340863, 8935141660164089791, 8935423131384840192, 9168765891372858879,
-        9169328841326329855, 9187201948305063935, 9187343239835811327, 9216616637413720063,
-        9223372036854775807, 9223372041149743103, 9223372586610589696, 9223934986808197120,
-        10371930679322607615, 10502394331027995967, 11078855083321979519, 11241233151490523135,
-        13006395723845991295, 13258596753222922239, 13609596598936928288, 13834776580305453567,
-        13907115649320091647, 14082190885810440174, 14123225865944680428, 16212958624174047247,
-        16412803692974677999, 16424062692043104238, 16424062692043104239, 16424062692043243502,
-        16424625641996804079, 16429129241624174575, 16717361816799141887, 16717361816799216127,
-        16788293510930366511, 17005555242810474495, 17293822569102704639, 17581979622616071300,
-        17870283321271910397, 17870283321406070975, 17870283321406128127, 17978369712463020031,
-        18158513764145585631, 18158781978395017215, 18194542490281852927, 18410715276682199039,
-        18428729675200069631, 18428729675200069632, 18433233274827440127, 18437455399478099968,
-        18437736870159843328, 18437736874452713471, 18437736874454812668, 18442240474082181119,
-        18444492273895866367, 18445618173802708993, 18446181192473632767, 18446216308128218879,
-        18446462598732840928, 18446462598732840959, 18446462598732840960, 18446462599806582783,
-        18446462615912710143, 18446462667452317695, 18446463149025525759, 18446463629525450752,
-        18446463698244468735, 18446464796682337663, 18446466966713532671, 18446466996645134335,
-        18446466996779287551, 18446471394825862144, 18446471394825863167, 18446480190918885375,
-        18446498607738650623, 18446532967477018623, 18446602782178705022, 18446603336221163519,
-        18446603336221196287, 18446638520593285119, 18446673709243564031, 18446708893632430079,
-        18446740770879700992, 18446741595513422027, 18446741874686295551, 18446743249075830783,
-        18446743798965862398, 18446744056529672000, 18446744060816261120, 18446744068886102015,
-        18446744069414584320, 18446744069414601696, 18446744069414617087, 18446744069414649855,
-        18446744069456527359, 18446744069548736512, 18446744069548802046, 18446744069683019775,
-        18446744069951455231, 18446744070421282815, 18446744070446333439, 18446744070475743231,
-        18446744070488326143, 18446744071553646463, 18446744071562067967, 18446744073696837631,
-        18446744073701162813, 18446744073707454463, 18446744073709027328, 18446744073709355007,
-        18446744073709419615, 18446744073709486080, 18446744073709520895, 18446744073709543424,
-        18446744073709550079, 18446744073709550595, 18446744073709551579, 18446744073709551599,
-        18446744073709551614, 18446744073709551615,
-    ];
-
     pub fn lookup(c: char) -> bool {
-        super::range_search(
+        super::skip_search(
             c as u32,
-            &BITSET_CHUNKS_MAP,
-            BITSET_LAST_CHUNK_MAP,
-            &BITSET_INDEX_CHUNKS,
-            &BITSET,
+            &SHORT_OFFSET_RUNS,
+            &OFFSETS,
         )
     }
 }
 
 #[rustfmt::skip]
 pub mod case_ignorable {
-    static BITSET_LAST_CHUNK_MAP: (u16, u8) = (896, 33);
-    static BITSET_CHUNKS_MAP: [u8; 125] = [
-        25, 14, 21, 30, 28, 4, 17, 23, 22, 0, 0, 16, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 13, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 0, 0, 0, 3, 6, 9, 0, 7, 11, 32, 31, 26, 29, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0,
-        10, 0, 8, 0, 19, 0, 12, 0, 1,
-    ];
-    static BITSET_INDEX_CHUNKS: [[u8; 16]; 34] = [
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 47, 57],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 173, 3],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 90, 136, 38],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 104, 7, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 78, 27, 0, 148, 138, 81, 44, 119],
-        [0, 0, 0, 0, 0, 0, 0, 0, 154, 0, 0, 58, 0, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 167, 99, 77, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 48, 0, 116, 0, 0],
-        [0, 0, 0, 0, 0, 172, 70, 0, 0, 8, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 67, 0, 0, 24, 0, 0],
-        [0, 0, 0, 29, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 135, 0, 0, 0, 0, 16, 162, 46, 86, 51, 80, 13, 111],
-        [0, 0, 12, 0, 0, 43, 163, 92, 35, 82, 0, 71, 175, 14, 83, 131],
-        [0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 133, 0, 87, 0, 150, 0, 178, 75, 0, 0, 0, 0, 0, 0, 0],
-        [20, 5, 61, 0, 120, 0, 0, 0, 32, 156, 176, 1, 126, 91, 69, 88],
-        [26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [62, 0, 0, 0, 137, 0, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0],
-        [66, 0, 0, 152, 72, 25, 134, 59, 102, 124, 165, 101, 0, 64, 0, 68],
-        [73, 33, 0, 181, 125, 85, 122, 139, 123, 100, 123, 169, 155, 54, 4, 18],
-        [74, 151, 36, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [106, 135, 0, 112, 177, 107, 180, 168, 0, 0, 0, 0, 0, 0, 157, 142],
-        [109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [113, 50, 108, 0, 0, 0, 0, 0, 0, 0, 174, 182, 182, 114, 10, 0],
-        [115, 0, 0, 0, 141, 5, 0, 49, 145, 34, 31, 0, 0, 0, 0, 0],
-        [118, 0, 42, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [143, 95, 37, 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0],
-        [161, 0, 103, 0, 160, 11, 30, 0, 0, 0, 0, 93, 0, 0, 0, 0],
-        [164, 55, 155, 53, 127, 52, 2, 28, 117, 21, 128, 19, 110, 147, 129, 9],
-        [170, 41, 153, 6, 0, 0, 159, 39, 158, 1, 105, 0, 65, 0, 0, 0],
-        [171, 149, 132, 17, 98, 89, 146, 23, 140, 0, 0, 63, 127, 97, 0, 0],
-        [179, 182, 0, 0, 182, 182, 182, 79, 0, 0, 0, 0, 0, 0, 0, 0],
+    static SHORT_OFFSET_RUNS: [u32; 32] = [
+        688, 44045149, 555751186, 559947709, 794831996, 866136069, 891330581, 916497656, 920692236,
+        924908318, 1122041344, 1130430973, 1193347585, 1205931300, 1231097515, 1235294255,
+        1445009723, 1453399088, 1512120051, 1575040048, 1579248368, 1583443791, 1596046493,
+        1612829031, 1621219840, 1642192896, 1667359024, 1688330988, 1692526800, 1696723963,
+        1705902081, 1711210992,
     ];
-    static BITSET: [u64; 183] = [
-        0, 1, 2, 3, 4, 8, 13, 15, 28, 64, 176, 191, 1016, 1792, 2047, 4080, 4096, 8192, 8193,
-        16192, 30720, 32704, 32768, 40448, 131008, 262016, 2097152, 2359296, 6030336, 8323072,
-        10682368, 58719232, 159383552, 234881024, 243138688, 402587711, 536805376, 536879204,
-        546307648, 805306369, 1073741824, 1073741916, 2113929216, 2181038080, 3221225472,
-        3758096384, 4026531840, 4294934528, 4294967296, 4512022528, 5368709120, 17179869183,
-        51539615774, 51539619904, 51545907230, 51545914817, 66035122176, 115964116992, 412316860416,
-        412316893184, 1030792151040, 2199023255648, 8641373536127, 8763880767488, 15397323538432,
-        17303886364672, 18004502906948, 26388279066624, 36421322670080, 65128884076547,
-        65970697670631, 68168642985984, 70093866270720, 70368739983360, 136957967529984,
-        140737488355328, 263882790666240, 281470547525648, 281470682333183, 281474976710655,
-        281474976710656, 281474976710657, 281479271675905, 562675075514368, 562949953355776,
-        563001509683710, 844424930131968, 985162418487296, 1023920203366400, 2251799813685248,
-        3377699721314304, 4494803534348292, 4503599627370678, 6755399441055744, 7881299349733376,
-        8444256867844096, 8725724278030336, 8760633772212225, 8989057312882695, 9042383626829823,
-        9851624185018758, 24822575045541890, 28848986089586688, 30958948903026688,
-        35747322042253312, 53805701016846336, 58529202969772032, 72066390130950143,
-        112767012056334336, 143833713099145216, 189151184399892480, 216172782113783808,
-        220713756545974272, 288301294651703296, 302022650010533887, 504262420777140224,
-        558446353793941504, 572520102629474304, 593978171557150752, 1008806350890729472,
-        1009933895770046464, 1152921504606846976, 1152921504606846978, 1152921504606846982,
-        1153202979583561736, 1441151880758558727, 1715871458028158991, 1729382256910270467,
-        2301902359539744768, 2305843009196908767, 2305843009213693952, 2612078987781865472,
-        2771965570646540291, 3458764513820540928, 3731232291276455943, 4539628424389459968,
-        4589168020290535424, 4611404543450677248, 4611686018494513280, 4611686069967003678,
-        4671217976001691648, 6341068275337658368, 6917775322003857411, 7421334051581067264,
-        8070450532247928832, 8788774672813524990, 9205357638345293827, 9222809086901354496,
-        9223372036854775808, 9223372036854775935, 9223512774343131136, 9224216320050987008,
-        9224497932466651184, 9653465801268658176, 9727775195120332910, 10376293541461622786,
-        11526998316797657088, 11529215046068469760, 12103423998558208000, 12699025049277956096,
-        13005832773892571136, 13798747783286489088, 13832665517980123136, 13835058055282032640,
-        13835058055282163729, 13951307220663664640, 17870283321406128128, 17906312118425092095,
-        18158513697557839871, 18158513749097456062, 18374686479671623680, 18374686479671623682,
-        18444496122186563584, 18445618173802708992, 18446462598732840960, 18446462598733004800,
-        18446463148488654848, 18446726481523507200, 18446744069414584320, 18446744069414584322,
-        18446744073575333888, 18446744073709027328, 18446744073709551615,
+    static OFFSETS: [u8; 821] = [
+        39, 1, 6, 1, 11, 1, 35, 1, 1, 1, 71, 1, 4, 1, 1, 1, 4, 1, 2, 2, 0, 192, 4, 2, 4, 1, 9, 2,
+        1, 1, 251, 7, 207, 1, 5, 1, 49, 45, 1, 1, 1, 2, 1, 2, 1, 1, 44, 1, 11, 6, 10, 11, 1, 1, 35,
+        1, 10, 21, 16, 1, 101, 8, 1, 10, 1, 4, 33, 1, 1, 1, 30, 27, 91, 11, 58, 11, 4, 1, 2, 1, 24,
+        24, 43, 3, 119, 48, 55, 1, 1, 1, 4, 8, 4, 1, 3, 7, 10, 2, 13, 1, 15, 1, 58, 1, 4, 4, 8, 1,
+        20, 2, 26, 1, 2, 2, 57, 1, 4, 2, 4, 2, 2, 3, 3, 1, 30, 2, 3, 1, 11, 2, 57, 1, 4, 5, 1, 2, 4,
+        1, 20, 2, 22, 6, 1, 1, 58, 1, 2, 1, 1, 4, 8, 1, 7, 2, 11, 2, 30, 1, 61, 1, 12, 1, 50, 1, 3,
+        1, 57, 3, 5, 3, 1, 4, 7, 2, 11, 2, 29, 1, 58, 1, 2, 1, 6, 1, 5, 2, 20, 2, 28, 2, 57, 2, 4,
+        4, 8, 1, 20, 2, 29, 1, 72, 1, 7, 3, 1, 1, 90, 1, 2, 7, 11, 9, 98, 1, 2, 9, 9, 1, 1, 6, 74,
+        2, 27, 1, 1, 1, 1, 1, 55, 14, 1, 5, 1, 2, 5, 11, 1, 36, 9, 1, 102, 4, 1, 6, 1, 2, 2, 2, 25,
+        2, 4, 3, 16, 4, 13, 1, 2, 2, 6, 1, 15, 1, 94, 1, 0, 3, 0, 3, 29, 3, 29, 2, 30, 2, 64, 2, 1,
+        7, 8, 1, 2, 11, 3, 1, 5, 1, 45, 4, 52, 1, 65, 2, 34, 1, 118, 3, 4, 2, 9, 1, 6, 3, 219, 2, 2,
+        1, 58, 1, 1, 7, 1, 1, 1, 1, 2, 8, 6, 10, 2, 1, 39, 1, 8, 17, 63, 4, 48, 1, 1, 5, 1, 1, 5, 1,
+        40, 9, 12, 2, 32, 4, 2, 2, 1, 3, 56, 1, 1, 2, 3, 1, 1, 3, 58, 8, 2, 2, 64, 6, 82, 3, 1, 13,
+        1, 7, 4, 1, 6, 1, 3, 2, 50, 63, 13, 1, 34, 95, 1, 5, 0, 1, 1, 3, 11, 3, 13, 3, 13, 3, 13, 2,
+        12, 5, 8, 2, 10, 1, 2, 1, 2, 5, 49, 5, 1, 10, 1, 1, 13, 1, 16, 13, 51, 33, 0, 2, 113, 3,
+        125, 1, 15, 1, 96, 32, 47, 1, 0, 1, 36, 4, 3, 5, 5, 1, 93, 6, 93, 3, 0, 1, 0, 6, 0, 1, 98,
+        4, 1, 10, 1, 1, 28, 4, 80, 2, 14, 34, 78, 1, 23, 3, 109, 2, 8, 1, 3, 1, 4, 1, 25, 2, 5, 1,
+        151, 2, 26, 18, 13, 1, 38, 8, 25, 11, 46, 3, 48, 1, 2, 4, 2, 2, 17, 1, 21, 2, 66, 6, 2, 2,
+        2, 2, 12, 1, 8, 1, 35, 1, 11, 1, 51, 1, 1, 3, 2, 2, 5, 2, 1, 1, 27, 1, 14, 2, 5, 2, 1, 1,
+        100, 5, 9, 3, 121, 1, 2, 1, 4, 1, 0, 1, 147, 16, 0, 16, 3, 1, 12, 16, 34, 1, 2, 1, 169, 1,
+        7, 1, 6, 1, 11, 1, 35, 1, 1, 1, 47, 1, 45, 2, 67, 1, 21, 3, 0, 1, 226, 1, 149, 5, 0, 3, 1,
+        2, 5, 4, 40, 3, 4, 1, 165, 2, 0, 4, 0, 2, 153, 11, 176, 1, 54, 15, 56, 3, 49, 4, 2, 2, 2, 1,
+        15, 1, 50, 3, 36, 5, 1, 8, 62, 1, 12, 2, 52, 9, 10, 4, 2, 1, 95, 3, 2, 1, 1, 2, 6, 1, 160,
+        1, 3, 8, 21, 2, 57, 2, 3, 1, 37, 7, 3, 5, 195, 8, 2, 3, 1, 1, 23, 1, 84, 6, 1, 1, 4, 2, 1,
+        2, 238, 4, 6, 2, 1, 2, 27, 2, 85, 8, 2, 1, 1, 2, 106, 1, 1, 1, 2, 6, 1, 1, 101, 3, 2, 4, 1,
+        5, 0, 9, 1, 2, 0, 2, 1, 1, 4, 1, 144, 4, 2, 2, 4, 1, 32, 10, 40, 6, 2, 4, 8, 1, 9, 6, 2, 3,
+        46, 13, 1, 2, 0, 7, 1, 6, 1, 1, 82, 22, 2, 7, 1, 2, 1, 2, 122, 6, 3, 1, 1, 2, 1, 7, 1, 1,
+        72, 2, 3, 1, 1, 1, 0, 2, 0, 9, 0, 5, 59, 7, 9, 4, 0, 1, 63, 17, 64, 2, 1, 2, 0, 2, 1, 4, 0,
+        3, 9, 16, 2, 7, 30, 4, 148, 3, 0, 55, 4, 50, 8, 1, 14, 1, 22, 5, 1, 15, 0, 7, 1, 17, 2, 7,
+        1, 2, 1, 5, 0, 14, 0, 4, 0, 7, 109, 8, 0, 5, 0, 1, 30, 96, 128, 240, 0,
     ];
-
     pub fn lookup(c: char) -> bool {
-        super::range_search(
+        super::skip_search(
             c as u32,
-            &BITSET_CHUNKS_MAP,
-            BITSET_LAST_CHUNK_MAP,
-            &BITSET_INDEX_CHUNKS,
-            &BITSET,
+            &SHORT_OFFSET_RUNS,
+            &OFFSETS,
         )
     }
 }
 
 #[rustfmt::skip]
 pub mod cased {
-    static BITSET_LAST_CHUNK_MAP: (u16, u8) = (124, 6);
-    static BITSET_CHUNKS_MAP: [u8; 123] = [
-        13, 18, 0, 0, 12, 0, 0, 9, 14, 10, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 0, 1, 2, 0, 16, 0, 8, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0,
-        0, 0, 0, 7,
+    static SHORT_OFFSET_RUNS: [u32; 19] = [
+        4256, 115348384, 136322176, 144711446, 163587254, 320875520, 325101120, 358656816,
+        392231680, 404815649, 413205504, 421596288, 434182304, 442592832, 446813184, 451008166,
+        528607488, 576844080, 582152586,
     ];
-    static BITSET_INDEX_CHUNKS: [[u8; 16]; 19] = [
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 43, 62, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 10, 0, 50, 62, 58, 20],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 42, 44, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 62, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 16, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 31, 0, 62, 62, 62, 0, 62, 62, 62, 62, 54, 26, 27, 24],
-        [0, 0, 39, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 51, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 51, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 25],
-        [0, 22, 19, 37, 62, 62, 36, 61, 62, 62, 18, 12, 0, 30, 49, 38],
-        [0, 29, 9, 0, 34, 52, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [46, 55, 62, 17, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [62, 6, 42, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [62, 56, 33, 60, 28, 57, 62, 62, 62, 62, 48, 35, 40, 45, 47, 5],
-        [62, 62, 59, 62, 41, 53, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-    ];
-    static BITSET: [u64; 63] = [
-        0, 15, 24, 511, 1023, 4087, 65535, 16253055, 134217726, 536805376, 1073741823, 4294967295,
-        133143986179, 4398046511103, 36009005809663, 70368744177663, 2251799813685247,
-        3509778554814463, 144115188074807295, 297241973452963840, 531424756029720572,
-        576460743713488896, 576460743847706622, 1152921504591118335, 2295745090394464220,
-        4557642822898941951, 4611686017001275199, 6908521828386340863, 8935141660164089791,
-        9223934986808197120, 13605092999309557792, 16717361816799216127, 16717361816799223999,
-        17005555242810474495, 17446871633794956420, 17870283321271910397, 17870283321406128127,
-        18410715276682199039, 18428729675200069631, 18428729675200069632, 18437736874452713471,
-        18446462598732840959, 18446462598732840960, 18446464797621878783, 18446466996779287551,
-        18446603336221163519, 18446603336221196287, 18446741874686295551, 18446743249075830783,
-        18446744056529672000, 18446744056529682432, 18446744069414584320, 18446744069414601696,
-        18446744069422972927, 18446744070475743231, 18446744071562067967, 18446744073707454463,
-        18446744073709419615, 18446744073709517055, 18446744073709550595, 18446744073709551599,
-        18446744073709551600, 18446744073709551615,
+    static OFFSETS: [u8; 283] = [
+        65, 26, 6, 26, 47, 1, 10, 1, 4, 1, 5, 23, 1, 31, 1, 195, 1, 4, 4, 208, 1, 36, 7, 2, 30, 5,
+        96, 1, 42, 4, 2, 2, 2, 4, 1, 1, 6, 1, 1, 3, 1, 1, 1, 20, 1, 83, 1, 139, 8, 166, 1, 38, 9,
+        41, 0, 38, 1, 1, 5, 1, 2, 43, 2, 3, 0, 86, 2, 6, 0, 9, 7, 43, 2, 3, 64, 192, 64, 0, 2, 6, 2,
+        38, 2, 6, 2, 8, 1, 1, 1, 1, 1, 1, 1, 31, 2, 53, 1, 7, 1, 1, 3, 3, 1, 7, 3, 4, 2, 6, 4, 13,
+        5, 3, 1, 7, 116, 1, 13, 1, 16, 13, 101, 1, 4, 1, 2, 10, 1, 1, 3, 5, 6, 1, 1, 1, 1, 1, 1, 4,
+        1, 6, 4, 1, 2, 4, 5, 5, 4, 1, 17, 32, 3, 2, 0, 52, 0, 47, 1, 47, 1, 133, 6, 4, 3, 2, 12, 38,
+        1, 1, 5, 1, 0, 46, 18, 30, 132, 102, 3, 4, 1, 48, 2, 9, 42, 2, 1, 3, 0, 43, 1, 13, 7, 80, 0,
+        7, 12, 5, 0, 26, 6, 26, 0, 80, 96, 36, 4, 36, 0, 51, 13, 51, 0, 64, 0, 64, 0, 85, 1, 71, 1,
+        2, 2, 1, 2, 2, 2, 4, 1, 12, 1, 1, 1, 7, 1, 65, 1, 4, 2, 8, 1, 7, 1, 28, 1, 4, 1, 5, 1, 1, 3,
+        7, 1, 0, 2, 25, 1, 25, 1, 31, 1, 25, 1, 31, 1, 25, 1, 31, 1, 25, 1, 31, 1, 25, 1, 8, 0, 68,
+        0, 26, 6, 26, 6, 26, 0,
     ];
-
     pub fn lookup(c: char) -> bool {
-        super::range_search(
+        super::skip_search(
             c as u32,
-            &BITSET_CHUNKS_MAP,
-            BITSET_LAST_CHUNK_MAP,
-            &BITSET_INDEX_CHUNKS,
-            &BITSET,
+            &SHORT_OFFSET_RUNS,
+            &OFFSETS,
         )
     }
 }
 
 #[rustfmt::skip]
 pub mod cc {
-    static BITSET_LAST_CHUNK_MAP: (u16, u8) = (0, 0);
-    static BITSET_CHUNKS_MAP: [u8; 0] = [
+    static SHORT_OFFSET_RUNS: [u32; 1] = [
+        1114272,
     ];
-    static BITSET_INDEX_CHUNKS: [[u8; 16]; 1] = [
-        [1, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+    static OFFSETS: [u8; 5] = [
+        0, 32, 95, 33, 0,
     ];
-    static BITSET: [u64; 3] = [
-        0, 4294967295, 9223372036854775808,
-    ];
-
     pub fn lookup(c: char) -> bool {
-        super::range_search(
+        super::skip_search(
             c as u32,
-            &BITSET_CHUNKS_MAP,
-            BITSET_LAST_CHUNK_MAP,
-            &BITSET_INDEX_CHUNKS,
-            &BITSET,
+            &SHORT_OFFSET_RUNS,
+            &OFFSETS,
         )
     }
 }
 
 #[rustfmt::skip]
 pub mod grapheme_extend {
-    static BITSET_LAST_CHUNK_MAP: (u16, u8) = (896, 30);
-    static BITSET_CHUNKS_MAP: [u8; 123] = [
-        4, 15, 21, 27, 25, 3, 18, 23, 17, 0, 0, 14, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 0, 0, 0, 2, 7, 10, 0, 8, 12, 29, 28, 24, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0,
-        11, 0, 9, 0, 19, 0, 13,
+    static SHORT_OFFSET_RUNS: [u32; 31] = [
+        768, 2098307, 6292881, 10490717, 513808146, 518004748, 723528943, 731918378, 744531567,
+        752920578, 769719070, 899743232, 903937950, 912327165, 916523521, 929107236, 954273451,
+        958470191, 1180769328, 1252073203, 1315007216, 1319202639, 1327611037, 1340199269,
+        1344395776, 1373757440, 1398923568, 1419895532, 1424091344, 1429078048, 1438581232,
     ];
-    static BITSET_INDEX_CHUNKS: [[u8; 16]; 31] = [
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 20, 46],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 74, 106, 31],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 66, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 87, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 37, 70, 0, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 37, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 121, 0, 0, 48, 0, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 134, 82, 64, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 39, 0, 94, 0, 0],
-        [0, 0, 0, 0, 0, 133, 58, 0, 0, 5, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 55, 0, 0, 18, 0, 0],
-        [0, 0, 0, 21, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 71, 0, 118, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 9, 0, 0, 0, 129, 7, 26, 67, 0, 59, 140, 11, 68, 104],
-        [0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [12, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [13, 0, 50, 0, 96, 0, 0, 0, 27, 123, 139, 1, 100, 75, 57, 72],
-        [51, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0],
-        [54, 0, 0, 120, 61, 19, 105, 47, 85, 98, 131, 84, 0, 0, 0, 56],
-        [60, 28, 0, 141, 99, 45, 111, 109, 97, 83, 97, 136, 132, 44, 108, 22],
-        [63, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [89, 0, 0, 91, 0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0],
-        [93, 0, 0, 0, 113, 3, 0, 40, 115, 29, 24, 0, 0, 0, 0, 0],
-        [114, 78, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0],
-        [128, 0, 86, 0, 127, 8, 23, 0, 0, 0, 0, 76, 0, 0, 0, 0],
-        [130, 42, 122, 41, 112, 43, 2, 36, 95, 15, 101, 14, 90, 117, 102, 6],
-        [137, 34, 124, 4, 0, 0, 126, 32, 125, 1, 88, 0, 53, 0, 0, 0],
-        [138, 119, 92, 0, 81, 73, 116, 17, 110, 0, 0, 52, 112, 80, 0, 0],
-        [142, 143, 0, 0, 143, 143, 143, 66, 0, 0, 0, 0, 0, 0, 0, 0],
-    ];
-    static BITSET: [u64; 144] = [
-        0, 1, 2, 8, 13, 28, 64, 182, 191, 1016, 2032, 2047, 4096, 14336, 16128, 32640, 32768,
-        40448, 131008, 262016, 491520, 8323072, 8396801, 10682368, 58719232, 100663296, 134152192,
-        159383552, 234881024, 243138688, 536879204, 537919040, 805306369, 1073741824, 1073741916,
-        1610612736, 2153546752, 3221225472, 3758096384, 4294967296, 4512022528, 51545911364,
-        51545914817, 51548004382, 51554295838, 51556262398, 68719476736, 137438953472, 412316860416,
-        1030792151040, 2199023255648, 8641373536127, 8763880767488, 17303886364672, 18004502906948,
-        26388279066624, 36421322670080, 65128884076547, 65970697670631, 67755789254656,
-        69200441769984, 70093866270720, 263882790666240, 277076930199552, 281470547525648,
-        281470681808895, 281474976710655, 281479271675904, 562675075514368, 562949953355776,
-        844424930131968, 985162418487296, 1023920203366400, 2251799813685248, 3377699721314304,
-        4494803534348292, 6755399441055744, 7881299349733376, 8444256867844096, 8725724278030336,
-        8760633780600833, 8989057312882695, 9042383626829823, 9851624185018758, 18067175067615234,
-        28848986089586688, 30958948903026688, 35747322042253312, 53805701016846336,
-        58529202969772032, 189151184399892480, 220713756545974272, 466122561432846339,
-        504262420777140224, 558446353793941504, 572520102629474304, 1009933895770046464,
-        1152921504606846982, 1152921504606851080, 1441151880758558727, 1724878657282899983,
-        2301902359539744768, 2305843009196908767, 2305843009213693952, 2310337812748042240,
-        3731232291276455943, 4589168020290535424, 4609293481125347328, 4611686018427387908,
-        4611686069975392286, 4671217976001691648, 5764607523034234882, 6341068275337658371,
-        6341349750314369024, 7421334051581067264, 8788774672813524990, 9205357638345293827,
-        9222809086901354496, 9223372036854775808, 9223372036854775935, 9224497932466651184,
-        9727775195120332910, 10376293541461622786, 11526998316797657088, 11959590285459062784,
-        12103423998558208000, 12699165786766311424, 13005832773892571136, 13798747783286489088,
-        13835058055282032640, 13835058055282163729, 13951307220663664640, 14987979559889010690,
-        17872468738205286400, 17906312118425092095, 18158513697557839871, 18158513749097456062,
-        18374686479671623680, 18374686479671623682, 18446462598732840960, 18446462598732972032,
-        18446744056529158144, 18446744069414584320, 18446744073709551615,
+    static OFFSETS: [u8; 689] = [
+        0, 112, 0, 7, 0, 45, 1, 1, 1, 2, 1, 2, 1, 1, 72, 11, 48, 21, 16, 1, 101, 7, 2, 6, 2, 2, 1,
+        4, 35, 1, 30, 27, 91, 11, 58, 9, 9, 1, 24, 4, 1, 9, 1, 3, 1, 5, 43, 3, 119, 15, 1, 32, 55,
+        1, 1, 1, 4, 8, 4, 1, 3, 7, 10, 2, 29, 1, 58, 1, 1, 1, 2, 4, 8, 1, 9, 1, 10, 2, 26, 1, 2, 2,
+        57, 1, 4, 2, 4, 2, 2, 3, 3, 1, 30, 2, 3, 1, 11, 2, 57, 1, 4, 5, 1, 2, 4, 1, 20, 2, 22, 6, 1,
+        1, 58, 1, 1, 2, 1, 4, 8, 1, 7, 3, 10, 2, 30, 1, 59, 1, 1, 1, 12, 1, 9, 1, 40, 1, 3, 1, 57,
+        3, 5, 3, 1, 4, 7, 2, 11, 2, 29, 1, 58, 1, 2, 1, 2, 1, 3, 1, 5, 2, 7, 2, 11, 2, 28, 2, 57, 2,
+        1, 1, 2, 4, 8, 1, 9, 1, 10, 2, 29, 1, 72, 1, 4, 1, 2, 3, 1, 1, 8, 1, 81, 1, 2, 7, 12, 8, 98,
+        1, 2, 9, 11, 6, 74, 2, 27, 1, 1, 1, 1, 1, 55, 14, 1, 5, 1, 2, 5, 11, 1, 36, 9, 1, 102, 4, 1,
+        6, 1, 2, 2, 2, 25, 2, 4, 3, 16, 4, 13, 1, 2, 2, 6, 1, 15, 1, 0, 3, 0, 3, 29, 3, 29, 2, 30,
+        2, 64, 2, 1, 7, 8, 1, 2, 11, 9, 1, 45, 3, 119, 2, 34, 1, 118, 3, 4, 2, 9, 1, 6, 3, 219, 2,
+        2, 1, 58, 1, 1, 7, 1, 1, 1, 1, 2, 8, 6, 10, 2, 1, 48, 17, 63, 4, 48, 7, 1, 1, 5, 1, 40, 9,
+        12, 2, 32, 4, 2, 2, 1, 3, 56, 1, 1, 2, 3, 1, 1, 3, 58, 8, 2, 2, 152, 3, 1, 13, 1, 7, 4, 1,
+        6, 1, 3, 2, 198, 58, 1, 5, 0, 1, 195, 33, 0, 3, 141, 1, 96, 32, 0, 6, 105, 2, 0, 4, 1, 10,
+        32, 2, 80, 2, 0, 1, 3, 1, 4, 1, 25, 2, 5, 1, 151, 2, 26, 18, 13, 1, 38, 8, 25, 11, 46, 3,
+        48, 1, 2, 4, 2, 2, 39, 1, 67, 6, 2, 2, 2, 2, 12, 1, 8, 1, 47, 1, 51, 1, 1, 3, 2, 2, 5, 2, 1,
+        1, 42, 2, 8, 1, 238, 1, 2, 1, 4, 1, 0, 1, 0, 16, 16, 16, 0, 2, 0, 1, 226, 1, 149, 5, 0, 3,
+        1, 2, 5, 4, 40, 3, 4, 1, 165, 2, 0, 4, 0, 2, 153, 11, 176, 1, 54, 15, 56, 3, 49, 4, 2, 2,
+        69, 3, 36, 5, 1, 8, 62, 1, 12, 2, 52, 9, 10, 4, 2, 1, 95, 3, 2, 1, 1, 2, 6, 1, 160, 1, 3, 8,
+        21, 2, 57, 2, 1, 1, 1, 1, 22, 1, 14, 7, 3, 5, 195, 8, 2, 3, 1, 1, 23, 1, 81, 1, 2, 6, 1, 1,
+        2, 1, 1, 2, 1, 2, 235, 1, 2, 4, 6, 2, 1, 2, 27, 2, 85, 8, 2, 1, 1, 2, 106, 1, 1, 1, 2, 6, 1,
+        1, 101, 3, 2, 4, 1, 5, 0, 9, 1, 2, 245, 1, 10, 2, 1, 1, 4, 1, 144, 4, 2, 2, 4, 1, 32, 10,
+        40, 6, 2, 4, 8, 1, 9, 6, 2, 3, 46, 13, 1, 2, 0, 7, 1, 6, 1, 1, 82, 22, 2, 7, 1, 2, 1, 2,
+        122, 6, 3, 1, 1, 2, 1, 7, 1, 1, 72, 2, 3, 1, 1, 1, 0, 2, 0, 5, 59, 7, 0, 1, 63, 4, 81, 1, 0,
+        2, 0, 1, 1, 3, 4, 5, 8, 8, 2, 7, 30, 4, 148, 3, 0, 55, 4, 50, 8, 1, 14, 1, 22, 5, 1, 15, 0,
+        7, 1, 17, 2, 7, 1, 2, 1, 5, 0, 7, 0, 4, 0, 7, 109, 7, 0, 96, 128, 240, 0,
     ];
-
     pub fn lookup(c: char) -> bool {
-        super::range_search(
+        super::skip_search(
             c as u32,
-            &BITSET_CHUNKS_MAP,
-            BITSET_LAST_CHUNK_MAP,
-            &BITSET_INDEX_CHUNKS,
-            &BITSET,
+            &SHORT_OFFSET_RUNS,
+            &OFFSETS,
         )
     }
 }
 
 #[rustfmt::skip]
 pub mod lowercase {
-    static BITSET_LAST_CHUNK_MAP: (u16, u8) = (122, 6);
-    static BITSET_CHUNKS_MAP: [u8; 118] = [
-        12, 16, 0, 0, 10, 0, 0, 11, 13, 8, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 0, 2, 1, 0, 17, 0, 9, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14,
+    static BITSET_CHUNKS_MAP: [u8; 123] = [
+        13, 16, 0, 0, 8, 0, 0, 11, 12, 9, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 3, 1, 0, 14, 0, 7, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0,
+        0, 0, 6,
     ];
     static BITSET_INDEX_CHUNKS: [[u8; 16]; 18] = [
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 62, 71, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 9, 0, 50, 42, 44, 28],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 68, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35],
-        [0, 0, 3, 0, 71, 71, 71, 0, 46, 46, 48, 46, 24, 37, 38, 23],
-        [0, 29, 27, 57, 39, 51, 52, 43, 41, 70, 26, 11, 0, 34, 64, 32],
-        [0, 40, 8, 0, 33, 60, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [22, 13, 54, 66, 25, 15, 56, 63, 30, 19, 12, 55, 58, 61, 65, 4],
-        [59, 36, 46, 21, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [59, 49, 45, 47, 18, 69, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [67, 5, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0],
+        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 14, 52, 0],
+        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 39, 0, 47, 43, 45, 30],
+        [0, 0, 0, 0, 10, 53, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26],
+        [0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 0, 54, 0, 52, 52, 52, 0, 21, 21, 64, 21, 33, 24, 23, 34],
+        [0, 5, 71, 0, 28, 15, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [0, 61, 31, 17, 22, 48, 49, 44, 42, 8, 32, 38, 0, 27, 13, 29],
+        [11, 55, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [16, 25, 21, 35, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [16, 46, 2, 20, 63, 9, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+        [60, 37, 51, 12, 70, 58, 18, 1, 6, 59, 68, 19, 65, 66, 3, 41],
+    ];
+    static BITSET_CANONICAL: [u64; 52] = [
+        0b0000000000000000000000000000000000000000000000000000000000000000,
+        0b1111111111111111110000000000000000000000000011111111111111111111,
+        0b1010101010101010101010101010101010101010101010101010100000000010,
+        0b1111111111111111111111000000000000000000000000001111110111111111,
+        0b0000111111111111111111111111111111111111000000000000000000000000,
+        0b1000000000000010000000000000000000000000000000000000000000000000,
+        0b0000111111111111111111111111110000000000000000000000000011111111,
+        0b0000000000000111111111111111111111111111111111111111111111111111,
+        0b1111111111111111111111111111111111111111111111111010101010000101,
+        0b1111111111111111111111111111111100000000000000000000000000000000,
+        0b1111111111111111111111111111110000000000000000000000000000000000,
+        0b1111111111111111111111110000000000000000000000000000000000000000,
+        0b1111111111111111111111000000000000000000000000001111111111101111,
+        0b1111111111111111111100000000000000000000000000010000000000000000,
+        0b1111111111111111000000011111111111110111111111111111111111111111,
+        0b1111111111111111000000000000000000000000000000000100001111000000,
+        0b1111111111111111000000000000000000000000000000000000000000000000,
+        0b1111111101111111111111111111111110000000000000000000000000000000,
+        0b1111110000000000000000000000000011111111111111111111111111000000,
+        0b1111000000000000000000000000001111110111111111111111111111111100,
+        0b1010101010101010101010101010101010101010101010101101010101010100,
+        0b1010101010101010101010101010101010101010101010101010101010101010,
+        0b0101010110101010101010101010101010101010101010101010101010101010,
+        0b0100000011011111000000001111111100000000111111110000000011111111,
+        0b0011111111111111000000001111111100000000111111110000000000111111,
+        0b0011111111011010000101010110001001111111111111111111111111111111,
+        0b0011111100000000000000000000000000000000000000000000000000000000,
+        0b0011110010001010000000000000000000000000000000000000000000100000,
+        0b0011001000010000100000000000000000000000000010001100010000000000,
+        0b0001100100101111101010101010101010101010111000110111111111111111,
+        0b0000011101000000000000000000000000000000000000000000010100001000,
+        0b0000010000100000000001000000000000000000000000000000000000000000,
+        0b0000000111111111111111111111111111111111111011111111111111111111,
+        0b0000000011111111000000001111111100000000001111110000000011111111,
+        0b0000000011011100000000001111111100000000110011110000000011011100,
+        0b0000000000001000010100000001101010101010101010101010101010101010,
+        0b0000000000000000001000001011111111111111111111111111111111111111,
+        0b0000000000000000000000001111111111111111110111111100000000000000,
+        0b0000000000000000000000000001111100000000000000000000000000000011,
+        0b0000000000000000000000000000000000111010101010101010101010101010,
+        0b0000000000000000000000000000000000000000111110000000000001111111,
+        0b0000000000000000000000000000000000000000000000000000101111110111,
+        0b1001001111111010101010101010101010101010101010101010101010101010,
+        0b1001010111111111101010101010101010101010101010101010101010101010,
+        0b1010101000101001101010101010101010110101010101010101001001000000,
+        0b1010101010100000100000101010101010101010101110100101000010101010,
+        0b1010101010101010101010101010101011111111111111111111111111111111,
+        0b1010101010101011101010101010100000000000000000000000000000000000,
+        0b1101010010101010101010101010101010101010101010101010101101010101,
+        0b1110011001010001001011010010101001001110001001000011000100101001,
+        0b1110011111111111111111111111111111111111111111110000000000000000,
+        0b1110101111000000000000000000000000001111111111111111111111111100,
     ];
-    static BITSET: [u64; 72] = [
-        0, 15, 16, 511, 3063, 65535, 16253055, 134217726, 536805376, 984263338, 4294967295,
-        133143986179, 274877905920, 1099509514240, 4398046445568, 17592185782272, 36009005809663,
-        46912496118442, 187649984473770, 281474972516352, 2251799813685247, 2339875276368554,
-        4503599560261632, 61925590106570972, 71777214282006783, 72057592964186127,
-        144115188074807295, 297241973452963840, 522417556774978824, 576460743713488896,
-        1152921487426978047, 1152921504590069760, 1814856824841797631, 3607524039012697088,
-        4362299189061746720, 4539628424389459968, 4601013482110844927, 4611405638684049471,
-        4674456033467236607, 6172933889249159850, 9223934986808197120, 10663022717737544362,
-        10808545280696953514, 12261519110656315968, 12294970652241842346, 12297829382473033730,
-        12297829382473034410, 12297829382473045332, 12297829382829550250, 12297829383904690175,
-        12298110845996498944, 15324248332066007893, 16596095761559859497, 16717361816799215616,
-        16987577794709946364, 17293822586148356092, 18158513701852807104, 18410715274543104000,
-        18428729675466407935, 18446462598732840960, 18446462598732858304, 18446462598737002495,
-        18446464797621878783, 18446673704966422527, 18446726481523572736, 18446739675663105535,
-        18446739675663106031, 18446742974197923840, 18446744056529682432, 18446744069414584320,
-        18446744073709529733, 18446744073709551615,
+    static BITSET_MAPPING: [(u8, u8); 20] = [
+        (0, 64), (1, 188), (1, 183), (1, 176), (1, 109), (1, 124), (1, 126), (1, 66), (1, 70),
+        (1, 77), (2, 146), (2, 144), (2, 83), (3, 12), (3, 6), (4, 156), (4, 78), (5, 187),
+        (6, 132), (7, 93),
     ];
 
     pub fn lookup(c: char) -> bool {
-        super::range_search(
+        super::bitset_search(
             c as u32,
             &BITSET_CHUNKS_MAP,
-            BITSET_LAST_CHUNK_MAP,
             &BITSET_INDEX_CHUNKS,
-            &BITSET,
+            &BITSET_CANONICAL,
+            &BITSET_MAPPING,
         )
     }
 }
 
 #[rustfmt::skip]
 pub mod n {
-    static BITSET_LAST_CHUNK_MAP: (u16, u8) = (127, 0);
-    static BITSET_CHUNKS_MAP: [u8; 127] = [
-        31, 8, 11, 25, 19, 4, 29, 21, 24, 28, 0, 16, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 0, 0, 0, 0, 3, 13, 18, 26, 17, 23, 20, 15, 22, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 0, 0, 0, 7, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        5, 2, 0, 0, 10, 0, 14, 27, 12, 0, 1,
+    static SHORT_OFFSET_RUNS: [u32; 38] = [
+        1632, 18876774, 31461440, 102765417, 111154926, 115349830, 132128880, 165684320, 186656630,
+        195046653, 199241735, 203436434, 216049184, 241215536, 249605104, 274792208, 278987015,
+        283181793, 295766104, 320933114, 383848032, 392238160, 434181712, 442570976, 455154768,
+        463544256, 476128256, 480340576, 484535936, 497144544, 501340110, 509731136, 513925872,
+        518121671, 522316913, 530706688, 551681008, 556989434,
     ];
-    static BITSET_INDEX_CHUNKS: [[u8; 16]; 34] = [
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 49],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 43, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 22, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 47, 0, 0, 0, 2],
-        [0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 31, 0, 0, 0, 0],
-        [0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0],
-        [0, 0, 0, 0, 0, 31, 0, 45, 0, 31, 0, 31, 0, 41, 0, 34],
-        [0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 37, 44, 4, 0, 0, 0, 0, 52, 23, 3, 0, 13],
-        [0, 0, 0, 7, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 35, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 62, 47, 0, 0, 0, 0, 60, 0, 0, 24, 10, 0, 5],
-        [0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 2, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0],
-        [0, 15, 0, 15, 0, 0, 0, 0, 0, 15, 0, 2, 51, 0, 0, 0],
-        [0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 26, 0, 0, 0, 15, 25, 0, 0, 0, 0, 0, 0, 0, 0, 11],
-        [0, 32, 0, 47, 65, 0, 0, 39, 0, 0, 0, 47, 0, 0, 0, 0],
-        [0, 46, 2, 0, 0, 71, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 59, 0, 31, 0, 42, 0, 31, 0, 15, 0, 15, 36, 0, 0, 0],
-        [0, 63, 30, 61, 18, 0, 55, 70, 0, 57, 20, 28, 0, 64, 29, 0],
-        [0, 66, 38, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 69, 19, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 9, 0],
-        [15, 0, 0, 0, 0, 8, 0, 17, 0, 0, 16, 0, 0, 15, 47, 0],
-        [40, 0, 0, 15, 2, 0, 0, 48, 0, 15, 0, 0, 0, 0, 0, 47],
-        [47, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [50, 0, 0, 0, 0, 0, 12, 0, 25, 21, 67, 0, 0, 0, 0, 0],
-        [73, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-    ];
-    static BITSET: [u64; 74] = [
-        0, 999, 1023, 1026, 3072, 4064, 8191, 65408, 65472, 1048575, 1966080, 2097151, 3932160,
-        4063232, 8388607, 67043328, 67044351, 134152192, 264241152, 268435455, 3758096384,
-        4294901504, 17112694784, 64424509440, 549218942976, 4393751543808, 35184372023296,
-        140737488355327, 272678883688448, 279275953455104, 280925220896768, 281200098803712,
-        281474976448512, 492581209243648, 2251524935778304, 2251795518717952, 4503595332403200,
-        4503599627370368, 8708132091985919, 9007190731849728, 17732923532771328, 71212894229889024,
-        144114915328655360, 144115183780888576, 144115188075855871, 284007976623144960,
-        284008251501051904, 287948901175001088, 287948901242044416, 287953294926544896,
-        504407547722072192, 1152640029630136320, 1152921496016912384, 2305840810190438400,
-        2305843009213693952, 3458764513820540928, 4611615649683210238, 6917529027641082367,
-        8217943420044312576, 9151595642915651584, 9223372032559808512, 17870283321406128128,
-        18158513697557839872, 18302628889911885824, 18374686483949813760, 18428729675200069632,
-        18446181123756130304, 18446181123756131327, 18446739675663040512, 18446744069414584320,
-        18446744073709355007, 18446744073709486080, 18446744073709535232, 18446744073709551615,
+    static OFFSETS: [u8; 267] = [
+        48, 10, 120, 2, 5, 1, 2, 3, 0, 10, 134, 10, 198, 10, 0, 10, 118, 10, 4, 6, 108, 10, 118,
+        10, 118, 10, 2, 6, 110, 13, 115, 10, 8, 7, 103, 10, 104, 7, 7, 19, 109, 10, 96, 10, 118, 10,
+        70, 20, 0, 10, 70, 10, 0, 20, 0, 3, 239, 10, 6, 10, 22, 10, 0, 10, 128, 11, 165, 10, 6, 10,
+        182, 10, 86, 10, 134, 10, 6, 10, 0, 1, 3, 6, 6, 10, 198, 51, 2, 5, 0, 60, 78, 22, 0, 30, 0,
+        1, 0, 1, 25, 9, 14, 3, 0, 4, 138, 10, 30, 8, 1, 15, 32, 10, 39, 15, 0, 10, 188, 10, 0, 6,
+        154, 10, 38, 10, 198, 10, 22, 10, 86, 10, 0, 10, 0, 10, 0, 45, 12, 57, 17, 2, 0, 27, 36, 4,
+        29, 1, 8, 1, 134, 5, 202, 10, 0, 8, 25, 7, 39, 9, 75, 5, 22, 6, 160, 2, 2, 16, 2, 46, 64, 9,
+        52, 2, 30, 3, 75, 5, 104, 8, 24, 8, 41, 7, 0, 6, 48, 10, 0, 31, 158, 10, 42, 4, 112, 7, 134,
+        30, 128, 10, 60, 10, 144, 10, 7, 20, 251, 10, 0, 10, 118, 10, 0, 10, 102, 10, 102, 12, 0,
+        19, 93, 10, 0, 29, 227, 10, 70, 10, 0, 21, 0, 111, 0, 10, 230, 10, 1, 7, 0, 23, 0, 20, 108,
+        25, 0, 50, 0, 10, 0, 10, 0, 9, 128, 10, 0, 59, 1, 3, 1, 4, 76, 45, 1, 15, 0, 13, 0, 10, 0,
     ];
-
     pub fn lookup(c: char) -> bool {
-        super::range_search(
+        super::skip_search(
             c as u32,
-            &BITSET_CHUNKS_MAP,
-            BITSET_LAST_CHUNK_MAP,
-            &BITSET_INDEX_CHUNKS,
-            &BITSET,
+            &SHORT_OFFSET_RUNS,
+            &OFFSETS,
         )
     }
 }
 
 #[rustfmt::skip]
 pub mod uppercase {
-    static BITSET_LAST_CHUNK_MAP: (u16, u8) = (124, 6);
-    static BITSET_CHUNKS_MAP: [u8; 123] = [
-        12, 15, 0, 0, 11, 0, 0, 8, 5, 9, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 0, 1, 0, 13, 0, 7, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0,
-        0, 0, 4,
+    static BITSET_CHUNKS_MAP: [u8; 125] = [
+        12, 15, 5, 5, 0, 5, 5, 2, 4, 11, 5, 14, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
+        5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 8, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
+        5, 5, 5, 6, 5, 13, 5, 10, 5, 5, 1, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
+        5, 7, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 16, 5, 5,
+        5, 5, 9, 5, 3,
     ];
     static BITSET_INDEX_CHUNKS: [[u8; 16]; 17] = [
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 9, 0, 38, 46, 44, 28],
-        [0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 51, 23, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 60, 62, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 54, 0, 0, 0, 0, 0, 43, 43, 40, 43, 56, 22, 34, 35],
-        [0, 0, 57, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 66, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 66, 30],
-        [0, 10, 0, 11, 50, 37, 36, 45, 47, 5, 0, 0, 0, 49, 17, 53],
-        [14, 0, 60, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [21, 52, 43, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [24, 39, 42, 41, 59, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [58, 65, 29, 16, 48, 63, 31, 19, 55, 61, 64, 32, 27, 20, 15, 3],
+        [41, 41, 5, 33, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 5, 0],
+        [41, 41, 5, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41],
+        [41, 41, 38, 41, 41, 41, 41, 41, 17, 17, 61, 17, 40, 29, 24, 23],
+        [41, 41, 41, 41, 9, 8, 42, 41, 41, 41, 41, 41, 41, 41, 41, 41],
+        [41, 41, 41, 41, 35, 28, 65, 41, 41, 41, 41, 41, 41, 41, 41, 41],
+        [41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41],
+        [41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 56, 41, 41, 41],
+        [41, 41, 41, 41, 41, 41, 41, 41, 41, 46, 41, 41, 41, 41, 41, 41],
+        [41, 41, 41, 41, 41, 41, 41, 41, 41, 60, 59, 41, 20, 14, 16, 4],
+        [41, 41, 41, 41, 47, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41],
+        [41, 41, 51, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41],
+        [41, 41, 52, 43, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41],
+        [41, 53, 41, 31, 34, 21, 22, 15, 13, 32, 41, 41, 41, 11, 30, 37],
+        [48, 41, 9, 44, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41],
+        [49, 36, 17, 27, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41],
+        [50, 19, 2, 18, 10, 45, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41],
+        [57, 1, 26, 54, 12, 7, 25, 55, 39, 58, 6, 3, 64, 63, 62, 66],
+    ];
+    static BITSET_CANONICAL: [u64; 41] = [
+        0b0000000000111111111111111111111111111111111111111111111111111111,
+        0b1111111111111111111111110000000000000000000000000011111111111111,
+        0b0101010101010101010101010101010101010101010101010101010000000001,
+        0b0000011111111111111111111111110000000000000000000000000000000001,
+        0b0000000000100000000000000000000000000000000000000000001011110100,
+        0b1111111111111111111111111111111100000000000000000000000000000000,
+        0b1111111111111111111111110000000000000000000000000000001111111111,
+        0b1111111111111111111100000000000000000000000000011111110001011111,
+        0b1111111111111111000000111111111111111111111111110000001111111111,
+        0b1111111111111111000000000000000000000000000000000000000000000000,
+        0b1111111111111110010101010101010101010101010101010101010101010101,
+        0b1000000001000101000000000000000000000000000000000000000000000000,
+        0b0111101100000000000000000000000000011111110111111110011110110000,
+        0b0110110000000101010101010101010101010101010101010101010101010101,
+        0b0110101000000000010101010101010101010101010101010101010101010101,
+        0b0101010111010010010101010101010101001010101010101010010010010000,
+        0b0101010101011111011111010101010101010101010001010010100001010101,
+        0b0101010101010101010101010101010101010101010101010101010101010101,
+        0b0101010101010101010101010101010101010101010101010010101010101011,
+        0b0101010101010101010101010101010100000000000000000000000000000000,
+        0b0101010101010100010101010101010000000000000000000000000000000000,
+        0b0010101101010101010101010101010101010101010101010101010010101010,
+        0b0001000110101110110100101101010110110001110110111100111011010110,
+        0b0000111100000000000111110000000000001111000000000000111100000000,
+        0b0000111100000000000000000000000000000000000000000000000000000000,
+        0b0000001111111111111111111111111100000000000000000000000000111111,
+        0b0000000000111111110111100110010011010000000000000000000000000011,
+        0b0000000000000100001010000000010101010101010101010101010101010101,
+        0b0000000000000000111111111111111100000000000000000000000000100000,
+        0b0000000000000000111111110000000010101010000000000011111100000000,
+        0b0000000000000000000011111111101111111111111111101101011101000000,
+        0b0000000000000000000000000000000001111111011111111111111111111111,
+        0b0000000000000000000000000000000000000000000000000101010101111010,
+        0b0000000000000000000000000000000000000000000000000010000010111111,
+        0b1010101001010101010101010101010101010101010101010101010101010101,
+        0b1100000000001111001111010101000000111110001001110011100010000100,
+        0b1100000000100101111010101001110100000000000000000000000000000000,
+        0b1110011010010000010101010101010101010101000111001000000000000000,
+        0b1110011111111111111111111111111111111111111111110000000000000000,
+        0b1111000000000000000000000000001111111111111111111111111100000000,
+        0b1111111100000000111111110000000000111111000000001111111100000000,
     ];
-    static BITSET: [u64; 67] = [
-        0, 8, 1023, 1024, 8383, 21882, 65535, 1048575, 8388607, 89478485, 134217726, 2139095039,
-        4294967295, 17179869183, 1099511627775, 2199023190016, 4398046445568, 17575006099264,
-        23456248059221, 70368743129088, 140737484161024, 140737488355327, 280378317225728,
-        281470681743392, 281474976710655, 1169903278445909, 2251799813685247, 9007198986305536,
-        9007199254741748, 17977448100528131, 18014398509481983, 288230371856744511,
-        576460735123554305, 576460743713488896, 1080863910568919040, 1080897995681042176,
-        1274187559846268630, 3122495741643543722, 6148633210533183488, 6148914689804861440,
-        6148914690880001365, 6148914691236506283, 6148914691236516865, 6148914691236517205,
-        6151773421467674709, 6184099063146390672, 7638198793012598101, 7783721355972007253,
-        8863084067199903664, 9242793810247811072, 12273810184460391765, 13839347594782259332,
-        13845730589451223040, 16613872850358272000, 16717361816799215616, 17293822586282573568,
-        18374966856193736448, 18428729675200069632, 18442240474149289983, 18446274948748367189,
-        18446462598732840960, 18446462598737035263, 18446466996779287551, 18446726481523637343,
-        18446742974197924863, 18446742974197940223, 18446744069414584320,
+    static BITSET_MAPPING: [(u8, u8); 26] = [
+        (0, 182), (0, 74), (0, 166), (0, 162), (0, 159), (0, 150), (0, 148), (0, 142), (0, 135),
+        (0, 134), (0, 131), (0, 64), (1, 115), (1, 66), (1, 70), (1, 83), (1, 12), (1, 8), (2, 164),
+        (2, 146), (2, 20), (3, 146), (3, 140), (3, 134), (4, 178), (4, 171),
     ];
 
     pub fn lookup(c: char) -> bool {
-        super::range_search(
+        super::bitset_search(
             c as u32,
             &BITSET_CHUNKS_MAP,
-            BITSET_LAST_CHUNK_MAP,
             &BITSET_INDEX_CHUNKS,
-            &BITSET,
+            &BITSET_CANONICAL,
+            &BITSET_MAPPING,
         )
     }
 }
 
 #[rustfmt::skip]
 pub mod white_space {
-    static BITSET_LAST_CHUNK_MAP: (u16, u8) = (12, 2);
-    static BITSET_CHUNKS_MAP: [u8; 9] = [
-        3, 0, 0, 0, 0, 1, 0, 0, 4,
-    ];
-    static BITSET_INDEX_CHUNKS: [[u8; 16]; 5] = [
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0],
-        [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [4, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-        [5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+    static SHORT_OFFSET_RUNS: [u32; 4] = [
+        5760, 18882560, 23080960, 40972289,
     ];
-    static BITSET: [u64; 6] = [
-        0, 1, 2147483648, 4294967328, 4294983168, 144036023240703,
+    static OFFSETS: [u8; 21] = [
+        9, 5, 18, 1, 100, 1, 26, 1, 0, 1, 0, 11, 29, 2, 5, 1, 47, 1, 0, 1, 0,
     ];
-
     pub fn lookup(c: char) -> bool {
-        super::range_search(
+        super::skip_search(
             c as u32,
-            &BITSET_CHUNKS_MAP,
-            BITSET_LAST_CHUNK_MAP,
-            &BITSET_INDEX_CHUNKS,
-            &BITSET,
+            &SHORT_OFFSET_RUNS,
+            &OFFSETS,
         )
     }
 }

diff --git a/src/librustc_ast/ast.rs b/src/librustc_ast/ast.rs
index 646294a7cca7871801a49db77a392fbef11ab9cb..6586280d21454aa63aa5e2a8fcbfdc46cf5235d5 100644 (file)
--- a/src/librustc_ast/ast.rs
+++ b/src/librustc_ast/ast.rs
@@ -214,11 +214,18 @@ pub fn span(&self) -> Span {
 pub struct AngleBracketedArgs {
     /// The overall span.
     pub span: Span,
-    /// The arguments for this path segment.
-    pub args: Vec<GenericArg>,
-    /// Constraints on associated types, if any.
-    /// E.g., `Foo<A = Bar, B: Baz>`.
-    pub constraints: Vec<AssocTyConstraint>,
+    /// The comma separated parts in the `<...>`.
+    pub args: Vec<AngleBracketedArg>,
+}
+
+/// Either an argument for a parameter e.g., `'a`, `Vec<u8>`, `0`,
+/// or a constraint on an associated item, e.g., `Item = String` or `Item: Bound`.
+#[derive(Clone, RustcEncodable, RustcDecodable, Debug)]
+pub enum AngleBracketedArg {
+    /// Argument for a generic parameter.
+    Arg(GenericArg),
+    /// Constraint for an associated item.
+    Constraint(AssocTyConstraint),
 }
 
 impl Into<Option<P<GenericArgs>>> for AngleBracketedArgs {
@@ -248,11 +255,13 @@ pub struct ParenthesizedArgs {
 
 impl ParenthesizedArgs {
     pub fn as_angle_bracketed_args(&self) -> AngleBracketedArgs {
-        AngleBracketedArgs {
-            span: self.span,
-            args: self.inputs.iter().cloned().map(GenericArg::Type).collect(),
-            constraints: vec![],
-        }
+        let args = self
+            .inputs
+            .iter()
+            .cloned()
+            .map(|input| AngleBracketedArg::Arg(GenericArg::Type(input)))
+            .collect();
+        AngleBracketedArgs { span: self.span, args }
     }
 }
 

diff --git a/src/librustc_ast/mut_visit.rs b/src/librustc_ast/mut_visit.rs
index 67f7764d5bb27bf94a8f3923abd04060a8f6f300..a72a60c30b28a049dec4f28b47cb42368534cecd 100644 (file)
--- a/src/librustc_ast/mut_visit.rs
+++ b/src/librustc_ast/mut_visit.rs
@@ -546,9 +546,11 @@ pub fn noop_visit_angle_bracketed_parameter_data<T: MutVisitor>(
     data: &mut AngleBracketedArgs,
     vis: &mut T,
 ) {
-    let AngleBracketedArgs { args, constraints, span } = data;
-    visit_vec(args, |arg| vis.visit_generic_arg(arg));
-    visit_vec(constraints, |constraint| vis.visit_ty_constraint(constraint));
+    let AngleBracketedArgs { args, span } = data;
+    visit_vec(args, |arg| match arg {
+        AngleBracketedArg::Arg(arg) => vis.visit_generic_arg(arg),
+        AngleBracketedArg::Constraint(constraint) => vis.visit_ty_constraint(constraint),
+    });
     vis.visit_span(span);
 }
 

diff --git a/src/librustc_ast/visit.rs b/src/librustc_ast/visit.rs
index cc2b1c48b51acc093fda84303c9a2ae618a3c23c..51863350a8cf38f3508a1ede7e9616741a41d3f7 100644 (file)
--- a/src/librustc_ast/visit.rs
+++ b/src/librustc_ast/visit.rs
@@ -464,8 +464,12 @@ pub fn walk_generic_args<'a, V>(visitor: &mut V, _path_span: Span, generic_args:
 {
     match *generic_args {
         GenericArgs::AngleBracketed(ref data) => {
-            walk_list!(visitor, visit_generic_arg, &data.args);
-            walk_list!(visitor, visit_assoc_ty_constraint, &data.constraints);
+            for arg in &data.args {
+                match arg {
+                    AngleBracketedArg::Arg(a) => visitor.visit_generic_arg(a),
+                    AngleBracketedArg::Constraint(c) => visitor.visit_assoc_ty_constraint(c),
+                }
+            }
         }
         GenericArgs::Parenthesized(ref data) => {
             walk_list!(visitor, visit_ty, &data.inputs);

diff --git a/src/librustc_ast_lowering/lib.rs b/src/librustc_ast_lowering/lib.rs
index 68cb1f8585fc721e7dc76aebde1de89f18bbf993..45ee7265c15fcb2b6124bb21dad05fb7ee0391db 100644 (file)
--- a/src/librustc_ast_lowering/lib.rs
+++ b/src/librustc_ast_lowering/lib.rs
@@ -34,6 +34,7 @@
 #![feature(crate_visibility_modifier)]
 #![feature(marker_trait_attr)]
 #![feature(specialization)]
+#![feature(or_patterns)]
 #![recursion_limit = "256"]
 
 use rustc_ast::ast;

diff --git a/src/librustc_ast_lowering/path.rs b/src/librustc_ast_lowering/path.rs
index e996bdac7cb370c7e9813495e06001b600abf352..dde734756517c099fcc94bb843ba64b8d5795ca7 100644 (file)
--- a/src/librustc_ast_lowering/path.rs
+++ b/src/librustc_ast_lowering/path.rs
@@ -366,22 +366,27 @@ fn lower_angle_bracketed_parameter_data(
         param_mode: ParamMode,
         mut itctx: ImplTraitContext<'_, 'hir>,
     ) -> (GenericArgsCtor<'hir>, bool) {
-        let &AngleBracketedArgs { ref args, ref constraints, .. } = data;
-        let has_non_lt_args = args.iter().any(|arg| match arg {
-            ast::GenericArg::Lifetime(_) => false,
-            ast::GenericArg::Type(_) => true,
-            ast::GenericArg::Const(_) => true,
+        let has_non_lt_args = data.args.iter().any(|arg| match arg {
+            AngleBracketedArg::Arg(ast::GenericArg::Lifetime(_))
+            | AngleBracketedArg::Constraint(_) => false,
+            AngleBracketedArg::Arg(ast::GenericArg::Type(_) | ast::GenericArg::Const(_)) => true,
         });
-        (
-            GenericArgsCtor {
-                args: args.iter().map(|a| self.lower_generic_arg(a, itctx.reborrow())).collect(),
-                bindings: self.arena.alloc_from_iter(
-                    constraints.iter().map(|b| self.lower_assoc_ty_constraint(b, itctx.reborrow())),
-                ),
-                parenthesized: false,
-            },
-            !has_non_lt_args && param_mode == ParamMode::Optional,
-        )
+        let args = data
+            .args
+            .iter()
+            .filter_map(|arg| match arg {
+                AngleBracketedArg::Arg(arg) => Some(self.lower_generic_arg(arg, itctx.reborrow())),
+                AngleBracketedArg::Constraint(_) => None,
+            })
+            .collect();
+        let bindings = self.arena.alloc_from_iter(data.args.iter().filter_map(|arg| match arg {
+            AngleBracketedArg::Constraint(c) => {
+                Some(self.lower_assoc_ty_constraint(c, itctx.reborrow()))
+            }
+            AngleBracketedArg::Arg(_) => None,
+        }));
+        let ctor = GenericArgsCtor { args, bindings, parenthesized: false };
+        (ctor, !has_non_lt_args && param_mode == ParamMode::Optional)
     }
 
     fn lower_parenthesized_parameter_data(

diff --git a/src/librustc_ast_passes/ast_validation.rs b/src/librustc_ast_passes/ast_validation.rs
index 885f2f939a653d2f060721c67de747cc933fff45..de7ae10723f4de482e993e1a9ee05aacf58fdb5d 100644 (file)
--- a/src/librustc_ast_passes/ast_validation.rs
+++ b/src/librustc_ast_passes/ast_validation.rs
@@ -639,6 +639,34 @@ fn deny_items(&self, trait_items: &[P<AssocItem>], ident_span: Span) {
             .emit();
         }
     }
+
+    /// Enforce generic args coming before constraints in `<...>` of a path segment.
+    fn check_generic_args_before_constraints(&self, data: &AngleBracketedArgs) {
+        // Early exit in case it's partitioned as it should be.
+        if data.args.iter().is_partitioned(|arg| matches!(arg, AngleBracketedArg::Arg(_))) {
+            return;
+        }
+        // Find all generic argument coming after the first constraint...
+        let mut misplaced_args = Vec::new();
+        let mut first = None;
+        for arg in &data.args {
+            match (arg, first) {
+                (AngleBracketedArg::Arg(a), Some(_)) => misplaced_args.push(a.span()),
+                (AngleBracketedArg::Constraint(c), None) => first = Some(c.span),
+                (AngleBracketedArg::Arg(_), None) | (AngleBracketedArg::Constraint(_), Some(_)) => {
+                }
+            }
+        }
+        // ...and then error:
+        self.err_handler()
+            .struct_span_err(
+                misplaced_args.clone(),
+                "generic arguments must come before the first constraint",
+            )
+            .span_label(first.unwrap(), "the first constraint is provided here")
+            .span_labels(misplaced_args, "generic argument")
+            .emit();
+    }
 }
 
 /// Checks that generic parameters are in the correct order,
@@ -1008,17 +1036,20 @@ fn visit_foreign_item(&mut self, fi: &'a ForeignItem) {
     fn visit_generic_args(&mut self, _: Span, generic_args: &'a GenericArgs) {
         match *generic_args {
             GenericArgs::AngleBracketed(ref data) => {
-                walk_list!(self, visit_generic_arg, &data.args);
-
-                // Type bindings such as `Item = impl Debug` in `Iterator<Item = Debug>`
-                // are allowed to contain nested `impl Trait`.
-                self.with_impl_trait(None, |this| {
-                    walk_list!(
-                        this,
-                        visit_assoc_ty_constraint_from_generic_args,
-                        &data.constraints
-                    );
-                });
+                self.check_generic_args_before_constraints(data);
+
+                for arg in &data.args {
+                    match arg {
+                        AngleBracketedArg::Arg(arg) => self.visit_generic_arg(arg),
+                        // Type bindings such as `Item = impl Debug` in `Iterator<Item = Debug>`
+                        // are allowed to contain nested `impl Trait`.
+                        AngleBracketedArg::Constraint(constraint) => {
+                            self.with_impl_trait(None, |this| {
+                                this.visit_assoc_ty_constraint_from_generic_args(constraint);
+                            });
+                        }
+                    }
+                }
             }
             GenericArgs::Parenthesized(ref data) => {
                 walk_list!(self, visit_ty, &data.inputs);

diff --git a/src/librustc_ast_passes/lib.rs b/src/librustc_ast_passes/lib.rs
index 10081d36754ba43f652593ee3ab13e91a69fdcf4..bfe304419801d5f2c7833170ceb6eaa5012ce23b 100644 (file)
--- a/src/librustc_ast_passes/lib.rs
+++ b/src/librustc_ast_passes/lib.rs
@@ -1,10 +1,12 @@
-#![feature(bindings_after_at)]
 //! The `rustc_ast_passes` crate contains passes which validate the AST in `syntax`
 //! parsed by `rustc_parse` and then lowered, after the passes in this crate,
 //! by `rustc_ast_lowering`.
 //!
 //! The crate also contains other misc AST visitors, e.g. `node_count` and `show_span`.
 
+#![feature(bindings_after_at)]
+#![feature(iter_is_partitioned)]
+
 pub mod ast_validation;
 pub mod feature_gate;
 pub mod node_count;

diff --git a/src/librustc_ast_pretty/pprust.rs b/src/librustc_ast_pretty/pprust.rs
index 2d6932ffbeedff3bae25e8906837162a165a9041..4aabbe7efbef456866710ed954d93109d96d219d 100644 (file)
--- a/src/librustc_ast_pretty/pprust.rs
+++ b/src/librustc_ast_pretty/pprust.rs
@@ -796,31 +796,10 @@ fn print_generic_args(&mut self, args: &ast::GenericArgs, colons_before_params:
         match *args {
             ast::GenericArgs::AngleBracketed(ref data) => {
                 self.s.word("<");
-
-                self.commasep(Inconsistent, &data.args, |s, generic_arg| {
-                    s.print_generic_arg(generic_arg)
+                self.commasep(Inconsistent, &data.args, |s, arg| match arg {
+                    ast::AngleBracketedArg::Arg(a) => s.print_generic_arg(a),
+                    ast::AngleBracketedArg::Constraint(c) => s.print_assoc_constraint(c),
                 });
-
-                let mut comma = !data.args.is_empty();
-
-                for constraint in data.constraints.iter() {
-                    if comma {
-                        self.word_space(",")
-                    }
-                    self.print_ident(constraint.ident);
-                    self.s.space();
-                    match constraint.kind {
-                        ast::AssocTyConstraintKind::Equality { ref ty } => {
-                            self.word_space("=");
-                            self.print_type(ty);
-                        }
-                        ast::AssocTyConstraintKind::Bound { ref bounds } => {
-                            self.print_type_bounds(":", &*bounds);
-                        }
-                    }
-                    comma = true;
-                }
-
                 self.s.word(">")
             }
 
@@ -891,6 +870,20 @@ pub fn print_opt_lifetime(&mut self, lifetime: &Option<ast::Lifetime>) {
         }
     }
 
+    fn print_assoc_constraint(&mut self, constraint: &ast::AssocTyConstraint) {
+        self.print_ident(constraint.ident);
+        self.s.space();
+        match &constraint.kind {
+            ast::AssocTyConstraintKind::Equality { ty } => {
+                self.word_space("=");
+                self.print_type(ty);
+            }
+            ast::AssocTyConstraintKind::Bound { bounds } => {
+                self.print_type_bounds(":", &*bounds);
+            }
+        }
+    }
+
     crate fn print_generic_arg(&mut self, generic_arg: &GenericArg) {
         match generic_arg {
             GenericArg::Lifetime(lt) => self.print_lifetime(*lt),

diff --git a/src/librustc_codegen_ssa/back/link.rs b/src/librustc_codegen_ssa/back/link.rs
index 1fb08a058034dddbf135b735097e9f7fc45864bd..7169b79c3bc36bcbc02b850589fbf82daf70e45d 100644 (file)
--- a/src/librustc_codegen_ssa/back/link.rs
+++ b/src/librustc_codegen_ssa/back/link.rs
@@ -1391,10 +1391,17 @@ fn link_args<'a, B: ArchiveBuilder<'a>>(
     // link line. And finally upstream native libraries can't depend on anything
     // in this DAG so far because they're only dylibs and dylibs can only depend
     // on other dylibs (e.g., other native deps).
-    add_local_native_libraries(cmd, sess, codegen_results);
+    //
+    // If -Zlink-native-libraries=false is set, then the assumption is that an
+    // external build system already has the native dependencies defined, and it
+    // will provide them to the linker itself.
+    if sess.opts.debugging_opts.link_native_libraries.unwrap_or(true) {
+        add_local_native_libraries(cmd, sess, codegen_results);
+    }
     add_upstream_rust_crates::<B>(cmd, sess, codegen_results, crate_type, tmpdir);
-    add_upstream_native_libraries(cmd, sess, codegen_results, crate_type);
-
+    if sess.opts.debugging_opts.link_native_libraries.unwrap_or(true) {
+        add_upstream_native_libraries(cmd, sess, codegen_results, crate_type);
+    }
     // Tell the linker what we're doing.
     if crate_type != config::CrateType::Executable {
         cmd.build_dylib(out_filename);

diff --git a/src/librustc_error_codes/error_codes.rs b/src/librustc_error_codes/error_codes.rs
index fe68e9eacbfdae2436a7216e07b2d4479f9bfdb4..2f0a3fc1d1c386817112c4e17db4b087af0e62c3 100644 (file)
--- a/src/librustc_error_codes/error_codes.rs
+++ b/src/librustc_error_codes/error_codes.rs
@@ -389,6 +389,7 @@
 E0699: include_str!("./error_codes/E0699.md"),
 E0700: include_str!("./error_codes/E0700.md"),
 E0701: include_str!("./error_codes/E0701.md"),
+E0703: include_str!("./error_codes/E0703.md"),
 E0704: include_str!("./error_codes/E0704.md"),
 E0705: include_str!("./error_codes/E0705.md"),
 E0706: include_str!("./error_codes/E0706.md"),
@@ -603,7 +604,6 @@
 //  E0694, // an unknown tool name found in scoped attributes
     E0696, // `continue` pointing to a labeled block
 //  E0702, // replaced with a generic attribute input check
-    E0703, // invalid ABI
 //  E0707, // multiple elided lifetimes used in arguments of `async fn`
     E0708, // `async` non-`move` closures with parameters are not currently
            // supported

diff --git a/src/librustc_error_codes/error_codes/E0703.md b/src/librustc_error_codes/error_codes/E0703.md
new file mode 100644 (file)
index 0000000..b42677d
--- /dev/null
+++ b/src/librustc_error_codes/error_codes/E0703.md
@@ -0,0 +1,17 @@
+Invalid ABI (Application Binary Interface) used in the code.
+
+Erroneous code example:
+
+```compile_fail,E0703
+extern "invalid" fn foo() {} // error!
+# fn main() {}
+```
+
+At present few predefined ABI's (like Rust, C, system, etc.) can be
+used in Rust. Verify that the ABI is predefined. For example you can
+replace the given ABI from 'Rust'.
+
+```
+extern "Rust" fn foo() {} // ok!
+# fn main() { }
+```

diff --git a/src/librustc_expand/build.rs b/src/librustc_expand/build.rs
index 48ceaf5fccd8efb089a57e88dfa324ec8ebd0295..4d89bf79e7c67e16d2171ee3796cda2e3f31a707 100644 (file)
--- a/src/librustc_expand/build.rs
+++ b/src/librustc_expand/build.rs
@@ -36,7 +36,8 @@ pub fn path_all(
             idents.into_iter().map(|ident| ast::PathSegment::from_ident(ident.with_span_pos(span))),
         );
         let args = if !args.is_empty() {
-            ast::AngleBracketedArgs { args, constraints: Vec::new(), span }.into()
+            let args = args.into_iter().map(ast::AngleBracketedArg::Arg).collect();
+            ast::AngleBracketedArgs { args, span }.into()
         } else {
             None
         };

diff --git a/src/librustc_interface/util.rs b/src/librustc_interface/util.rs
index c6f2d1b82fcf484da7c4cc4293d01106edd03d11..eff7dacef7953f21b0ae753d51d8990b27f51985 100644 (file)
--- a/src/librustc_interface/util.rs
+++ b/src/librustc_interface/util.rs
@@ -634,17 +634,19 @@ fn involves_impl_trait(ty: &ast::Ty) -> bool {
                         match seg.args.as_ref().map(|generic_arg| &**generic_arg) {
                             None => false,
                             Some(&ast::GenericArgs::AngleBracketed(ref data)) => {
-                                let types = data.args.iter().filter_map(|arg| match arg {
-                                    ast::GenericArg::Type(ty) => Some(ty),
-                                    _ => None,
-                                });
-                                any_involves_impl_trait(types)
-                                    || data.constraints.iter().any(|c| match c.kind {
+                                data.args.iter().any(|arg| match arg {
+                                    ast::AngleBracketedArg::Arg(arg) => match arg {
+                                        ast::GenericArg::Type(ty) => involves_impl_trait(ty),
+                                        ast::GenericArg::Lifetime(_)
+                                        | ast::GenericArg::Const(_) => false,
+                                    },
+                                    ast::AngleBracketedArg::Constraint(c) => match c.kind {
                                         ast::AssocTyConstraintKind::Bound { .. } => true,
                                         ast::AssocTyConstraintKind::Equality { ref ty } => {
                                             involves_impl_trait(ty)
                                         }
-                                    })
+                                    },
+                                })
                             }
                             Some(&ast::GenericArgs::Parenthesized(ref data)) => {
                                 any_involves_impl_trait(data.inputs.iter())

diff --git a/src/librustc_parse/parser/path.rs b/src/librustc_parse/parser/path.rs
index f88b4fe6ff0a8dfedec357ab9a8802e6797599ce..9fa7bc027b87800379b3a0cad54b3dba6a897603 100644 (file)
--- a/src/librustc_parse/parser/path.rs
+++ b/src/librustc_parse/parser/path.rs
@@ -1,12 +1,10 @@
 use super::ty::{AllowPlus, RecoverQPath};
 use super::{Parser, TokenType};
 use crate::maybe_whole;
-use rustc_ast::ast::{
-    self, AngleBracketedArgs, Ident, ParenthesizedArgs, Path, PathSegment, QSelf,
-};
-use rustc_ast::ast::{
-    AnonConst, AssocTyConstraint, AssocTyConstraintKind, BlockCheckMode, GenericArg,
-};
+use rustc_ast::ast::{self, AngleBracketedArg, AngleBracketedArgs, GenericArg, ParenthesizedArgs};
+use rustc_ast::ast::{AnonConst, AssocTyConstraint, AssocTyConstraintKind, BlockCheckMode};
+use rustc_ast::ast::{Ident, Path, PathSegment, QSelf};
+use rustc_ast::ptr::P;
 use rustc_ast::token::{self, Token};
 use rustc_errors::{pluralize, Applicability, PResult};
 use rustc_span::source_map::{BytePos, Span};
@@ -218,11 +216,11 @@ pub(super) fn parse_path_segment(&mut self, style: PathStyle) -> PResult<'a, Pat
                 let lo = self.token.span;
                 let args = if self.eat_lt() {
                     // `<'a, T, A = U>`
-                    let (args, constraints) =
-                        self.parse_generic_args_with_leading_angle_bracket_recovery(style, lo)?;
+                    let args =
+                        self.parse_angle_args_with_leading_angle_bracket_recovery(style, lo)?;
                     self.expect_gt()?;
                     let span = lo.to(self.prev_token.span);
-                    AngleBracketedArgs { args, constraints, span }.into()
+                    AngleBracketedArgs { args, span }.into()
                 } else {
                     // `(T, U) -> R`
                     let (inputs, _) = self.parse_paren_comma_seq(|p| p.parse_ty())?;
@@ -251,18 +249,18 @@ pub(super) fn parse_path_segment_ident(&mut self) -> PResult<'a, Ident> {
 
     /// Parses generic args (within a path segment) with recovery for extra leading angle brackets.
     /// For the purposes of understanding the parsing logic of generic arguments, this function
-    /// can be thought of being the same as just calling `self.parse_generic_args()` if the source
+    /// can be thought of being the same as just calling `self.parse_angle_args()` if the source
     /// had the correct amount of leading angle brackets.
     ///
     /// ```ignore (diagnostics)
     /// bar::<<<<T as Foo>::Output>();
     ///      ^^ help: remove extra angle brackets
     /// ```
-    fn parse_generic_args_with_leading_angle_bracket_recovery(
+    fn parse_angle_args_with_leading_angle_bracket_recovery(
         &mut self,
         style: PathStyle,
         lo: Span,
-    ) -> PResult<'a, (Vec<GenericArg>, Vec<AssocTyConstraint>)> {
+    ) -> PResult<'a, Vec<AngleBracketedArg>> {
         // We need to detect whether there are extra leading left angle brackets and produce an
         // appropriate error and suggestion. This cannot be implemented by looking ahead at
         // upcoming tokens for a matching `>` character - if there are unmatched `<` tokens
@@ -337,8 +335,8 @@ fn parse_generic_args_with_leading_angle_bracket_recovery(
         let snapshot = if is_first_invocation { Some(self.clone()) } else { None };
 
         debug!("parse_generic_args_with_leading_angle_bracket_recovery: (snapshotting)");
-        match self.parse_generic_args() {
-            Ok(value) => Ok(value),
+        match self.parse_angle_args() {
+            Ok(args) => Ok(args),
             Err(ref mut e) if is_first_invocation && self.unmatched_angle_bracket_count > 0 => {
                 // Cancel error from being unable to find `>`. We know the error
                 // must have been this due to a non-zero unmatched angle bracket
@@ -381,110 +379,136 @@ fn parse_generic_args_with_leading_angle_bracket_recovery(
                 .emit();
 
                 // Try again without unmatched angle bracket characters.
-                self.parse_generic_args()
+                self.parse_angle_args()
             }
             Err(e) => Err(e),
         }
     }
 
-    /// Parses (possibly empty) list of lifetime and type arguments and associated type bindings,
+    /// Parses (possibly empty) list of generic arguments / associated item constraints,
     /// possibly including trailing comma.
-    fn parse_generic_args(&mut self) -> PResult<'a, (Vec<GenericArg>, Vec<AssocTyConstraint>)> {
+    fn parse_angle_args(&mut self) -> PResult<'a, Vec<AngleBracketedArg>> {
         let mut args = Vec::new();
-        let mut constraints = Vec::new();
-        let mut misplaced_assoc_ty_constraints: Vec<Span> = Vec::new();
-        let mut assoc_ty_constraints: Vec<Span> = Vec::new();
-
-        let args_lo = self.token.span;
-
-        loop {
-            if self.check_lifetime() && self.look_ahead(1, |t| !t.is_like_plus()) {
-                // Parse lifetime argument.
-                args.push(GenericArg::Lifetime(self.expect_lifetime()));
-                misplaced_assoc_ty_constraints.append(&mut assoc_ty_constraints);
-            } else if self.check_ident()
-                && self.look_ahead(1, |t| t == &token::Eq || t == &token::Colon)
-            {
-                // Parse associated type constraint.
-                let lo = self.token.span;
-                let ident = self.parse_ident()?;
-                let kind = if self.eat(&token::Eq) {
-                    AssocTyConstraintKind::Equality { ty: self.parse_ty()? }
-                } else if self.eat(&token::Colon) {
-                    AssocTyConstraintKind::Bound {
-                        bounds: self.parse_generic_bounds(Some(self.prev_token.span))?,
-                    }
-                } else {
-                    unreachable!();
-                };
+        while let Some(arg) = self.parse_angle_arg()? {
+            args.push(arg);
+            if !self.eat(&token::Comma) {
+                break;
+            }
+        }
+        Ok(args)
+    }
 
-                let span = lo.to(self.prev_token.span);
+    /// Parses a single argument in the angle arguments `<...>` of a path segment.
+    fn parse_angle_arg(&mut self) -> PResult<'a, Option<AngleBracketedArg>> {
+        if self.check_ident() && self.look_ahead(1, |t| matches!(t.kind, token::Eq | token::Colon))
+        {
+            // Parse associated type constraint.
+            let lo = self.token.span;
+            let ident = self.parse_ident()?;
+            let kind = if self.eat(&token::Eq) {
+                let ty = self.parse_assoc_equality_term(ident, self.prev_token.span)?;
+                AssocTyConstraintKind::Equality { ty }
+            } else if self.eat(&token::Colon) {
+                let bounds = self.parse_generic_bounds(Some(self.prev_token.span))?;
+                AssocTyConstraintKind::Bound { bounds }
+            } else {
+                unreachable!();
+            };
 
-                // Gate associated type bounds, e.g., `Iterator<Item: Ord>`.
-                if let AssocTyConstraintKind::Bound { .. } = kind {
-                    self.sess.gated_spans.gate(sym::associated_type_bounds, span);
-                }
+            let span = lo.to(self.prev_token.span);
 
-                constraints.push(AssocTyConstraint { id: ast::DUMMY_NODE_ID, ident, kind, span });
-                assoc_ty_constraints.push(span);
-            } else if self.check_const_arg() {
-                // Parse const argument.
-                let expr = if let token::OpenDelim(token::Brace) = self.token.kind {
-                    self.parse_block_expr(
-                        None,
-                        self.token.span,
-                        BlockCheckMode::Default,
-                        ast::AttrVec::new(),
-                    )?
-                } else if self.token.is_ident() {
-                    // FIXME(const_generics): to distinguish between idents for types and consts,
-                    // we should introduce a GenericArg::Ident in the AST and distinguish when
-                    // lowering to the HIR. For now, idents for const args are not permitted.
-                    if self.token.is_bool_lit() {
-                        self.parse_literal_maybe_minus()?
-                    } else {
-                        let span = self.token.span;
-                        let msg = "identifiers may currently not be used for const generics";
-                        self.struct_span_err(span, msg).emit();
-                        let block = self.mk_block_err(span);
-                        self.mk_expr(span, ast::ExprKind::Block(block, None), ast::AttrVec::new())
-                    }
-                } else {
-                    self.parse_literal_maybe_minus()?
-                };
-                let value = AnonConst { id: ast::DUMMY_NODE_ID, value: expr };
-                args.push(GenericArg::Const(value));
-                misplaced_assoc_ty_constraints.append(&mut assoc_ty_constraints);
-            } else if self.check_type() {
-                // Parse type argument.
-                args.push(GenericArg::Type(self.parse_ty()?));
-                misplaced_assoc_ty_constraints.append(&mut assoc_ty_constraints);
-            } else {
-                break;
+            // Gate associated type bounds, e.g., `Iterator<Item: Ord>`.
+            if let AssocTyConstraintKind::Bound { .. } = kind {
+                self.sess.gated_spans.gate(sym::associated_type_bounds, span);
             }
 
-            if !self.eat(&token::Comma) {
-                break;
-            }
+            let constraint = AssocTyConstraint { id: ast::DUMMY_NODE_ID, ident, kind, span };
+            Ok(Some(AngleBracketedArg::Constraint(constraint)))
+        } else {
+            Ok(self.parse_generic_arg()?.map(AngleBracketedArg::Arg))
         }
+    }
 
-        // FIXME: we would like to report this in ast_validation instead, but we currently do not
-        // preserve ordering of generic parameters with respect to associated type binding, so we
-        // lose that information after parsing.
-        if !misplaced_assoc_ty_constraints.is_empty() {
-            let mut err = self.struct_span_err(
-                args_lo.to(self.prev_token.span),
-                "associated type bindings must be declared after generic parameters",
-            );
-            for span in misplaced_assoc_ty_constraints {
-                err.span_label(
-                    span,
-                    "this associated type binding should be moved after the generic parameters",
-                );
+    /// Parse the term to the right of an associated item equality constraint.
+    /// That is, parse `<term>` in `Item = <term>`.
+    /// Right now, this only admits types in `<term>`.
+    fn parse_assoc_equality_term(&mut self, ident: Ident, eq: Span) -> PResult<'a, P<ast::Ty>> {
+        let arg = self.parse_generic_arg()?;
+        let span = ident.span.to(self.prev_token.span);
+        match arg {
+            Some(GenericArg::Type(ty)) => return Ok(ty),
+            Some(GenericArg::Const(expr)) => {
+                self.struct_span_err(span, "cannot constrain an associated constant to a value")
+                    .span_label(ident.span, "this associated constant...")
+                    .span_label(expr.value.span, "...cannot be constrained to this value")
+                    .emit();
+            }
+            Some(GenericArg::Lifetime(lt)) => {
+                self.struct_span_err(span, "associated lifetimes are not supported")
+                    .span_label(lt.ident.span, "the lifetime is given here")
+                    .help("if you meant to specify a trait object, write `dyn Trait + 'lifetime`")
+                    .emit();
+            }
+            None => {
+                let after_eq = eq.shrink_to_hi();
+                let before_next = self.token.span.shrink_to_lo();
+                self.struct_span_err(after_eq.to(before_next), "missing type to the right of `=`")
+                    .span_suggestion(
+                        self.sess.source_map().next_point(eq).to(before_next),
+                        "to constrain the associated type, add a type after `=`",
+                        " TheType".to_string(),
+                        Applicability::HasPlaceholders,
+                    )
+                    .span_suggestion(
+                        eq.to(before_next),
+                        &format!("remove the `=` if `{}` is a type", ident),
+                        String::new(),
+                        Applicability::MaybeIncorrect,
+                    )
+                    .emit();
             }
-            err.emit();
         }
+        Ok(self.mk_ty(span, ast::TyKind::Err))
+    }
 
-        Ok((args, constraints))
+    /// Parse a generic argument in a path segment.
+    /// This does not include constraints, e.g., `Item = u8`, which is handled in `parse_angle_arg`.
+    fn parse_generic_arg(&mut self) -> PResult<'a, Option<GenericArg>> {
+        let arg = if self.check_lifetime() && self.look_ahead(1, |t| !t.is_like_plus()) {
+            // Parse lifetime argument.
+            GenericArg::Lifetime(self.expect_lifetime())
+        } else if self.check_const_arg() {
+            // Parse const argument.
+            let expr = if let token::OpenDelim(token::Brace) = self.token.kind {
+                self.parse_block_expr(
+                    None,
+                    self.token.span,
+                    BlockCheckMode::Default,
+                    ast::AttrVec::new(),
+                )?
+            } else if self.token.is_ident() {
+                // FIXME(const_generics): to distinguish between idents for types and consts,
+                // we should introduce a GenericArg::Ident in the AST and distinguish when
+                // lowering to the HIR. For now, idents for const args are not permitted.
+                if self.token.is_bool_lit() {
+                    self.parse_literal_maybe_minus()?
+                } else {
+                    let span = self.token.span;
+                    let msg = "identifiers may currently not be used for const generics";
+                    self.struct_span_err(span, msg).emit();
+                    let block = self.mk_block_err(span);
+                    self.mk_expr(span, ast::ExprKind::Block(block, None), ast::AttrVec::new())
+                }
+            } else {
+                self.parse_literal_maybe_minus()?
+            };
+            GenericArg::Const(AnonConst { id: ast::DUMMY_NODE_ID, value: expr })
+        } else if self.check_type() {
+            // Parse type argument.
+            GenericArg::Type(self.parse_ty()?)
+        } else {
+            return Ok(None);
+        };
+        Ok(Some(arg))
     }
 }

diff --git a/src/librustc_save_analysis/dump_visitor.rs b/src/librustc_save_analysis/dump_visitor.rs
index c0ac9e7f6b61ff1b62540960a16705a84aad9733..e5637e4e6e694f3ef624904eaba7d840d964dfc1 100644 (file)
--- a/src/librustc_save_analysis/dump_visitor.rs
+++ b/src/librustc_save_analysis/dump_visitor.rs
@@ -783,7 +783,7 @@ fn process_path(&mut self, id: NodeId, path: &'l ast::Path) {
                 match **generic_args {
                     ast::GenericArgs::AngleBracketed(ref data) => {
                         for arg in &data.args {
-                            if let ast::GenericArg::Type(ty) = arg {
+                            if let ast::AngleBracketedArg::Arg(ast::GenericArg::Type(ty)) = arg {
                                 self.visit_ty(ty);
                             }
                         }
@@ -849,7 +849,7 @@ fn process_method_call(
             if let ast::GenericArgs::AngleBracketed(ref data) = **generic_args {
                 for arg in &data.args {
                     match arg {
-                        ast::GenericArg::Type(ty) => self.visit_ty(ty),
+                        ast::AngleBracketedArg::Arg(ast::GenericArg::Type(ty)) => self.visit_ty(ty),
                         _ => {}
                     }
                 }

diff --git a/src/librustc_session/options.rs b/src/librustc_session/options.rs
index 9d49075ebbc8e81043d23dcb926038cf298af9b8..e43908a79143b5f63464676a2ac34a409d5a6205 100644 (file)
--- a/src/librustc_session/options.rs
+++ b/src/librustc_session/options.rs
@@ -957,4 +957,6 @@ fn parse_symbol_mangling_version(
         "link the `.rlink` file generated by `-Z no-link`"),
     new_llvm_pass_manager: Option<bool> = (None, parse_opt_bool, [TRACKED],
         "use new LLVM pass manager"),
+    link_native_libraries: Option<bool> = (None, parse_opt_bool, [UNTRACKED],
+        "Link native libraries in the linker invocation."),
 }

diff --git a/src/librustdoc/markdown.rs b/src/librustdoc/markdown.rs
index a41fdd2ff17af6cee98ee19915fee4073e38bfcb..0a1b5f588158f5eb6ab5f669c7197cbae6ee0862 100644 (file)
--- a/src/librustdoc/markdown.rs
+++ b/src/librustdoc/markdown.rs
@@ -1,4 +1,4 @@
-use std::fs::File;
+use std::fs::{create_dir_all, File};
 use std::io::prelude::*;
 use std::path::PathBuf;
 
@@ -40,6 +40,11 @@ pub fn render(
     diag: &rustc_errors::Handler,
     edition: Edition,
 ) -> i32 {
+    if let Err(e) = create_dir_all(&options.output) {
+        diag.struct_err(&format!("{}: {}", options.output.display(), e)).emit();
+        return 4;
+    }
+
     let mut output = options.output;
     output.push(input.file_name().unwrap());
     output.set_extension("html");

diff --git a/src/test/ui/codemap_tests/unicode.stderr b/src/test/ui/codemap_tests/unicode.stderr
index fc2a0b5950e7dd8c42e79863062d7b907f3a7726..01d54ac8cc8a2bc23f711c5c29031d349d1c8038 100644 (file)
--- a/src/test/ui/codemap_tests/unicode.stderr
+++ b/src/test/ui/codemap_tests/unicode.stderr
@@ -8,3 +8,4 @@ LL | extern "路濫狼á́́" fn foo() {}
 
 error: aborting due to previous error
 
+For more information about this error, try `rustc --explain E0703`.

diff --git a/src/test/ui/issues/issue-70093.rs b/src/test/ui/issues/issue-70093.rs
new file mode 100644 (file)
index 0000000..95ab86e
--- /dev/null
+++ b/src/test/ui/issues/issue-70093.rs
@@ -0,0 +1,8 @@
+// run-pass
+// compile-flags: -Zlink-native-libraries=no -Cdefault-linker-libraries=yes
+// ignore-windows - this will probably only work on unixish systems
+
+#[link(name = "some-random-non-existent-library", kind = "static")]
+extern "C" {}
+
+fn main() {}

diff --git a/src/test/ui/parser/constraints-before-generic-args-syntactic-pass.rs b/src/test/ui/parser/constraints-before-generic-args-syntactic-pass.rs
new file mode 100644 (file)
index 0000000..afbd13e
--- /dev/null
+++ b/src/test/ui/parser/constraints-before-generic-args-syntactic-pass.rs
@@ -0,0 +1,9 @@
+// check-pass
+
+#[cfg(FALSE)]
+fn syntax() {
+    foo::<T = u8, T: Ord, String>();
+    foo::<T = u8, 'a, T: Ord>();
+}
+
+fn main() {}

diff --git a/src/test/ui/parser/issue-32214.rs b/src/test/ui/parser/issue-32214.rs
index 82f7ce62b9457804dc80c7c30a9c6d942ec3aaa2..1379eeb58e6e861c87fc0c175c415739ad097d9d 100644 (file)
--- a/src/test/ui/parser/issue-32214.rs
+++ b/src/test/ui/parser/issue-32214.rs
@@ -1,6 +1,6 @@
 trait Trait<T> { type Item; }
 
 pub fn test<W, I: Trait<Item=(), W> >() {}
-//~^ ERROR associated type bindings must be declared after generic parameters
+//~^ ERROR generic arguments must come before the first constraint
 
 fn main() { }

diff --git a/src/test/ui/parser/issue-32214.stderr b/src/test/ui/parser/issue-32214.stderr
index 08b230a14f50ebc8ba6213b4fb9997548a029da4..742f4fdc38bbdd40ccb659c6ea1524b9c77f6624 100644 (file)
--- a/src/test/ui/parser/issue-32214.stderr
+++ b/src/test/ui/parser/issue-32214.stderr
@@ -1,10 +1,10 @@
-error: associated type bindings must be declared after generic parameters
-  --> $DIR/issue-32214.rs:3:25
+error: generic arguments must come before the first constraint
+  --> $DIR/issue-32214.rs:3:34
    |
 LL | pub fn test<W, I: Trait<Item=(), W> >() {}
-   |                         -------^^^
+   |                         -------  ^ generic argument
    |                         |
-   |                         this associated type binding should be moved after the generic parameters
+   |                         the first constraint is provided here
 
 error: aborting due to previous error
 

diff --git a/src/test/ui/parser/issue-8537.stderr b/src/test/ui/parser/issue-8537.stderr
index b20226f87e8f8e1f761b823b2a73d7506b3e6793..a0793d94653da33978cd3e8259738cf10558ecc1 100644 (file)
--- a/src/test/ui/parser/issue-8537.stderr
+++ b/src/test/ui/parser/issue-8537.stderr
@@ -8,3 +8,4 @@ LL |   "invalid-ab_isize"
 
 error: aborting due to previous error
 
+For more information about this error, try `rustc --explain E0703`.

diff --git a/src/test/ui/parser/recover-assoc-const-constraint.rs b/src/test/ui/parser/recover-assoc-const-constraint.rs
new file mode 100644 (file)
index 0000000..06be3cd
--- /dev/null
+++ b/src/test/ui/parser/recover-assoc-const-constraint.rs
@@ -0,0 +1,7 @@
+#[cfg(FALSE)]
+fn syntax() {
+    bar::<Item = 42>(); //~ ERROR cannot constrain an associated constant to a value
+    bar::<Item = { 42 }>(); //~ ERROR cannot constrain an associated constant to a value
+}
+
+fn main() {}

diff --git a/src/test/ui/parser/recover-assoc-const-constraint.stderr b/src/test/ui/parser/recover-assoc-const-constraint.stderr
new file mode 100644 (file)
index 0000000..c6733b3
--- /dev/null
+++ b/src/test/ui/parser/recover-assoc-const-constraint.stderr
@@ -0,0 +1,20 @@
+error: cannot constrain an associated constant to a value
+  --> $DIR/recover-assoc-const-constraint.rs:3:11
+   |
+LL |     bar::<Item = 42>();
+   |           ----^^^--
+   |           |      |
+   |           |      ...cannot be constrained to this value
+   |           this associated constant...
+
+error: cannot constrain an associated constant to a value
+  --> $DIR/recover-assoc-const-constraint.rs:4:11
+   |
+LL |     bar::<Item = { 42 }>();
+   |           ----^^^------
+   |           |      |
+   |           |      ...cannot be constrained to this value
+   |           this associated constant...
+
+error: aborting due to 2 previous errors
+

diff --git a/src/test/ui/parser/recover-assoc-eq-missing-term.rs b/src/test/ui/parser/recover-assoc-eq-missing-term.rs
new file mode 100644 (file)
index 0000000..4b42c44
--- /dev/null
+++ b/src/test/ui/parser/recover-assoc-eq-missing-term.rs
@@ -0,0 +1,6 @@
+#[cfg(FALSE)]
+fn syntax() {
+    bar::<Item =   >(); //~ ERROR missing type to the right of `=`
+}
+
+fn main() {}

diff --git a/src/test/ui/parser/recover-assoc-eq-missing-term.stderr b/src/test/ui/parser/recover-assoc-eq-missing-term.stderr
new file mode 100644 (file)
index 0000000..6e41e13
--- /dev/null
+++ b/src/test/ui/parser/recover-assoc-eq-missing-term.stderr
@@ -0,0 +1,17 @@
+error: missing type to the right of `=`
+  --> $DIR/recover-assoc-eq-missing-term.rs:3:17
+   |
+LL |     bar::<Item =   >();
+   |                 ^^^
+   |
+help: to constrain the associated type, add a type after `=`
+   |
+LL |     bar::<Item = TheType>();
+   |                  ^^^^^^^
+help: remove the `=` if `Item` is a type
+   |
+LL |     bar::<Item >();
+   |               --
+
+error: aborting due to previous error
+

diff --git a/src/test/ui/parser/recover-assoc-lifetime-constraint.rs b/src/test/ui/parser/recover-assoc-lifetime-constraint.rs
new file mode 100644 (file)
index 0000000..558fcdf
--- /dev/null
+++ b/src/test/ui/parser/recover-assoc-lifetime-constraint.rs
@@ -0,0 +1,6 @@
+#[cfg(FALSE)]
+fn syntax() {
+    bar::<Item = 'a>(); //~ ERROR associated lifetimes are not supported
+}
+
+fn main() {}

diff --git a/src/test/ui/parser/recover-assoc-lifetime-constraint.stderr b/src/test/ui/parser/recover-assoc-lifetime-constraint.stderr
new file mode 100644 (file)
index 0000000..7943753
--- /dev/null
+++ b/src/test/ui/parser/recover-assoc-lifetime-constraint.stderr
@@ -0,0 +1,12 @@
+error: associated lifetimes are not supported
+  --> $DIR/recover-assoc-lifetime-constraint.rs:3:11
+   |
+LL |     bar::<Item = 'a>();
+   |           ^^^^^^^--
+   |                  |
+   |                  the lifetime is given here
+   |
+   = help: if you meant to specify a trait object, write `dyn Trait + 'lifetime`
+
+error: aborting due to previous error
+

diff --git a/src/test/ui/suggestions/suggest-move-types.rs b/src/test/ui/suggestions/suggest-move-types.rs
index 6505a97de6e4bfcbb8c556fd01d9476b133e0be5..27930626a6209c52f601e5b316d65229a8bbb5bb 100644 (file)
--- a/src/test/ui/suggestions/suggest-move-types.rs
+++ b/src/test/ui/suggestions/suggest-move-types.rs
@@ -1,5 +1,3 @@
-// ignore-tidy-linelength
-
 #![allow(warnings)]
 
 // This test verifies that the suggestion to move types before associated type bindings
@@ -25,20 +23,22 @@ trait ThreeWithLifetime<'a, 'b, 'c, T, U, V> {
   type C;
 }
 
-struct A<T, M: One<A=(), T>> { //~ ERROR associated type bindings must be declared after generic parameters
+struct A<T, M: One<A=(), T>> {
+//~^ ERROR generic arguments must come before the first constraint
     m: M,
     t: T,
 }
 
 
 struct Al<'a, T, M: OneWithLifetime<A=(), T, 'a>> {
-//~^ ERROR associated type bindings must be declared after generic parameters
+//~^ ERROR generic arguments must come before the first constraint
 //~^^ ERROR type provided when a lifetime was expected
     m: M,
     t: &'a T,
 }
 
-struct B<T, U, V, M: Three<A=(), B=(), C=(), T, U, V>> { //~ ERROR associated type bindings must be declared after generic parameters
+struct B<T, U, V, M: Three<A=(), B=(), C=(), T, U, V>> {
+//~^ ERROR generic arguments must come before the first constraint
     m: M,
     t: T,
     u: U,
@@ -46,7 +46,7 @@ struct B<T, U, V, M: Three<A=(), B=(), C=(), T, U, V>> { //~ ERROR associated ty
 }
 
 struct Bl<'a, 'b, 'c, T, U, V, M: ThreeWithLifetime<A=(), B=(), C=(), T, U, V, 'a, 'b, 'c>> {
-//~^ ERROR associated type bindings must be declared after generic parameters
+//~^ ERROR generic arguments must come before the first constraint
 //~^^ ERROR type provided when a lifetime was expected
     m: M,
     t: &'a T,
@@ -54,7 +54,8 @@ struct Bl<'a, 'b, 'c, T, U, V, M: ThreeWithLifetime<A=(), B=(), C=(), T, U, V, '
     v: &'c V,
 }
 
-struct C<T, U, V, M: Three<T, A=(), B=(), C=(), U, V>> { //~ ERROR associated type bindings must be declared after generic parameters
+struct C<T, U, V, M: Three<T, A=(), B=(), C=(), U, V>> {
+//~^ ERROR generic arguments must come before the first constraint
     m: M,
     t: T,
     u: U,
@@ -62,7 +63,7 @@ struct C<T, U, V, M: Three<T, A=(), B=(), C=(), U, V>> { //~ ERROR associated ty
 }
 
 struct Cl<'a, 'b, 'c, T, U, V, M: ThreeWithLifetime<T, 'a, A=(), B=(), C=(), U, 'b, V, 'c>> {
-//~^ ERROR associated type bindings must be declared after generic parameters
+//~^ ERROR generic arguments must come before the first constraint
 //~^^ ERROR lifetime provided when a type was expected
     m: M,
     t: &'a T,
@@ -70,7 +71,8 @@ struct Cl<'a, 'b, 'c, T, U, V, M: ThreeWithLifetime<T, 'a, A=(), B=(), C=(), U,
     v: &'c V,
 }
 
-struct D<T, U, V, M: Three<T, A=(), B=(), U, C=(), V>> { //~ ERROR associated type bindings must be declared after generic parameters
+struct D<T, U, V, M: Three<T, A=(), B=(), U, C=(), V>> {
+//~^ ERROR generic arguments must come before the first constraint
     m: M,
     t: T,
     u: U,
@@ -78,7 +80,7 @@ struct D<T, U, V, M: Three<T, A=(), B=(), U, C=(), V>> { //~ ERROR associated ty
 }
 
 struct Dl<'a, 'b, 'c, T, U, V, M: ThreeWithLifetime<T, 'a, A=(), B=(), U, 'b, C=(), V, 'c>> {
-//~^ ERROR associated type bindings must be declared after generic parameters
+//~^ ERROR generic arguments must come before the first constraint
 //~^^ ERROR lifetime provided when a type was expected
     m: M,
     t: &'a T,

diff --git a/src/test/ui/suggestions/suggest-move-types.stderr b/src/test/ui/suggestions/suggest-move-types.stderr
index ac91813f928396bcd5f9b7da113d02464b009520..4dd0613757a959bddbf352dea2aaf352abd826f5 100644 (file)
--- a/src/test/ui/suggestions/suggest-move-types.stderr
+++ b/src/test/ui/suggestions/suggest-move-types.stderr
@@ -1,81 +1,85 @@
-error: associated type bindings must be declared after generic parameters
-  --> $DIR/suggest-move-types.rs:28:20
+error: generic arguments must come before the first constraint
+  --> $DIR/suggest-move-types.rs:26:26
    |
 LL | struct A<T, M: One<A=(), T>> {
-   |                    ----^^^
+   |                    ----  ^ generic argument
    |                    |
-   |                    this associated type binding should be moved after the generic parameters
+   |                    the first constraint is provided here
 
-error: associated type bindings must be declared after generic parameters
-  --> $DIR/suggest-move-types.rs:34:37
+error: generic arguments must come before the first constraint
+  --> $DIR/suggest-move-types.rs:33:43
    |
 LL | struct Al<'a, T, M: OneWithLifetime<A=(), T, 'a>> {
-   |                                     ----^^^^^^^
-   |                                     |
-   |                                     this associated type binding should be moved after the generic parameters
+   |                                     ----  ^  ^^ generic argument
+   |                                     |     |
+   |                                     |     generic argument
+   |                                     the first constraint is provided here
 
-error: associated type bindings must be declared after generic parameters
-  --> $DIR/suggest-move-types.rs:41:28
+error: generic arguments must come before the first constraint
+  --> $DIR/suggest-move-types.rs:40:46
    |
 LL | struct B<T, U, V, M: Three<A=(), B=(), C=(), T, U, V>> {
-   |                            ----^^----^^----^^^^^^^^^
-   |                            |     |     |
-   |                            |     |     this associated type binding should be moved after the generic parameters
-   |                            |     this associated type binding should be moved after the generic parameters
-   |                            this associated type binding should be moved after the generic parameters
+   |                            ----              ^  ^  ^ generic argument
+   |                            |                 |  |
+   |                            |                 |  generic argument
+   |                            |                 generic argument
+   |                            the first constraint is provided here
 
-error: associated type bindings must be declared after generic parameters
-  --> $DIR/suggest-move-types.rs:48:53
+error: generic arguments must come before the first constraint
+  --> $DIR/suggest-move-types.rs:48:71
    |
 LL | struct Bl<'a, 'b, 'c, T, U, V, M: ThreeWithLifetime<A=(), B=(), C=(), T, U, V, 'a, 'b, 'c>> {
-   |                                                     ----^^----^^----^^^^^^^^^^^^^^^^^^^^^
-   |                                                     |     |     |
-   |                                                     |     |     this associated type binding should be moved after the generic parameters
-   |                                                     |     this associated type binding should be moved after the generic parameters
-   |                                                     this associated type binding should be moved after the generic parameters
+   |                                                     ----              ^  ^  ^  ^^  ^^  ^^ generic argument
+   |                                                     |                 |  |  |  |   |
+   |                                                     |                 |  |  |  |   generic argument
+   |                                                     |                 |  |  |  generic argument
+   |                                                     |                 |  |  generic argument
+   |                                                     |                 |  generic argument
+   |                                                     |                 generic argument
+   |                                                     the first constraint is provided here
 
-error: associated type bindings must be declared after generic parameters
-  --> $DIR/suggest-move-types.rs:57:28
+error: generic arguments must come before the first constraint
+  --> $DIR/suggest-move-types.rs:57:49
    |
 LL | struct C<T, U, V, M: Three<T, A=(), B=(), C=(), U, V>> {
-   |                            ^^^----^^----^^----^^^^^^
-   |                               |     |     |
-   |                               |     |     this associated type binding should be moved after the generic parameters
-   |                               |     this associated type binding should be moved after the generic parameters
-   |                               this associated type binding should be moved after the generic parameters
+   |                               ----              ^  ^ generic argument
+   |                               |                 |
+   |                               |                 generic argument
+   |                               the first constraint is provided here
 
-error: associated type bindings must be declared after generic parameters
-  --> $DIR/suggest-move-types.rs:64:53
+error: generic arguments must come before the first constraint
+  --> $DIR/suggest-move-types.rs:65:78
    |
 LL | struct Cl<'a, 'b, 'c, T, U, V, M: ThreeWithLifetime<T, 'a, A=(), B=(), C=(), U, 'b, V, 'c>> {
-   |                                                     ^^^^^^^----^^----^^----^^^^^^^^^^^^^^
-   |                                                            |     |     |
-   |                                                            |     |     this associated type binding should be moved after the generic parameters
-   |                                                            |     this associated type binding should be moved after the generic parameters
-   |                                                            this associated type binding should be moved after the generic parameters
+   |                                                            ----              ^  ^^  ^  ^^ generic argument
+   |                                                            |                 |  |   |
+   |                                                            |                 |  |   generic argument
+   |                                                            |                 |  generic argument
+   |                                                            |                 generic argument
+   |                                                            the first constraint is provided here
 
-error: associated type bindings must be declared after generic parameters
-  --> $DIR/suggest-move-types.rs:73:28
+error: generic arguments must come before the first constraint
+  --> $DIR/suggest-move-types.rs:74:43
    |
 LL | struct D<T, U, V, M: Three<T, A=(), B=(), U, C=(), V>> {
-   |                            ^^^----^^----^^^^^----^^^
-   |                               |     |        |
-   |                               |     |        this associated type binding should be moved after the generic parameters
-   |                               |     this associated type binding should be moved after the generic parameters
-   |                               this associated type binding should be moved after the generic parameters
+   |                               ----        ^        ^ generic argument
+   |                               |           |
+   |                               |           generic argument
+   |                               the first constraint is provided here
 
-error: associated type bindings must be declared after generic parameters
-  --> $DIR/suggest-move-types.rs:80:53
+error: generic arguments must come before the first constraint
+  --> $DIR/suggest-move-types.rs:82:72
    |
 LL | struct Dl<'a, 'b, 'c, T, U, V, M: ThreeWithLifetime<T, 'a, A=(), B=(), U, 'b, C=(), V, 'c>> {
-   |                                                     ^^^^^^^----^^----^^^^^^^^^----^^^^^^^
-   |                                                            |     |            |
-   |                                                            |     |            this associated type binding should be moved after the generic parameters
-   |                                                            |     this associated type binding should be moved after the generic parameters
-   |                                                            this associated type binding should be moved after the generic parameters
+   |                                                            ----        ^  ^^        ^  ^^ generic argument
+   |                                                            |           |  |         |
+   |                                                            |           |  |         generic argument
+   |                                                            |           |  generic argument
+   |                                                            |           generic argument
+   |                                                            the first constraint is provided here
 
 error[E0747]: type provided when a lifetime was expected
-  --> $DIR/suggest-move-types.rs:34:43
+  --> $DIR/suggest-move-types.rs:33:43
    |
 LL | struct Al<'a, T, M: OneWithLifetime<A=(), T, 'a>> {
    |                                           ^
@@ -91,7 +95,7 @@ LL | struct Bl<'a, 'b, 'c, T, U, V, M: ThreeWithLifetime<A=(), B=(), C=(), T, U,
    = note: lifetime arguments must be provided before type arguments
 
 error[E0747]: lifetime provided when a type was expected
-  --> $DIR/suggest-move-types.rs:64:56
+  --> $DIR/suggest-move-types.rs:65:56
    |
 LL | struct Cl<'a, 'b, 'c, T, U, V, M: ThreeWithLifetime<T, 'a, A=(), B=(), C=(), U, 'b, V, 'c>> {
    |                                                        ^^
@@ -99,7 +103,7 @@ LL | struct Cl<'a, 'b, 'c, T, U, V, M: ThreeWithLifetime<T, 'a, A=(), B=(), C=()
    = note: type arguments must be provided before lifetime arguments
 
 error[E0747]: lifetime provided when a type was expected
-  --> $DIR/suggest-move-types.rs:80:56
+  --> $DIR/suggest-move-types.rs:82:56
    |
 LL | struct Dl<'a, 'b, 'c, T, U, V, M: ThreeWithLifetime<T, 'a, A=(), B=(), U, 'b, C=(), V, 'c>> {
    |                                                        ^^

diff --git a/src/tools/unicode-table-generator/src/main.rs b/src/tools/unicode-table-generator/src/main.rs
index 839d914baa9548ee856e7913ad76bcc9894daa05..d5562ff91df4da66cbad05d167f652f835b9a60c 100644 (file)
--- a/src/tools/unicode-table-generator/src/main.rs
+++ b/src/tools/unicode-table-generator/src/main.rs
@@ -1,9 +1,83 @@
+//! This implements the core logic of the compression scheme used to compactly
+//! encode Unicode properties.
+//!
+//! We have two primary goals with the encoding: we want to be compact, because
+//! these tables often end up in ~every Rust program (especially the
+//! grapheme_extend table, used for str debugging), including those for embedded
+//! targets (where space is important). We also want to be relatively fast,
+//! though this is more of a nice to have rather than a key design constraint.
+//! It is expected that libraries/applications which are performance-sensitive
+//! to Unicode property lookups are extremely rare, and those that care may find
+//! the tradeoff of the raw bitsets worth it. For most applications, a
+//! relatively fast but much smaller (and as such less cache-impacting, etc.)
+//! data set is likely preferable.
+//!
+//! We have two separate encoding schemes: a skiplist-like approach, and a
+//! compressed bitset. The datasets we consider mostly use the skiplist (it's
+//! smaller) but the lowercase and uppercase sets are sufficiently sparse for
+//! the bitset to be worthwhile -- for those sets the biset is a 2x size win.
+//! Since the bitset is also faster, this seems an obvious choice. (As a
+//! historical note, the bitset was also the prior implementation, so its
+//! relative complexity had already been paid).
+//!
+//! ## The bitset
+//!
+//! The primary idea is that we 'flatten' the Unicode ranges into an enormous
+//! bitset. To represent any arbitrary codepoint in a raw bitset, we would need
+//! over 17 kilobytes of data per character set -- way too much for our
+//! purposes.
+//!
+//! First, the raw bitset (one bit for every valid `char`, from 0 to 0x10FFFF,
+//! not skipping the small 'gap') is associated into words (u64) and
+//! deduplicated. On random data, this would be useless; on our data, this is
+//! incredibly beneficial -- our data sets have (far) less than 256 unique
+//! words.
+//!
+//! This gives us an array that maps `u8 -> word`; the current algorithm does
+//! not handle the case of more than 256 unique words, but we are relatively far
+//! from coming that close.
+//!
+//! With that scheme, we now have a single byte for every 64 codepoints.
+//!
+//! We further chunk these by some constant N (between 1 and 64 per group,
+//! dynamically chosen for smallest size), and again deduplicate and store in an
+//! array (u8 -> [u8; N]).
+//!
+//! The bytes of this array map into the words from the bitset above, but we
+//! apply another trick here: some of these words are similar enough that they
+//! can be represented by some function of another word. The particular
+//! functions chosen are rotation, inversion, and shifting (right).
+//!
+//! ## The skiplist
+//!
+//! The skip list arose out of the desire for an even smaller encoding than the
+//! bitset -- and was the answer to the question "what is the smallest
+//! representation we can imagine?". However, it is not necessarily the
+//! smallest, and if you have a better proposal, please do suggest it!
+//!
+//! This is a relatively straightforward encoding. First, we break up all the
+//! ranges in the input data into offsets from each other, essentially a gap
+//! encoding. In practice, most gaps are small -- less than u8::MAX -- so we
+//! store those directly. We make use of the larger gaps (which are nicely
+//! interspersed already) throughout the dataset to index this data set.
+//!
+//! In particular, each run of small gaps (terminating in a large gap) is
+//! indexed in a separate dataset. That data set stores an index into the
+//! primary offset list and a prefix sum of that offset list. These are packed
+//! into a single u32 (11 bits for the offset, 21 bits for the prefix sum).
+//!
+//! Lookup proceeds via a binary search in the index and then a straightforward
+//! linear scan (adding up the offsets) until we reach the needle, and then the
+//! index of that offset is utilized as the answer to whether we're in the set
+//! or not.
+
 use std::collections::{BTreeMap, HashMap};
 use std::ops::Range;
 use ucd_parse::Codepoints;
 
 mod case_mapping;
 mod raw_emitter;
+mod skiplist;
 mod unicode_download;
 
 use raw_emitter::{emit_codepoints, RawEmitter};
@@ -152,9 +226,17 @@ fn main() {
         std::process::exit(1);
     });
 
+    // Optional test path, which is a Rust source file testing that the unicode
+    // property lookups are correct.
+    let test_path = std::env::args().nth(2);
+
     let unicode_data = load_data();
     let ranges_by_property = &unicode_data.ranges;
 
+    if let Some(path) = test_path {
+        std::fs::write(&path, generate_tests(&write_location, &ranges_by_property)).unwrap();
+    }
+
     let mut total_bytes = 0;
     let mut modules = Vec::new();
     for (property, ranges) in ranges_by_property {
@@ -163,7 +245,16 @@ fn main() {
         emit_codepoints(&mut emitter, &ranges);
 
         modules.push((property.to_lowercase().to_string(), emitter.file));
-        println!("{:15}: {} bytes, {} codepoints", property, emitter.bytes_used, datapoints,);
+        println!(
+            "{:15}: {} bytes, {} codepoints in {} ranges ({} - {}) using {}",
+            property,
+            emitter.bytes_used,
+            datapoints,
+            ranges.len(),
+            ranges.first().unwrap().start,
+            ranges.last().unwrap().end,
+            emitter.desc,
+        );
         total_bytes += emitter.bytes_used;
     }
 
@@ -173,7 +264,10 @@ fn main() {
         "///! This file is generated by src/tools/unicode-table-generator; do not edit manually!\n",
     );
 
-    table_file.push_str("use super::range_search;\n\n");
+    // Include the range search function
+    table_file.push('\n');
+    table_file.push_str(include_str!("range_search.rs"));
+    table_file.push('\n');
 
     table_file.push_str(&version());
 
@@ -236,21 +330,97 @@ fn fmt_list<V: std::fmt::Debug>(values: impl IntoIterator<Item = V>) -> String {
     out
 }
 
+fn generate_tests(data_path: &str, ranges: &[(&str, Vec<Range<u32>>)]) -> String {
+    let mut s = String::new();
+    s.push_str("#![allow(incomplete_features, unused)]\n");
+    s.push_str("#![feature(const_generics)]\n\n");
+    s.push_str("\n#[allow(unused)]\nuse std::hint;\n");
+    s.push_str(&format!("#[path = \"{}\"]\n", data_path));
+    s.push_str("mod unicode_data;\n\n");
+
+    s.push_str("\nfn main() {\n");
+
+    for (property, ranges) in ranges {
+        s.push_str(&format!(r#"    println!("Testing {}");"#, property));
+        s.push('\n');
+        s.push_str(&format!("    {}_true();\n", property.to_lowercase()));
+        s.push_str(&format!("    {}_false();\n", property.to_lowercase()));
+        let mut is_true = Vec::new();
+        let mut is_false = Vec::new();
+        for ch_num in 0..(std::char::MAX as u32) {
+            if std::char::from_u32(ch_num).is_none() {
+                continue;
+            }
+            if ranges.iter().any(|r| r.contains(&ch_num)) {
+                is_true.push(ch_num);
+            } else {
+                is_false.push(ch_num);
+            }
+        }
+
+        s.push_str(&format!("    fn {}_true() {{\n", property.to_lowercase()));
+        generate_asserts(&mut s, property, &is_true, true);
+        s.push_str("    }\n\n");
+        s.push_str(&format!("    fn {}_false() {{\n", property.to_lowercase()));
+        generate_asserts(&mut s, property, &is_false, false);
+        s.push_str("    }\n\n");
+    }
+
+    s.push_str("}");
+    s
+}
+
+fn generate_asserts(s: &mut String, property: &str, points: &[u32], truthy: bool) {
+    for range in ranges_from_set(points) {
+        if range.end == range.start + 1 {
+            s.push_str(&format!(
+                "        assert!({}unicode_data::{}::lookup({:?}), \"{}\");\n",
+                if truthy { "" } else { "!" },
+                property.to_lowercase(),
+                std::char::from_u32(range.start).unwrap(),
+                range.start,
+            ));
+        } else {
+            s.push_str(&format!("        for chn in {:?}u32 {{\n", range));
+            s.push_str(&format!(
+                "            assert!({}unicode_data::{}::lookup(std::char::from_u32(chn).unwrap()), \"{{:?}}\", chn);\n",
+                if truthy { "" } else { "!" },
+                property.to_lowercase(),
+            ));
+            s.push_str("        }\n");
+        }
+    }
+}
+
+fn ranges_from_set(set: &[u32]) -> Vec<Range<u32>> {
+    let mut ranges = set.iter().map(|e| (*e)..(*e + 1)).collect::<Vec<Range<u32>>>();
+    merge_ranges(&mut ranges);
+    ranges
+}
+
 fn merge_ranges(ranges: &mut Vec<Range<u32>>) {
     loop {
         let mut new_ranges = Vec::new();
         let mut idx_iter = 0..(ranges.len() - 1);
+        let mut should_insert_last = true;
         while let Some(idx) = idx_iter.next() {
             let cur = ranges[idx].clone();
             let next = ranges[idx + 1].clone();
             if cur.end == next.start {
-                let _ = idx_iter.next(); // skip next as we're merging it in
+                if idx_iter.next().is_none() {
+                    // We're merging the last element
+                    should_insert_last = false;
+                }
                 new_ranges.push(cur.start..next.end);
             } else {
+                // We're *not* merging the last element
+                should_insert_last = true;
                 new_ranges.push(cur);
             }
         }
-        new_ranges.push(ranges.last().unwrap().clone());
+        if should_insert_last {
+            new_ranges.push(ranges.last().unwrap().clone());
+        }
         if new_ranges.len() == ranges.len() {
             *ranges = new_ranges;
             break;
@@ -258,4 +428,12 @@ fn merge_ranges(ranges: &mut Vec<Range<u32>>) {
             *ranges = new_ranges;
         }
     }
+
+    let mut last_end = None;
+    for range in ranges {
+        if let Some(last) = last_end {
+            assert!(range.start > last, "{:?}", range);
+        }
+        last_end = Some(range.end);
+    }
 }

diff --git a/src/tools/unicode-table-generator/src/range_search.rs b/src/tools/unicode-table-generator/src/range_search.rs
new file mode 100644 (file)
index 0000000..39b47ce
--- /dev/null
+++ b/src/tools/unicode-table-generator/src/range_search.rs
@@ -0,0 +1,93 @@
+#[inline(always)]
+fn bitset_search<
+    const N: usize,
+    const CHUNK_SIZE: usize,
+    const N1: usize,
+    const CANONICAL: usize,
+    const CANONICALIZED: usize,
+>(
+    needle: u32,
+    chunk_idx_map: &[u8; N],
+    bitset_chunk_idx: &[[u8; CHUNK_SIZE]; N1],
+    bitset_canonical: &[u64; CANONICAL],
+    bitset_canonicalized: &[(u8, u8); CANONICALIZED],
+) -> bool {
+    let bucket_idx = (needle / 64) as usize;
+    let chunk_map_idx = bucket_idx / CHUNK_SIZE;
+    let chunk_piece = bucket_idx % CHUNK_SIZE;
+    let chunk_idx = if let Some(&v) = chunk_idx_map.get(chunk_map_idx) {
+        v
+    } else {
+        return false;
+    };
+    let idx = bitset_chunk_idx[chunk_idx as usize][chunk_piece] as usize;
+    let word = if let Some(word) = bitset_canonical.get(idx) {
+        *word
+    } else {
+        let (real_idx, mapping) = bitset_canonicalized[idx - bitset_canonical.len()];
+        let mut word = bitset_canonical[real_idx as usize];
+        let should_invert = mapping & (1 << 6) != 0;
+        if should_invert {
+            word = !word;
+        }
+        // Lower 6 bits
+        let quantity = mapping & ((1 << 6) - 1);
+        if mapping & (1 << 7) != 0 {
+            // shift
+            word >>= quantity as u64;
+        } else {
+            word = word.rotate_left(quantity as u32);
+        }
+        word
+    };
+    (word & (1 << (needle % 64) as u64)) != 0
+}
+
+fn decode_prefix_sum(short_offset_run_header: u32) -> u32 {
+    short_offset_run_header & ((1 << 21) - 1)
+}
+
+fn decode_length(short_offset_run_header: u32) -> usize {
+    (short_offset_run_header >> 21) as usize
+}
+
+#[inline(always)]
+fn skip_search<const SOR: usize, const OFFSETS: usize>(
+    needle: u32,
+    short_offset_runs: &[u32; SOR],
+    offsets: &[u8; OFFSETS],
+) -> bool {
+    // Note that this *cannot* be past the end of the array, as the last
+    // element is greater than std::char::MAX (the largest possible needle).
+    //
+    // So, we cannot have found it (i.e. Ok(idx) + 1 != length) and the correct
+    // location cannot be past it, so Err(idx) != length either.
+    //
+    // This means that we can avoid bounds checking for the accesses below, too.
+    let last_idx =
+        match short_offset_runs.binary_search_by_key(&(needle << 11), |header| header << 11) {
+            Ok(idx) => idx + 1,
+            Err(idx) => idx,
+        };
+
+    let mut offset_idx = decode_length(short_offset_runs[last_idx]);
+    let length = if let Some(next) = short_offset_runs.get(last_idx + 1) {
+        decode_length(*next) - offset_idx
+    } else {
+        offsets.len() - offset_idx
+    };
+    let prev =
+        last_idx.checked_sub(1).map(|prev| decode_prefix_sum(short_offset_runs[prev])).unwrap_or(0);
+
+    let total = needle - prev;
+    let mut prefix_sum = 0;
+    for _ in 0..(length - 1) {
+        let offset = offsets[offset_idx];
+        prefix_sum += offset as u32;
+        if prefix_sum > total {
+            break;
+        }
+        offset_idx += 1;
+    }
+    offset_idx % 2 == 1
+}

diff --git a/src/tools/unicode-table-generator/src/raw_emitter.rs b/src/tools/unicode-table-generator/src/raw_emitter.rs
index 3e60ce13f9223fc24082eb167bfa5f6bb9e36f8a..95b63aca1549b45c2cb89b8a54c21aa14189a40a 100644 (file)
--- a/src/tools/unicode-table-generator/src/raw_emitter.rs
+++ b/src/tools/unicode-table-generator/src/raw_emitter.rs
@@ -1,55 +1,19 @@
-//! This implements the core logic of the compression scheme used to compactly
-//! encode the Unicode character classes.
-//!
-//! The primary idea is that we 'flatten' the Unicode ranges into an enormous
-//! bitset. To represent any arbitrary codepoint in a raw bitset, we would need
-//! over 17 kilobytes of data per character set -- way too much for our
-//! purposes.
-//!
-//! We have two primary goals with the encoding: we want to be compact, because
-//! these tables often end up in ~every Rust program (especially the
-//! grapheme_extend table, used for str debugging), including those for embedded
-//! targets (where space is important). We also want to be relatively fast,
-//! though this is more of a nice to have rather than a key design constraint.
-//! In practice, due to modern processor design these two are closely related.
-//!
-//! The encoding scheme here compresses the bitset by first deduplicating the
-//! "words" (64 bits on all platforms). In practice very few words are present
-//! in most data sets.
-//!
-//! This gives us an array that maps `u8 -> word` (if we ever went beyond 256
-//! words, we could go to u16 -> word or have some dual compression scheme
-//! mapping into two separate sets; currently this is not dealt with).
-//!
-//! With that scheme, we now have a single byte for every 64 codepoints. We
-//! further group these by 16 (arbitrarily chosen), and again deduplicate and
-//! store in an array (u8 -> [u8; 16]).
-//!
-//! The indices into this array represent ranges of 64*16 = 1024 codepoints.
-//!
-//! This already reduces the top-level array to at most 1,086 bytes, but in
-//! practice we usually can encode in far fewer (the first couple Unicode planes
-//! are dense).
-//!
-//! The last byte of this top-level array is pulled out to a separate static
-//! and trailing zeros are dropped; this is simply because grapheme_extend and
-//! case_ignorable have a single entry in the 896th entry, so this shrinks them
-//! down considerably.
-
 use crate::fmt_list;
-use std::collections::{BTreeSet, HashMap};
+use std::collections::{BTreeMap, BTreeSet, HashMap};
 use std::convert::TryFrom;
-use std::fmt::Write;
+use std::fmt::{self, Write};
 use std::ops::Range;
 
+#[derive(Clone)]
 pub struct RawEmitter {
     pub file: String,
+    pub desc: String,
     pub bytes_used: usize,
 }
 
 impl RawEmitter {
     pub fn new() -> RawEmitter {
-        RawEmitter { file: String::new(), bytes_used: 0 }
+        RawEmitter { file: String::new(), bytes_used: 0, desc: String::new() }
     }
 
     fn blank_line(&mut self) {
@@ -59,30 +23,100 @@ fn blank_line(&mut self) {
         writeln!(&mut self.file, "").unwrap();
     }
 
-    fn emit_bitset(&mut self, words: &[u64]) {
+    fn emit_bitset(&mut self, ranges: &[Range<u32>]) {
+        let last_code_point = ranges.last().unwrap().end;
+        // bitset for every bit in the codepoint range
+        //
+        // + 2 to ensure an all zero word to use for padding
+        let mut buckets = vec![0u64; (last_code_point as usize / 64) + 2];
+        for range in ranges {
+            for codepoint in range.clone() {
+                let bucket = codepoint as usize / 64;
+                let bit = codepoint as u64 % 64;
+                buckets[bucket] |= 1 << bit;
+            }
+        }
+
+        let mut words = buckets;
+        // Ensure that there's a zero word in the dataset, used for padding and
+        // such.
+        words.push(0);
         let unique_words =
             words.iter().cloned().collect::<BTreeSet<_>>().into_iter().collect::<Vec<_>>();
         if unique_words.len() > u8::max_value() as usize {
             panic!("cannot pack {} into 8 bits", unique_words.len());
         }
+        // needed for the chunk mapping to work
+        assert_eq!(unique_words[0], 0, "has a zero word");
+        let canonicalized = Canonicalized::canonicalize(&unique_words);
 
-        let word_indices = unique_words
-            .iter()
-            .cloned()
-            .enumerate()
-            .map(|(idx, word)| (word, u8::try_from(idx).unwrap()))
-            .collect::<HashMap<_, _>>();
+        let word_indices = canonicalized.unique_mapping.clone();
+        let compressed_words = words.iter().map(|w| word_indices[w]).collect::<Vec<u8>>();
+
+        let mut best = None;
+        for length in 1..=64 {
+            let mut temp = self.clone();
+            temp.emit_chunk_map(word_indices[&0], &compressed_words, length);
+            if let Some((_, size)) = best {
+                if temp.bytes_used < size {
+                    best = Some((length, temp.bytes_used));
+                }
+            } else {
+                best = Some((length, temp.bytes_used));
+            }
+        }
+        self.emit_chunk_map(word_indices[&0], &compressed_words, best.unwrap().0);
+
+        struct Bits(u64);
+        impl fmt::Debug for Bits {
+            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+                write!(f, "0b{:064b}", self.0)
+            }
+        }
+
+        writeln!(
+            &mut self.file,
+            "static BITSET_CANONICAL: [u64; {}] = [{}];",
+            canonicalized.canonical_words.len(),
+            fmt_list(canonicalized.canonical_words.iter().map(|v| Bits(*v))),
+        )
+        .unwrap();
+        self.bytes_used += 8 * canonicalized.canonical_words.len();
+        writeln!(
+            &mut self.file,
+            "static BITSET_MAPPING: [(u8, u8); {}] = [{}];",
+            canonicalized.canonicalized_words.len(),
+            fmt_list(&canonicalized.canonicalized_words),
+        )
+        .unwrap();
+        // 8 bit index into shifted words, 7 bits for shift + optional flip
+        // We only need it for the words that we removed by applying a shift and
+        // flip to them.
+        self.bytes_used += 2 * canonicalized.canonicalized_words.len();
+
+        self.blank_line();
 
-        let mut idx = words.iter().map(|w| word_indices[w]).collect::<Vec<u8>>();
-        let chunk_length = 16;
-        for _ in 0..(chunk_length - (idx.len() % chunk_length)) {
-            assert_eq!(unique_words[0], 0, "first word is all zeros");
-            // pad out bitset index with zero words so we have all chunks of 16
-            idx.push(0);
+        writeln!(&mut self.file, "pub fn lookup(c: char) -> bool {{").unwrap();
+        writeln!(&mut self.file, "    super::bitset_search(",).unwrap();
+        writeln!(&mut self.file, "        c as u32,").unwrap();
+        writeln!(&mut self.file, "        &BITSET_CHUNKS_MAP,").unwrap();
+        writeln!(&mut self.file, "        &BITSET_INDEX_CHUNKS,").unwrap();
+        writeln!(&mut self.file, "        &BITSET_CANONICAL,").unwrap();
+        writeln!(&mut self.file, "        &BITSET_MAPPING,").unwrap();
+        writeln!(&mut self.file, "    )").unwrap();
+        writeln!(&mut self.file, "}}").unwrap();
+    }
+
+    fn emit_chunk_map(&mut self, zero_at: u8, compressed_words: &[u8], chunk_length: usize) {
+        let mut compressed_words = compressed_words.to_vec();
+        for _ in 0..(chunk_length - (compressed_words.len() % chunk_length)) {
+            // pad out bitset index with zero words so we have all chunks of
+            // chunkchunk_length
+            compressed_words.push(zero_at);
         }
 
         let mut chunks = BTreeSet::new();
-        for chunk in idx.chunks(chunk_length) {
+        for chunk in compressed_words.chunks(chunk_length) {
             chunks.insert(chunk);
         }
         let chunk_map = chunks
@@ -92,23 +126,10 @@ fn emit_bitset(&mut self, words: &[u64]) {
             .map(|(idx, chunk)| (chunk, idx))
             .collect::<HashMap<_, _>>();
         let mut chunk_indices = Vec::new();
-        for chunk in idx.chunks(chunk_length) {
+        for chunk in compressed_words.chunks(chunk_length) {
             chunk_indices.push(chunk_map[chunk]);
         }
-        writeln!(
-            &mut self.file,
-            "static BITSET_LAST_CHUNK_MAP: (u16, u8) = ({}, {});",
-            chunk_indices.len() - 1,
-            chunk_indices.pop().unwrap(),
-        )
-        .unwrap();
-        self.bytes_used += 3;
-        // Strip out the empty pieces, presuming our above pop() made us now
-        // have some trailing zeros.
-        assert_eq!(unique_words[0], 0, "first word is all zeros");
-        while let Some(0) = chunk_indices.last() {
-            chunk_indices.pop();
-        }
+
         writeln!(
             &mut self.file,
             "static BITSET_CHUNKS_MAP: [u8; {}] = [{}];",
@@ -119,52 +140,253 @@ fn emit_bitset(&mut self, words: &[u64]) {
         self.bytes_used += chunk_indices.len();
         writeln!(
             &mut self.file,
-            "static BITSET_INDEX_CHUNKS: [[u8; 16]; {}] = [{}];",
+            "static BITSET_INDEX_CHUNKS: [[u8; {}]; {}] = [{}];",
+            chunk_length,
             chunks.len(),
             fmt_list(chunks.iter()),
         )
         .unwrap();
-        self.bytes_used += 16 * chunks.len();
-        writeln!(
-            &mut self.file,
-            "static BITSET: [u64; {}] = [{}];",
-            unique_words.len(),
-            fmt_list(&unique_words),
-        )
-        .unwrap();
-        self.bytes_used += 8 * unique_words.len();
-    }
-
-    pub fn emit_lookup(&mut self) {
-        writeln!(&mut self.file, "pub fn lookup(c: char) -> bool {{").unwrap();
-        writeln!(&mut self.file, "    super::range_search(",).unwrap();
-        writeln!(&mut self.file, "        c as u32,").unwrap();
-        writeln!(&mut self.file, "        &BITSET_CHUNKS_MAP,").unwrap();
-        writeln!(&mut self.file, "        BITSET_LAST_CHUNK_MAP,").unwrap();
-        writeln!(&mut self.file, "        &BITSET_INDEX_CHUNKS,").unwrap();
-        writeln!(&mut self.file, "        &BITSET,").unwrap();
-        writeln!(&mut self.file, "    )").unwrap();
-        writeln!(&mut self.file, "}}").unwrap();
+        self.bytes_used += chunk_length * chunks.len();
     }
 }
 
 pub fn emit_codepoints(emitter: &mut RawEmitter, ranges: &[Range<u32>]) {
     emitter.blank_line();
 
-    let last_code_point = ranges.last().unwrap().end;
-    // bitset for every bit in the codepoint range
-    //
-    // + 2 to ensure an all zero word to use for padding
-    let mut buckets = vec![0u64; (last_code_point as usize / 64) + 2];
-    for range in ranges {
-        for codepoint in range.clone() {
-            let bucket = codepoint as usize / 64;
-            let bit = codepoint as u64 % 64;
-            buckets[bucket] |= 1 << bit;
-        }
+    let mut bitset = emitter.clone();
+    bitset.emit_bitset(&ranges);
+
+    let mut skiplist = emitter.clone();
+    skiplist.emit_skiplist(&ranges);
+
+    if bitset.bytes_used <= skiplist.bytes_used {
+        *emitter = bitset;
+        emitter.desc = format!("bitset");
+    } else {
+        *emitter = skiplist;
+        emitter.desc = format!("skiplist");
     }
+}
 
-    emitter.emit_bitset(&buckets);
-    emitter.blank_line();
-    emitter.emit_lookup();
+struct Canonicalized {
+    canonical_words: Vec<u64>,
+    canonicalized_words: Vec<(u8, u8)>,
+
+    /// Maps an input unique word to the associated index (u8) which is into
+    /// canonical_words or canonicalized_words (in order).
+    unique_mapping: HashMap<u64, u8>,
+}
+
+impl Canonicalized {
+    fn canonicalize(unique_words: &[u64]) -> Self {
+        #[derive(Copy, Clone, Debug)]
+        enum Mapping {
+            Rotate(u32),
+            Invert,
+            RotateAndInvert(u32),
+            ShiftRight(u32),
+        }
+
+        // key is the word being mapped to
+        let mut mappings: BTreeMap<u64, Vec<(u64, Mapping)>> = BTreeMap::new();
+        for &a in unique_words {
+            'b: for &b in unique_words {
+                // skip self
+                if a == b {
+                    continue;
+                }
+
+                // All possible distinct rotations
+                for rotation in 1..64 {
+                    if a.rotate_right(rotation) == b {
+                        mappings.entry(b).or_default().push((a, Mapping::Rotate(rotation)));
+                        // We're not interested in further mappings between a and b
+                        continue 'b;
+                    }
+                }
+
+                if (!a) == b {
+                    mappings.entry(b).or_default().push((a, Mapping::Invert));
+                    // We're not interested in further mappings between a and b
+                    continue 'b;
+                }
+
+                // All possible distinct rotations, inverted
+                for rotation in 1..64 {
+                    if (!a.rotate_right(rotation)) == b {
+                        mappings
+                            .entry(b)
+                            .or_default()
+                            .push((a, Mapping::RotateAndInvert(rotation)));
+                        // We're not interested in further mappings between a and b
+                        continue 'b;
+                    }
+                }
+
+                // All possible shifts
+                for shift_by in 1..64 {
+                    if a == (b >> shift_by) {
+                        mappings
+                            .entry(b)
+                            .or_default()
+                            .push((a, Mapping::ShiftRight(shift_by as u32)));
+                        // We're not interested in further mappings between a and b
+                        continue 'b;
+                    }
+                }
+            }
+        }
+        // These are the bitset words which will be represented "raw" (as a u64)
+        let mut canonical_words = Vec::new();
+        // These are mapped words, which will be represented by an index into
+        // the canonical_words and a Mapping; u16 when encoded.
+        let mut canonicalized_words = Vec::new();
+        let mut unique_mapping = HashMap::new();
+
+        #[derive(Debug, PartialEq, Eq)]
+        enum UniqueMapping {
+            Canonical(usize),
+            Canonicalized(usize),
+        }
+
+        // Map 0 first, so that it is the first canonical word.
+        // This is realistically not inefficient because 0 is not mapped to by
+        // anything else (a shift pattern could do it, but would be wasteful).
+        //
+        // However, 0s are quite common in the overall dataset, and it is quite
+        // wasteful to have to go through a mapping function to determine that
+        // we have a zero.
+        //
+        // FIXME: Experiment with choosing most common words in overall data set
+        // for canonical when possible.
+        while let Some((&to, _)) = mappings
+            .iter()
+            .find(|(&to, _)| to == 0)
+            .or_else(|| mappings.iter().max_by_key(|m| m.1.len()))
+        {
+            // Get the mapping with the most entries. Currently, no mapping can
+            // only exist transitively (i.e., there is no A, B, C such that A
+            // does not map to C and but A maps to B maps to C), so this is
+            // guaranteed to be acceptable.
+            //
+            // In the future, we may need a more sophisticated algorithm to
+            // identify which keys to prefer as canonical.
+            let mapped_from = mappings.remove(&to).unwrap();
+            for (from, how) in &mapped_from {
+                // Remove the entries which mapped to this one.
+                // Noting that it should be associated with the Nth canonical word.
+                //
+                // We do not assert that this is present, because there may be
+                // no mappings to the `from` word; that's fine.
+                mappings.remove(from);
+                assert_eq!(
+                    unique_mapping
+                        .insert(*from, UniqueMapping::Canonicalized(canonicalized_words.len())),
+                    None
+                );
+                canonicalized_words.push((canonical_words.len(), *how));
+
+                // Remove the now-canonicalized word from other mappings,
+                // to ensure that we deprioritize them in the next iteration of
+                // the while loop.
+                for (_, mapped) in &mut mappings {
+                    let mut i = 0;
+                    while i != mapped.len() {
+                        if mapped[i].0 == *from {
+                            mapped.remove(i);
+                        } else {
+                            i += 1;
+                        }
+                    }
+                }
+            }
+            assert!(
+                unique_mapping
+                    .insert(to, UniqueMapping::Canonical(canonical_words.len()))
+                    .is_none()
+            );
+            canonical_words.push(to);
+
+            // Remove the now-canonical word from other mappings, to ensure that
+            // we deprioritize them in the next iteration of the while loop.
+            for (_, mapped) in &mut mappings {
+                let mut i = 0;
+                while i != mapped.len() {
+                    if mapped[i].0 == to {
+                        mapped.remove(i);
+                    } else {
+                        i += 1;
+                    }
+                }
+            }
+        }
+
+        // Any words which we couldn't shrink, just stick into the canonical
+        // words.
+        //
+        // FIXME: work harder -- there are more possibilities for mapping
+        // functions (e.g., multiplication, shifting instead of rotation, etc.)
+        // We'll probably always have some slack though so this loop will still
+        // be needed.
+        for &w in unique_words {
+            if !unique_mapping.contains_key(&w) {
+                assert!(
+                    unique_mapping
+                        .insert(w, UniqueMapping::Canonical(canonical_words.len()))
+                        .is_none()
+                );
+                canonical_words.push(w);
+            }
+        }
+        assert_eq!(canonicalized_words.len() + canonical_words.len(), unique_words.len());
+        assert_eq!(unique_mapping.len(), unique_words.len());
+
+        let unique_mapping = unique_mapping
+            .into_iter()
+            .map(|(key, value)| {
+                (
+                    key,
+                    match value {
+                        UniqueMapping::Canonicalized(idx) => {
+                            u8::try_from(canonical_words.len() + idx).unwrap()
+                        }
+                        UniqueMapping::Canonical(idx) => u8::try_from(idx).unwrap(),
+                    },
+                )
+            })
+            .collect::<HashMap<_, _>>();
+
+        let mut distinct_indices = BTreeSet::new();
+        for &w in unique_words {
+            let idx = unique_mapping.get(&w).unwrap();
+            assert!(distinct_indices.insert(idx));
+        }
+
+        const LOWER_6: u32 = (1 << 6) - 1;
+
+        let canonicalized_words = canonicalized_words
+            .into_iter()
+            .map(|v| {
+                (
+                    u8::try_from(v.0).unwrap(),
+                    match v.1 {
+                        Mapping::RotateAndInvert(amount) => {
+                            assert_eq!(amount, amount & LOWER_6);
+                            1 << 6 | (amount as u8)
+                        }
+                        Mapping::Rotate(amount) => {
+                            assert_eq!(amount, amount & LOWER_6);
+                            amount as u8
+                        }
+                        Mapping::Invert => 1 << 6,
+                        Mapping::ShiftRight(shift_by) => {
+                            assert_eq!(shift_by, shift_by & LOWER_6);
+                            1 << 7 | (shift_by as u8)
+                        }
+                    },
+                )
+            })
+            .collect::<Vec<(u8, u8)>>();
+        Canonicalized { unique_mapping, canonical_words, canonicalized_words }
+    }
 }

diff --git a/src/tools/unicode-table-generator/src/skiplist.rs b/src/tools/unicode-table-generator/src/skiplist.rs
new file mode 100644 (file)
index 0000000..6e43996
--- /dev/null
+++ b/src/tools/unicode-table-generator/src/skiplist.rs
@@ -0,0 +1,98 @@
+use crate::fmt_list;
+use crate::raw_emitter::RawEmitter;
+use std::convert::TryInto;
+use std::fmt::Write as _;
+use std::ops::Range;
+
+/// This will get packed into a single u32 before inserting into the data set.
+#[derive(Debug, PartialEq)]
+struct ShortOffsetRunHeader {
+    /// Note, we only allow for 21 bits here.
+    prefix_sum: u32,
+
+    /// Note, we actually only allow for 11 bits here. This should be enough --
+    /// our largest sets are around ~1400 offsets long.
+    start_idx: u16,
+}
+
+impl ShortOffsetRunHeader {
+    fn pack(&self) -> u32 {
+        assert!(self.start_idx < (1 << 11));
+        assert!(self.prefix_sum < (1 << 21));
+
+        (self.start_idx as u32) << 21 | self.prefix_sum
+    }
+}
+
+impl RawEmitter {
+    pub fn emit_skiplist(&mut self, ranges: &[Range<u32>]) {
+        let mut offsets = Vec::<u32>::new();
+        let points = ranges.iter().flat_map(|r| vec![r.start, r.end]).collect::<Vec<u32>>();
+        let mut offset = 0;
+        for pt in points {
+            let delta = pt - offset;
+            offsets.push(delta);
+            offset = pt;
+        }
+        // Guaranteed to terminate, as it's impossible to subtract a value this
+        // large from a valid char.
+        offsets.push(std::char::MAX as u32 + 1);
+        let mut coded_offsets: Vec<u8> = Vec::new();
+        let mut short_offset_runs: Vec<ShortOffsetRunHeader> = vec![];
+        let mut iter = offsets.iter().cloned();
+        let mut prefix_sum = 0;
+        loop {
+            let mut any_elements = false;
+            let mut inserted = false;
+            let start = coded_offsets.len();
+            for offset in iter.by_ref() {
+                any_elements = true;
+                prefix_sum += offset;
+                if let Ok(offset) = offset.try_into() {
+                    coded_offsets.push(offset);
+                } else {
+                    short_offset_runs.push(ShortOffsetRunHeader {
+                        start_idx: start.try_into().unwrap(),
+                        prefix_sum,
+                    });
+                    // This is just needed to maintain indices even/odd
+                    // correctly.
+                    coded_offsets.push(0);
+                    inserted = true;
+                    break;
+                }
+            }
+            if !any_elements {
+                break;
+            }
+            // We always append the huge char::MAX offset to the end which
+            // should never be able to fit into the u8 offsets.
+            assert!(inserted);
+        }
+
+        writeln!(
+            &mut self.file,
+            "static SHORT_OFFSET_RUNS: [u32; {}] = [{}];",
+            short_offset_runs.len(),
+            fmt_list(short_offset_runs.iter().map(|v| v.pack()))
+        )
+        .unwrap();
+        self.bytes_used += 4 * short_offset_runs.len();
+        writeln!(
+            &mut self.file,
+            "static OFFSETS: [u8; {}] = [{}];",
+            coded_offsets.len(),
+            fmt_list(&coded_offsets)
+        )
+        .unwrap();
+        self.bytes_used += coded_offsets.len();
+
+        writeln!(&mut self.file, "pub fn lookup(c: char) -> bool {{").unwrap();
+        writeln!(&mut self.file, "    super::skip_search(",).unwrap();
+        writeln!(&mut self.file, "        c as u32,").unwrap();
+        writeln!(&mut self.file, "        &SHORT_OFFSET_RUNS,").unwrap();
+        writeln!(&mut self.file, "        &OFFSETS,").unwrap();
+        writeln!(&mut self.file, "    )").unwrap();
+        writeln!(&mut self.file, "}}").unwrap();
+    }
+}

diff --git a/src/tools/unicode-table-generator/src/unicode_download.rs b/src/tools/unicode-table-generator/src/unicode_download.rs
index 3f6de9ea3bbd78b05b7f807876312da38f936e4e..fa57f650ac082706d11bbdd8bd3e6148b22ec3b1 100644 (file)
--- a/src/tools/unicode-table-generator/src/unicode_download.rs
+++ b/src/tools/unicode-table-generator/src/unicode_download.rs
@@ -11,10 +11,15 @@
 
 pub fn fetch_latest() {
     let directory = Path::new(UNICODE_DIRECTORY);
+    if directory.exists() {
+        eprintln!(
+            "Not refetching unicode data, already exists, please delete {:?} to regenerate",
+            directory
+        );
+        return;
+    }
     if let Err(e) = std::fs::create_dir_all(directory) {
-        if e.kind() != std::io::ErrorKind::AlreadyExists {
-            panic!("Failed to create {:?}: {}", UNICODE_DIRECTORY, e);
-        }
+        panic!("Failed to create {:?}: {}", UNICODE_DIRECTORY, e);
     }
     let output = Command::new("curl").arg(URL_PREFIX.to_owned() + README).output().unwrap();
     if !output.status.success() {