CodePointTrie support for normalizer and collator perf improvements

components/collections/Cargo.toml

@@ -29,7 +29,6 @@ yoke = { workspace = true, features = ["derive"] }
 zerofrom = { workspace = true, features = ["derive"] }
 zerovec = { workspace = true, features = ["derive", "yoke"] }
 potential_utf = { workspace = true, features = ["zerovec"] }
-utf8_iter = { workspace = true }
 
 serde = { workspace = true, features = ["derive"], optional = true }
 databake = { workspace = true, features = ["derive"], optional = true }

components/collections/src/codepointinvliststringlist/mod.rs

@@ -175,13 +175,22 @@ impl<'data> CodePointInversionListAndStringList<'data> {
 
     /// See [`Self::contains_str`]
     pub fn contains_utf8(&self, s: &[u8]) -> bool {
-        use utf8_iter::Utf8CharsEx;
-        let mut chars = s.chars();
-        if let Some(first_char) = chars.next() {
-            if chars.next().is_none() {
-                return self.contains(first_char);
+        let single_char = s.get(0).and_then(|&first_byte| {
+            if first_byte.is_ascii() && s.len() == 1 {
+                return Some(first_byte as char);
             }
+            let utf8_len = first_byte.leading_ones() as usize;
+            if utf8_len != s.len() {
+                return None;
+            }
+
+            str::from_utf8(&s[..utf8_len]).ok()?.chars().next()
+        });
+
+        if let Some(single_char) = single_char {
+            return self.contains(single_char);
         }
+
         self.str_list
             .binary_search_by(|t| t.as_bytes().cmp(s))
             .is_ok()

components/collections/src/codepointtrie/error.rs

@@ -25,6 +25,9 @@ pub enum Error {
     /// [`CodePointTrie`](super::CodePointTrie) must be constructed from data vector long enough to accommodate fast-path access
     #[displaydoc("CodePointTrie must be constructed from data vector long enough to accommodate fast-path access")]
     DataTooShortForFastAccess,
+    /// [`CodePointTrie`](super::CodePointTrie) must be constructed from data vector long enough to accommodate direct ASCII access
+    #[displaydoc("CodePointTrie must be constructed from data vector long enough to accommodate direct ASCII access")]
+    DataTooShortForAsciiAccess,
 }
 
 impl core::error::Error for Error {}