Don't reexport iterator types at the top level of segmenter (#6409)

We should keep the top level clean: only the segmenters and their
borrowed variants.

This also ends up exposing more of the public-but-behind-private-module
"line break type" etc markers.

Author: Manishearth

components/segmenter/src/complex/mod.rs

@@ -20,17 +20,13 @@ use lstm::*;
 #[allow(clippy::large_enum_variant)]
 enum DictOrLstm {
     Dict(DataPayload<UCharDictionaryBreakDataV1>),
-    #[cfg(not(feature = "lstm"))]
-    Lstm(core::convert::Infallible),
     #[cfg(feature = "lstm")]
     Lstm(DataPayload<SegmenterLstmAutoV1>),
 }
 
 #[derive(Debug, Clone, Copy)]
 enum DictOrLstmBorrowed<'data> {
     Dict(&'data UCharDictionaryBreakData<'data>),
-    #[cfg(not(feature = "lstm"))]
-    Lstm(core::convert::Infallible),
     #[cfg(feature = "lstm")]
     Lstm(&'data LstmData<'data>),
 }
@@ -40,8 +36,6 @@ fn borrow_dictor(dict_or: &DictOrLstm) -> DictOrLstmBorrowed<'_> {
         DictOrLstm::Dict(dict) => DictOrLstmBorrowed::Dict(dict.get()),
         #[cfg(feature = "lstm")]
         DictOrLstm::Lstm(lstm) => DictOrLstmBorrowed::Lstm(lstm.get()),
-        #[cfg(not(feature = "lstm"))]
-        DictOrLstm::Lstm(infallible) => DictOrLstmBorrowed::Lstm(*infallible),
     }
 }
 
@@ -50,8 +44,6 @@ fn fromstatic_dictor(dict_or: DictOrLstmBorrowed<'static>) -> DictOrLstm {
         DictOrLstmBorrowed::Dict(dict) => DictOrLstm::Dict(DataPayload::from_static_ref(dict)),
         #[cfg(feature = "lstm")]
         DictOrLstmBorrowed::Lstm(lstm) => DictOrLstm::Lstm(DataPayload::from_static_ref(lstm)),
-        #[cfg(not(feature = "lstm"))]
-        DictOrLstmBorrowed::Lstm(infallible) => DictOrLstm::Lstm(infallible),
     }
 }
 
@@ -131,8 +123,6 @@ impl<'data> ComplexPayloadsBorrowed<'data> {
                     let seg = LstmSegmenter::new(lstm, self.grapheme);
                     result.extend(seg.segment_str(slice).map(|n| offset + n));
                 }
-                #[cfg(not(feature = "lstm"))]
-                Some(DictOrLstmBorrowed::Lstm(_infallible)) => {} // should be refutable
                 None => {
                     result.push(offset + slice.len());
                 }
@@ -156,8 +146,6 @@ impl<'data> ComplexPayloadsBorrowed<'data> {
                     let seg = LstmSegmenter::new(lstm, self.grapheme);
                     result.extend(seg.segment_utf16(slice).map(|n| offset + n));
                 }
-                #[cfg(not(feature = "lstm"))]
-                Some(DictOrLstmBorrowed::Lstm(_infallible)) => {} // should be refutable
                 None => {
                     result.push(offset + slice.len());
                 }

components/segmenter/src/lib.rs

@@ -127,24 +127,24 @@ mod indices;
 mod iterator_helpers;
 mod rule_segmenter;
 
-mod grapheme;
-mod line;
-mod sentence;
-mod word;
+/// [`GraphemeClusterSegmenter`] and its related iterators, borrowed types, and options.
+pub mod grapheme;
+/// [`LineSegmenter`] and its related iterators, borrowed types, and options.
+pub mod line;
+/// [`SentenceSegmenter`] and its related iterators, borrowed types, and options.
+pub mod sentence;
+/// [`WordSegmenter`] and its related iterators, borrowed types, and options.
+pub mod word;
 
 pub mod provider;
 
 // Main Segmenter and BreakIterator public types
-pub use crate::grapheme::GraphemeClusterBreakIterator;
 pub use crate::grapheme::GraphemeClusterSegmenter;
 pub use crate::grapheme::GraphemeClusterSegmenterBorrowed;
-pub use crate::line::LineBreakIterator;
 pub use crate::line::LineSegmenter;
 pub use crate::line::LineSegmenterBorrowed;
-pub use crate::sentence::SentenceBreakIterator;
 pub use crate::sentence::SentenceSegmenter;
 pub use crate::sentence::SentenceSegmenterBorrowed;
-pub use crate::word::WordBreakIterator;
 pub use crate::word::WordSegmenter;
 pub use crate::word::WordSegmenterBorrowed;
 
@@ -160,24 +160,13 @@ pub mod options {
     pub use crate::word::WordType;
 }
 
-// Typedefs
-pub use crate::grapheme::GraphemeClusterBreakIteratorLatin1;
-pub use crate::grapheme::GraphemeClusterBreakIteratorPotentiallyIllFormedUtf8;
-pub use crate::grapheme::GraphemeClusterBreakIteratorUtf16;
-pub use crate::grapheme::GraphemeClusterBreakIteratorUtf8;
-pub use crate::line::LineBreakIteratorLatin1;
-pub use crate::line::LineBreakIteratorPotentiallyIllFormedUtf8;
-pub use crate::line::LineBreakIteratorUtf16;
-pub use crate::line::LineBreakIteratorUtf8;
-pub use crate::sentence::SentenceBreakIteratorLatin1;
-pub use crate::sentence::SentenceBreakIteratorPotentiallyIllFormedUtf8;
-pub use crate::sentence::SentenceBreakIteratorUtf16;
-pub use crate::sentence::SentenceBreakIteratorUtf8;
-pub use crate::word::WordBreakIteratorLatin1;
-pub use crate::word::WordBreakIteratorPotentiallyIllFormedUtf8;
-pub use crate::word::WordBreakIteratorUtf16;
-pub use crate::word::WordBreakIteratorUtf8;
-pub use crate::word::WordBreakIteratorWithWordType;
+/// Largely-internal scaffolding types (You should very rarely need to reference these directly)
+pub mod scaffold {
+    pub use crate::rule_segmenter::{
+        RuleBreakType, RuleBreakTypeLatin1, RuleBreakTypePotentiallyIllFormedUtf8,
+        RuleBreakTypeUtf16, RuleBreakTypeUtf8,
+    };
+}
 
 pub(crate) mod private {
     /// Trait marking other traits that are considered unstable and should not generally be

components/segmenter/src/line.rs

@@ -1102,6 +1102,7 @@ impl<'s, Y: LineBreakType<'s>> LineBreakIterator<'_, 's, Y> {
 
 #[derive(Debug)]
 #[non_exhaustive]
+/// [`LineBreakType`] for UTF-8 strings
 pub struct LineBreakTypeUtf8;
 
 impl crate::private::Sealed for LineBreakTypeUtf8 {}
@@ -1137,6 +1138,7 @@ impl<'s> LineBreakType<'s> for LineBreakTypeUtf8 {
 
 #[derive(Debug)]
 #[non_exhaustive]
+/// [`LineBreakType`] for potentially ill-formed UTF-8 strings
 pub struct LineBreakTypePotentiallyIllFormedUtf8;
 
 impl crate::private::Sealed for LineBreakTypePotentiallyIllFormedUtf8 {}
@@ -1225,6 +1227,7 @@ where
 
 #[derive(Debug)]
 #[non_exhaustive]
+/// [`LineBreakType`] for Latin-1 strings
 pub struct LineBreakTypeLatin1;
 impl crate::private::Sealed for LineBreakTypeLatin1 {}
 
@@ -1257,6 +1260,7 @@ impl<'s> LineBreakType<'s> for LineBreakTypeLatin1 {
 
 #[derive(Debug)]
 #[non_exhaustive]
+/// [`LineBreakType`] for UTF-16 strings
 pub struct LineBreakTypeUtf16;
 impl crate::private::Sealed for LineBreakTypeUtf16 {}
 

components/segmenter/src/provider/mod.rs

@@ -121,6 +121,8 @@ icu_provider::data_marker!(
     is_singleton = true
 );
 
+pub use crate::word::inner::WordTypeULE;
+
 #[cfg(feature = "datagen")]
 /// The latest minimum set of markers required by this component.
 pub const MARKERS: &[DataMarkerInfo] = &[

components/segmenter/src/rule_segmenter.rs

@@ -269,6 +269,7 @@ impl<'s, Y: RuleBreakType<'s> + ?Sized> RuleBreakIterator<'_, 's, Y> {
 
 #[derive(Debug)]
 #[non_exhaustive]
+/// [`RuleBreakType`] for UTF-8 strings
 pub struct RuleBreakTypeUtf8;
 
 impl crate::private::Sealed for RuleBreakTypeUtf8 {}
@@ -291,6 +292,7 @@ impl<'s> RuleBreakType<'s> for RuleBreakTypeUtf8 {
 
 #[derive(Debug)]
 #[non_exhaustive]
+/// [`RuleBreakType`] for potentially ill-formed UTF-8 strings
 pub struct RuleBreakTypePotentiallyIllFormedUtf8;
 
 impl crate::private::Sealed for RuleBreakTypePotentiallyIllFormedUtf8 {}
@@ -313,6 +315,7 @@ impl<'s> RuleBreakType<'s> for RuleBreakTypePotentiallyIllFormedUtf8 {
 
 #[derive(Debug)]
 #[non_exhaustive]
+/// [`RuleBreakType`] for Latin-1 strings
 pub struct RuleBreakTypeLatin1;
 
 impl crate::private::Sealed for RuleBreakTypeLatin1 {}
@@ -335,6 +338,7 @@ impl<'s> RuleBreakType<'s> for RuleBreakTypeLatin1 {
 
 #[derive(Debug)]
 #[non_exhaustive]
+/// [`RuleBreakType`] for UTF-16 strings
 pub struct RuleBreakTypeUtf16;
 
 impl crate::private::Sealed for RuleBreakTypeUtf16 {}

components/segmenter/src/word.rs

@@ -51,20 +51,27 @@ pub struct WordBreakIterator<'data, 's, Y: RuleBreakType<'s> + ?Sized>(
 
 derive_usize_iterator_with_type!(WordBreakIterator, 'data);
 
-/// The word type tag that is returned by [`WordBreakIterator::word_type()`].
-#[non_exhaustive]
-#[derive(Copy, Clone, PartialEq, Debug)]
-#[repr(u8)]
-#[zerovec::make_ule(WordTypeULE)]
-pub enum WordType {
-    /// No category tag.
-    None = 0,
-    /// Number category tag.
-    Number = 1,
-    /// Letter category tag, including CJK.
-    Letter = 2,
+/// Hide ULE type
+pub(crate) mod inner {
+    /// The word type tag that is returned by [`WordBreakIterator::word_type()`].
+    ///
+    /// [`WordBreakIterator::word_type()`]: super::WordBreakIterator::word_type
+    #[non_exhaustive]
+    #[derive(Copy, Clone, PartialEq, Debug)]
+    #[repr(u8)]
+    #[zerovec::make_ule(WordTypeULE)]
+    pub enum WordType {
+        /// No category tag.
+        None = 0,
+        /// Number category tag.
+        Number = 1,
+        /// Letter category tag, including CJK.
+        Letter = 2,
+    }
 }
 
+pub use inner::WordType;
+
 impl WordType {
     /// Whether the segment is word-like; word-like segments include numbers, as
     /// well as segments made up of letters (including CJKV ideographs).
@@ -580,6 +587,7 @@ impl WordSegmenterBorrowed<'static> {
 
 #[derive(Debug)]
 #[non_exhaustive]
+/// [`RuleBreakType`] for word-breaking UTF-16 strings
 pub struct WordBreakTypeUtf8;
 impl crate::private::Sealed for WordBreakTypeUtf8 {}
 
@@ -601,6 +609,7 @@ impl<'s> RuleBreakType<'s> for WordBreakTypeUtf8 {
 
 #[derive(Debug)]
 #[non_exhaustive]
+/// Word breaks for word-breaking potentially ill-formed UTF-8 strings
 pub struct WordBreakTypePotentiallyIllFormedUtf8;
 impl crate::private::Sealed for WordBreakTypePotentiallyIllFormedUtf8 {}
 
@@ -677,6 +686,7 @@ where
 
 #[derive(Debug)]
 #[non_exhaustive]
+/// Word breaks word-breaking for UTF-16 strings
 pub struct WordBreakTypeUtf16;
 
 impl crate::private::Sealed for WordBreakTypeUtf16 {}