Add UTF16 handling to ixdtf (unicode-org#6577)

This PR adds UTF16 handling to `ixdtf`.

For reference, an issue was recently filed on `temporal_rs`
(boa-dev/temporal#275). It was noted that `ixdtf` needs to support
UTF16.

The major changes occur in `utils/ixdtf/src/core.rs`.

Just to note as well, since this would be a breaking change that adds
the import `ixdtf::Slice`, the records module was moved out of parsers
module and into crate module to clean up imports.

Author: nekevss

Cargo.lock

@@ -192,7 +192,7 @@ databake = { version = "0.2.0", path = "utils/databake", default-features = fals
 databake-derive = { version = "0.2.0", path = "utils/databake/derive", default-features = false }
 deduplicating_array = { version = "0.1.6", path = "utils/deduplicating_array", default-features = false }
 fixed_decimal = { version = "0.7.0", path = "utils/fixed_decimal", default-features = false }
-ixdtf = { version = "0.5.0", path = "utils/ixdtf", default-features = false }
+ixdtf = { version = "0.6.0-dev", path = "utils/ixdtf", default-features = false }
 litemap = { version = "0.8.0", path = "utils/litemap", default-features = false }
 potential_utf = { version = "0.1.1", path = "utils/potential_utf", default-features = false }
 tzif = { version = "0.4.0", path = "utils/tzif", default-features = false }

Cargo.toml

@@ -6,8 +6,9 @@ use core::str::FromStr;
 
 use crate::{AsCalendar, Calendar, Date, Iso, RangeError};
 use icu_locale_core::preferences::extensions::unicode::keywords::CalendarAlgorithm;
-use ixdtf::parsers::records::IxdtfParseRecord;
+use ixdtf::encoding::Utf8;
 use ixdtf::parsers::IxdtfParser;
+use ixdtf::records::IxdtfParseRecord;
 use ixdtf::ParseError as Rfc9557Error;
 
 /// An error returned from parsing an RFC 9557 string to an `icu_calendar` type.
@@ -93,7 +94,7 @@ impl<A: AsCalendar> Date<A> {
 
     #[doc(hidden)]
     pub fn try_from_ixdtf_record(
-        ixdtf_record: &IxdtfParseRecord,
+        ixdtf_record: &IxdtfParseRecord<'_, Utf8>,
         calendar: A,
     ) -> Result<Self, ParseError> {
         let date_record = ixdtf_record.date.ok_or(ParseError::MissingFields)?;

components/calendar/src/ixdtf.rs

@@ -13,12 +13,11 @@ use core::str::FromStr;
 use icu_calendar::{AnyCalendarKind, AsCalendar, Date, DateError, Iso, RangeError};
 use icu_locale_core::subtags::subtag;
 use ixdtf::{
-    parsers::{
-        records::{
-            DateRecord, IxdtfParseRecord, TimeRecord, TimeZoneAnnotation, TimeZoneRecord,
-            UtcOffsetRecord, UtcOffsetRecordOrZ,
-        },
-        IxdtfParser,
+    encoding::Utf8,
+    parsers::IxdtfParser,
+    records::{
+        DateRecord, IxdtfParseRecord, TimeRecord, TimeZoneAnnotation, TimeZoneRecord,
+        UtcOffsetRecord, UtcOffsetRecordOrZ,
     },
     ParseError as Rfc9557ParseError,
 };
@@ -136,7 +135,9 @@ struct Intermediate<'a> {
 }
 
 impl<'a> Intermediate<'a> {
-    fn try_from_ixdtf_record(ixdtf_record: &'a IxdtfParseRecord) -> Result<Self, ParseError> {
+    fn try_from_ixdtf_record(
+        ixdtf_record: &'a IxdtfParseRecord<'_, Utf8>,
+    ) -> Result<Self, ParseError> {
         let (offset, is_z, iana_identifier) = match ixdtf_record {
             // empty
             IxdtfParseRecord {
@@ -740,7 +741,9 @@ impl Time {
         Self::try_from_ixdtf_record(&ixdtf_record)
     }
 
-    fn try_from_ixdtf_record(ixdtf_record: &IxdtfParseRecord) -> Result<Self, ParseError> {
+    fn try_from_ixdtf_record(
+        ixdtf_record: &IxdtfParseRecord<'_, Utf8>,
+    ) -> Result<Self, ParseError> {
         let time_record = ixdtf_record.time.ok_or(ParseError::MissingFields)?;
         Self::try_from_time_record(&time_record)
     }

components/time/src/ixdtf.rs

@@ -5,7 +5,7 @@
 [package]
 name = "ixdtf"
 description = "Parser for Internet eXtended DateTime Format"
-version = "0.5.0"
+version = "0.6.0-dev"
 
 authors.workspace = true
 categories.workspace = true

utils/ixdtf/Cargo.toml

@@ -0,0 +1,189 @@
+// This file is part of ICU4X. For terms of use, please see the file
+// called LICENSE at the top level of the ICU4X source tree
+// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
+
+//! Core functionality for `ixdtf`'s parsers
+
+use crate::{ParseError, ParserResult};
+
+mod private {
+    pub trait Sealed {}
+}
+
+/// A trait for defining various supported encodings
+/// and implementing functionality that is encoding
+/// sensitive / specific.
+pub trait EncodingType: private::Sealed {
+    type CodeUnit: PartialEq + core::fmt::Debug + Clone;
+
+    /// Get a slice from the underlying source using for start..end
+    fn slice(source: &[Self::CodeUnit], start: usize, end: usize) -> Option<&[Self::CodeUnit]>;
+
+    /// Retrieve the provided code unit index and returns the value as an ASCII byte
+    /// or None if the value is not ASCII representable.
+    fn get_ascii(source: &[Self::CodeUnit], index: usize) -> ParserResult<Option<u8>>;
+
+    /// Checks for the known calendar annotation key `u-ca`.
+    fn check_calendar_key(key: &[Self::CodeUnit]) -> bool;
+}
+
+/// A marker type that signals a parser should parse the source as UTF-16 bytes.
+#[derive(Debug, PartialEq, Clone)]
+#[allow(clippy::exhaustive_structs)] // ZST Marker trait, no fields should be added
+pub struct Utf16;
+
+impl private::Sealed for Utf16 {}
+
+impl EncodingType for Utf16 {
+    type CodeUnit = u16;
+    fn slice(source: &[Self::CodeUnit], start: usize, end: usize) -> Option<&[Self::CodeUnit]> {
+        source.get(start..end)
+    }
+
+    fn get_ascii(source: &[Self::CodeUnit], index: usize) -> ParserResult<Option<u8>> {
+        source.get(index).copied().map(to_ascii_byte).transpose()
+    }
+
+    fn check_calendar_key(key: &[Self::CodeUnit]) -> bool {
+        key == [0x75, 0x2d, 0x63, 0x61]
+    }
+}
+
+#[inline]
+fn to_ascii_byte(b: u16) -> ParserResult<u8> {
+    if !(0x01..0x7F).contains(&b) {
+        return Err(ParseError::NonAsciiCodePoint);
+    }
+    Ok(b as u8)
+}
+
+/// A marker type that signals a parser should parse the source as UTF-8 bytes.
+#[derive(Debug, PartialEq, Clone)]
+#[allow(clippy::exhaustive_structs)] // ZST Marker trait, no fields should be added.
+pub struct Utf8;
+
+impl private::Sealed for Utf8 {}
+
+impl EncodingType for Utf8 {
+    type CodeUnit = u8;
+
+    fn slice<'a>(source: &[Self::CodeUnit], start: usize, end: usize) -> Option<&[Self::CodeUnit]> {
+        source.get(start..end)
+    }
+
+    fn get_ascii(source: &[Self::CodeUnit], index: usize) -> ParserResult<Option<u8>> {
+        Ok(source.get(index).copied())
+    }
+
+    fn check_calendar_key(key: &[Self::CodeUnit]) -> bool {
+        key == "u-ca".as_bytes()
+    }
+}
+
+// ==== Mini cursor implementation for Iso8601 targets ====
+
+/// `Cursor` is a small cursor implementation for parsing Iso8601 grammar.
+#[derive(Debug)]
+pub(crate) struct Cursor<'a, T: EncodingType> {
+    pos: usize,
+    source: &'a [T::CodeUnit],
+}
+
+impl<'a, T: EncodingType> Cursor<'a, T> {
+    /// Create a new cursor from a source UTF8 string.
+    #[must_use]
+    pub fn new(source: &'a [T::CodeUnit]) -> Self {
+        Self { pos: 0, source }
+    }
+
+    /// Returns a string value from a slice of the cursor.
+    pub(crate) fn slice(&self, start: usize, end: usize) -> Option<&'a [T::CodeUnit]> {
+        T::slice(self.source, start, end)
+    }
+
+    /// Get current position
+    pub(crate) const fn pos(&self) -> usize {
+        self.pos
+    }
+
+    /// Peek the value at next position (current + 1).
+    pub(crate) fn peek(&self) -> ParserResult<Option<u8>> {
+        self.peek_n(1)
+    }
+
+    /// Returns current position in source as `char`.
+    pub(crate) fn current(&self) -> ParserResult<Option<u8>> {
+        self.peek_n(0)
+    }
+
+    /// Peeks the value at `n` as a `char`.
+    pub(crate) fn peek_n(&self, n: usize) -> ParserResult<Option<u8>> {
+        T::get_ascii(self.source, self.pos + n)
+    }
+
+    /// Runs the provided check on the current position.
+    pub(crate) fn check<F>(&self, f: F) -> ParserResult<Option<bool>>
+    where
+        F: FnOnce(u8) -> bool,
+    {
+        Ok(self.current()?.map(f))
+    }
+
+    /// Runs the provided check on current position returns the default value if None.
+    pub(crate) fn check_or<F>(&self, default: bool, f: F) -> ParserResult<bool>
+    where
+        F: FnOnce(u8) -> bool,
+    {
+        Ok(self.current()?.map_or(default, f))
+    }
+
+    /// Returns `Cursor`'s current char and advances to the next position.
+    pub(crate) fn next(&mut self) -> ParserResult<Option<u8>> {
+        let result = self.current();
+        self.advance_n(1);
+        result
+    }
+
+    /// Returns the next value as a digit
+    ///
+    /// # Errors
+    ///   - Returns an AbruptEnd error if cursor ends.
+    pub(crate) fn next_digit(&mut self) -> ParserResult<Option<u8>> {
+        let ascii_char = self.next_or(ParseError::AbruptEnd { location: "digit" })?;
+        if ascii_char.is_ascii_digit() {
+            Ok(Some(ascii_char - 48))
+        } else {
+            Ok(None)
+        }
+    }
+
+    /// A utility next method that returns an `AbruptEnd` error if invalid.
+    pub(crate) fn next_or(&mut self, err: ParseError) -> ParserResult<u8> {
+        self.next()?.ok_or(err)
+    }
+
+    /// Advances the cursor's position by n code points.
+    pub(crate) fn advance_n(&mut self, n: usize) {
+        self.pos += n;
+    }
+
+    // Advances the cursor by 1 code point.
+    pub(crate) fn advance(&mut self) {
+        self.advance_n(1)
+    }
+
+    /// Utility function to advance when a condition is true
+    pub(crate) fn advance_if(&mut self, condition: bool) {
+        if condition {
+            self.advance();
+        }
+    }
+
+    /// Closes the current cursor by checking if all contents have been consumed. If not, returns an error for invalid syntax.
+    pub(crate) fn close(&mut self) -> ParserResult<()> {
+        if self.pos < self.source.len() {
+            return Err(ParseError::InvalidEnd);
+        }
+        Ok(())
+    }
+}

utils/ixdtf/README.md

@@ -12,6 +12,8 @@ use displaydoc::Display;
 pub enum ParseError {
     #[displaydoc("Implementation error: this error must not throw.")]
     ImplAssert,
+    #[displaydoc("Code point was not ASCII")]
+    NonAsciiCodePoint,
     #[displaydoc("Invalid float while parsing fraction part.")]
     ParseFloat,
     #[displaydoc("Parsing ended abruptly while parsing {location}")]

utils/ixdtf/src/core.rs

@@ -19,9 +19,9 @@
 //! ## Example Usage
 //!
 //! ```
-//! use ixdtf::parsers::{
+//! use ixdtf::{
 //!     records::{Sign, TimeZoneRecord},
-//!     IxdtfParser,
+//!     parsers::IxdtfParser,
 //! };
 //!
 //! let ixdtf_str = "2024-03-02T08:48:00-05:00[America/New_York]";
@@ -73,9 +73,9 @@
 //! RFC 9557 updates the interpretation of `Z` to align with `-00:00`.
 //!
 //! ```rust
-//! use ixdtf::parsers::{
+//! use ixdtf::{
+//!     parsers::IxdtfParser,
 //!     records::{Sign, TimeZoneRecord},
-//!     IxdtfParser,
 //! };
 //!
 //! let ixdtf_str = "2024-03-02T08:48:00Z[America/New_York]";
@@ -134,9 +134,9 @@
 //! zone annotation if it is provided.
 //!
 //! ```rust
-//! use ixdtf::parsers::{
+//! use ixdtf::{
+//!     parsers::IxdtfParser,
 //!     records::{Sign, TimeZoneRecord},
-//!     IxdtfParser,
 //! };
 //!
 //! let zulu_offset = "2024-03-02T08:48:00Z[!America/New_York]";
@@ -289,7 +289,7 @@
 //! between the offset and annotation.
 //!
 //! ```rust
-//! use ixdtf::parsers::{IxdtfParser, records::TimeZoneRecord};
+//! use ixdtf::{parsers::IxdtfParser, records::TimeZoneRecord};
 //!
 //! let example_two = "2024-03-02T08:48:00+01:00[!America/New_York]";
 //!
@@ -388,12 +388,19 @@
     )
 )]
 
+pub(crate) mod core;
 mod error;
 pub mod parsers;
+pub mod records;
 
 extern crate alloc;
 
 pub use error::ParseError;
 
+/// This module contains the supported encoding for `ixdtf` parsing.
+pub mod encoding {
+    pub use crate::core::{Utf16, Utf8};
+}
+
 /// The `ixdtf` crate's Result type.
 pub type ParserResult<T> = Result<T, ParseError>;