WIP

Signed-off-by: Christian Parpart <[email protected]>

Author: christianparpart

src/libunicode/grapheme_line_segmenter.cpp

@@ -55,6 +55,7 @@ namespace
 void grapheme_line_segmenter::reset(std::string_view buffer) noexcept
 {
     _buffer = buffer;
+    _next = buffer.data();
 
     _utf8 = {};
     _lastCodepointHint = 0;
@@ -70,7 +71,7 @@ void grapheme_line_segmenter::move_forward_to(char const* pos) noexcept
 {
     assert(_buffer.data() <= pos && pos <= _buffer.data() + _buffer.size());
     auto const skippedBytesCount = static_cast<size_t>(pos - _buffer.data());
-    _buffer.remove_prefix(skippedBytesCount);
+    _next += skippedBytesCount;
     _lastCodepointHint = 0;
     _currentClusterWidth = 0;
     _utf8 = {};
@@ -83,7 +84,7 @@ grapheme_line_segmenter::result_type grapheme_line_segmenter::process(unsigned m
     if (_buffer.empty())
         return result_type { .text = _buffer.substr(0, 0), .width = 0 };
 
-    char const* start = _buffer.data();
+    char const* start = _next;
     char const* const resultStart = _utf8.expectedLength ? start - _utf8.currentLength : start;
 
     // Number of bytes used in the current line.
@@ -118,17 +119,20 @@ grapheme_line_segmenter::result_type grapheme_line_segmenter::process(unsigned m
                 maxWidth -= count;
                 totalWidthProcessed += count;
                 totalByteCountProcessed += count;
-                _buffer.remove_prefix(count);
+                _next += count;
                 break;
             }
             case State::ComplexUnicode: {
                 auto const sub = process_complex_unicode(maxWidth);
-                if (sub.width == 0)
+                if (sub.graphemeClusterCount == 0)
+                {
+                    _next += sub.byteCount;
                     return makeResult();
-                maxWidth -= sub.width;
-                totalWidthProcessed += sub.width;
-                totalByteCountProcessed += sub.text.size();
-                _buffer.remove_prefix(sub.text.size());
+                }
+                maxWidth -= sub.graphemeClusterCount;
+                totalWidthProcessed += sub.graphemeClusterCount;
+                totalByteCountProcessed += sub.byteCount;
+                _next += sub.byteCount;
                 break;
             }
         }
@@ -137,7 +141,7 @@ grapheme_line_segmenter::result_type grapheme_line_segmenter::process(unsigned m
     return makeResult();
 }
 
-unsigned grapheme_line_segmenter::process_ascii(unsigned maxWidth) noexcept
+unsigned grapheme_line_segmenter::process_ascii(unsigned maxWidth) const noexcept
 {
     auto input = _buffer.data();
     auto const end = _buffer.data() + std::min(static_cast<unsigned>(_buffer.size()), maxWidth);
@@ -173,20 +177,19 @@ unsigned grapheme_line_segmenter::process_ascii(unsigned maxWidth) noexcept
     return static_cast<unsigned>(std::distance(_buffer.data(), input));
 }
 
-grapheme_line_segmenter::result_type grapheme_line_segmenter::process_complex_unicode(
-    unsigned maxWidth) noexcept
+auto grapheme_line_segmenter::process_complex_unicode(unsigned maxWidth) noexcept -> unicode_process_result
 {
     char const* const start = _buffer.data();
     char const* const end = start + _buffer.size();
 
-    char const* input = start;              // current input processing position
+    char const* input = _next;              // current input processing position
     char const* clusterStart = start;       // start position of current grapheme cluster
     char const* lastCodepointStart = start; // start position of last codepoint
     unsigned consumedWidth = 0;             // width consumed for the current line
     unsigned currentCodepointLength = 0;    // bytes consumed for the current codepoint
 
     char const* const lastClusterStart =
-        _utf8.expectedLength ? start - _utf8.currentLength : start; // start position of last grapheme cluster
+        _utf8.expectedLength ? input - _utf8.currentLength : input; // start position of last grapheme cluster
 
     char const* lastClusterEnd = lastClusterStart; // end position of last grapheme cluster
 
@@ -228,8 +231,7 @@ grapheme_line_segmenter::result_type grapheme_line_segmenter::process_complex_un
         {
             auto const prevCodepoint = _lastCodepointHint;
             auto const nextCodepoint = std::get<Success>(result).value;
-            auto const nextWidth =
-                std::max(_currentClusterWidth, static_cast<unsigned>(unicode::width(nextCodepoint)));
+            auto const nextWidth = std::max(_currentClusterWidth, static_cast<unsigned>(unicode::width(nextCodepoint)));
             _lastCodepointHint = nextCodepoint;
             if (grapheme_segmenter::breakable(prevCodepoint, nextCodepoint))
             {
@@ -293,16 +295,15 @@ grapheme_line_segmenter::result_type grapheme_line_segmenter::process_complex_un
 
     _currentClusterWidth = 0;
 
-    // if (currentCodepointLength <= _buffer.size())
-    //     _buffer.remove_prefix(currentCodepointLength);
-    // else
-    //     abort();
+    _next = input;
 
     assert(lastClusterStart <= lastClusterEnd);
 
     auto const resultLength = static_cast<size_t>(std::distance(lastClusterStart, lastClusterEnd));
     printf("lastClusterEnd: %p, size: %zu\n", (void*) lastClusterEnd, resultLength);
-    return result_type { .text = std::string_view(lastClusterStart, resultLength), .width = consumedWidth };
+    return unicode_process_result { .graphemeClusterCount = consumedWidth, .byteCount = resultLength };
+    // return result_type { .text = std::string_view(lastClusterStart, resultLength), .width = consumedWidth
+    // };
 }
 
 ConvertResult grapheme_line_segmenter::process_single_byte(uint8_t byte) noexcept

src/libunicode/grapheme_line_segmenter.h

@@ -71,23 +71,16 @@ class grapheme_line_segmenter final
         virtual void on_grapheme_cluster(std::string_view cluster, unsigned width) = 0;
     };
 
-    class null_listener final: public event_listener
+    class null_event_listener final: public event_listener
     {
       public:
         void on_invalid(std::string_view) override {}
         void on_ascii(std::string_view) override {}
-        void on_grapheme_cluster(std::string_view sequence, unsigned width) override
-        {
-            printf("on_grapheme_cluster(%u): %s\n", width, std::string(sequence).c_str());
-        }
-
-        static null_listener& instance() noexcept
-        {
-            static null_listener instance;
-            return instance;
-        }
+        void on_grapheme_cluster(std::string_view, unsigned) override {}
     };
 
+    static inline null_event_listener null_listener {};
+
     explicit grapheme_line_segmenter(event_listener& events) noexcept: _events(events) {}
 
     /**
@@ -100,6 +93,8 @@ class grapheme_line_segmenter final
 
     /**
      * Expands the internal buffer by the given number of bytes.
+     *
+     * @param count the number of bytes to expand the buffer by
      */
     void expand_buffer_by(size_t count) noexcept;
 
@@ -108,6 +103,8 @@ class grapheme_line_segmenter final
      *
      * A call to this function will also reset the internal UTF-8 decoding state,
      * and will reset the last codepoint hint.
+     *
+     * @p pos must be a pointer to a position within the current buffer
      */
     void move_forward_to(char const* pos) noexcept;
 
@@ -179,7 +176,16 @@ class grapheme_line_segmenter final
      *
      * @returns the number of ASCII characters processed (equal to the sum of East Asian Width for each).
      */
-    unsigned process_ascii(unsigned maxWidth) noexcept;
+    unsigned process_ascii(unsigned maxWidth) const noexcept;
+
+    struct unicode_process_result
+    {
+        // The number of grapheme clusters processed.
+        unsigned graphemeClusterCount;
+
+        // The number of bytes processed.
+        size_t byteCount;
+    };
 
     /**
      * Processes up to maxWidth grapheme clusters.
@@ -194,7 +200,7 @@ class grapheme_line_segmenter final
      *
      * @see process()
      */
-    result_type process_complex_unicode(unsigned maxWidth) noexcept;
+    auto process_complex_unicode(unsigned maxWidth) noexcept -> unicode_process_result;
 
     event_listener& _events;
 
@@ -212,6 +218,9 @@ class grapheme_line_segmenter final
     // The buffer to scan. Its underlying storage must be used by the subsequent calls to process().
     // While consuming the buffer, the front of the buffer will be moved forward.
     std::string_view _buffer;
+
+    // processing position within the buffer
+    char const* _next {};
     // }}}
 };
 

src/libunicode/grapheme_line_segmenter_test.cpp

@@ -70,12 +70,47 @@ class event_recorder final: public grapheme_line_segmenter::event_listener
 
 scan_result scan_text(std::string_view text, unsigned width)
 {
-    grapheme_line_segmenter segmenter { grapheme_line_segmenter::null_listener::instance() };
+    grapheme_line_segmenter segmenter { grapheme_line_segmenter::null_listener };
     segmenter.reset(text);
     auto result = segmenter.process(width);
     UNSCOPED_INFO("result: " << result);
     return result;
 }
+
+enum class NumericEscape
+{
+    Octal,
+    Hex
+};
+
+std::string escape(uint8_t ch, NumericEscape numericEscape = NumericEscape::Hex)
+{
+    switch (ch)
+    {
+        case '\\': return "\\\\";
+        case 0x1B: return "\\e";
+        case '\t': return "\\t";
+        case '\r': return "\\r";
+        case '\n': return "\\n";
+        case '"': return "\\\"";
+        default:
+            if (0x20 <= ch && ch < 0x7E)
+                return fmt::format("{}", static_cast<char>(ch));
+            else if (numericEscape == NumericEscape::Hex)
+                return fmt::format("\\x{:02x}", static_cast<uint8_t>(ch) & 0xFF);
+            else
+                return fmt::format("\\{:03o}", static_cast<uint8_t>(ch) & 0xFF);
+    }
+}
+
+inline std::string e(std::string_view s, NumericEscape numericEscape = NumericEscape::Hex)
+{
+    auto result = std::string {};
+    for (char c: s)
+        result += escape(static_cast<uint8_t>(c), numericEscape);
+    return result;
+}
+
 } // namespace
 
 TEST_CASE("grapheme_line_segmenter.ascii.empty")
@@ -120,14 +155,14 @@ TEST_CASE("grapheme_line_segmenter.complex.grapheme_cluster.2")
 {
     auto const familyEmoji8 = u8(FamilyEmoji) + u8(FamilyEmoji);
     auto const result = scan_text(familyEmoji8, 80);
-    CHECK(result == scan_result { familyEmoji8, 4 });
+    CHECK(result == scan_result { .text = familyEmoji8, .width = 4 });
 }
 
 TEST_CASE("grapheme_line_segmenter.complex.mixed")
 {
     auto const text = u8(FamilyEmoji) + "ABC"s + u8(FamilyEmoji);
     auto const result = scan_text(text, 80);
-    CHECK(result == scan_result { text, 7 });
+    CHECK(result == scan_result { .text = text, .width = 7 });
 }
 
 TEST_CASE("grapheme_line_segmenter.complex.half-overflowing")
@@ -148,12 +183,13 @@ TEST_CASE("grapheme_line_segmenter.complex.half-overflowing")
 TEST_CASE("grapheme_line_segmenter.complex.sliced_calls")
 {
     // auto const text = u8(SmileyEmoji) + "\033\\0123456789ABCDEF"s; // U+1F600
-    auto const text = "\xF0\x9F\x98\x80\033\\0123456789ABCDEF"sv; // U+1F600
+    auto constexpr text = "\xF0\x9F\x98\x80\033\\0123456789ABCDEF"sv; // U+1F600
     auto constexpr splitOffset = 3;
-    auto const chunkOne = std::string_view(text.data(), splitOffset);
-    auto const chunkTwo = std::string_view(text.data() + splitOffset, text.size() - splitOffset);
+    auto constexpr chunkOne = text.substr(0, splitOffset);
+    auto constexpr chunkTwo = text.substr(splitOffset);
+    REQUIRE(chunkTwo.data() == chunkOne.data() + chunkOne.size());
 
-    grapheme_line_segmenter segmenter { grapheme_line_segmenter::null_listener::instance() };
+    auto segmenter = grapheme_line_segmenter { grapheme_line_segmenter::null_listener };
     segmenter.reset(chunkOne);
     auto result = segmenter.process(3);
 
@@ -168,5 +204,5 @@ TEST_CASE("grapheme_line_segmenter.complex.sliced_calls")
 
     REQUIRE(segmenter.utf8_state().expectedLength == 0);
     CHECK(result.width == 2);
-    REQUIRE(result.text == u8(SmileyEmoji));
+    REQUIRE(e(result.text) == e(u8(SmileyEmoji)));
 }