Introduce new API grapheme_line_segmenter to replace scan API

Signed-off-by: Christian Parpart <[email protected]>

Author: christianparpart

src/libunicode/CMakeLists.txt

@@ -86,6 +86,7 @@ add_library(unicode ${LIBUNICODE_LIB_MODE}
     capi.cpp
     codepoint_properties.cpp
     emoji_segmenter.cpp
+    grapheme_line_segmenter.cpp
     grapheme_segmenter.cpp
     scan.cpp
     script_segmenter.cpp
@@ -103,6 +104,7 @@ set(public_headers
     codepoint_properties.h
     convert.h
     emoji_segmenter.h
+    grapheme_line_segmenter.h
     grapheme_segmenter.h
     intrinsics.h
     multistage_table_view.h
@@ -186,6 +188,7 @@ if(LIBUNICODE_TESTING)
         capi_test.cpp
         convert_test.cpp
         emoji_segmenter_test.cpp
+        grapheme_line_segmenter_test.cpp
         grapheme_segmenter_test.cpp
         run_segmenter_test.cpp
         scan_test.cpp

src/libunicode/grapheme_line_segmenter.cpp

@@ -0,0 +1,295 @@
+/**
+ * This file is part of the "libunicode" project
+ *   Copyright (c) 2023 Christian Parpart <[email protected]>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <libunicode/grapheme_line_segmenter.h>
+#include <libunicode/grapheme_segmenter.h>
+#include <libunicode/intrinsics.h>
+#include <libunicode/utf8.h>
+#include <libunicode/width.h>
+
+#include <algorithm>
+#include <cassert>
+
+namespace unicode
+{
+
+namespace
+{
+    [[maybe_unused]] int countTrailingZeroBits(unsigned int value) noexcept
+    {
+#if defined(_WIN32)
+        return _tzcnt_u32(value);
+#else
+        return __builtin_ctz(value);
+#endif
+    }
+
+    constexpr bool is_control(char ch) noexcept
+    {
+        return static_cast<uint8_t>(ch) < 0x20;
+    }
+
+    // Tests if given UTF-8 byte is part of a complex Unicode codepoint, that is, a value greater than U+7E.
+    constexpr bool is_complex(char ch) noexcept
+    {
+        return static_cast<uint8_t>(ch) & 0x80;
+    }
+
+    // Tests if given UTF-8 byte is a single US-ASCII text codepoint. This excludes control characters.
+    constexpr bool is_ascii(char ch) noexcept
+    {
+        return !is_control(ch) && !is_complex(ch);
+    }
+} // namespace
+
+size_t grapheme_line_segmenter::process_ascii() noexcept
+{
+    auto input = _buffer.data();
+    auto const end = _buffer.data() + std::min(_buffer.size(), _maxWidth);
+
+    intrinsics::m128i const ControlCodeMax = intrinsics::set1_epi8(0x20); // 0..0x1F
+    intrinsics::m128i const Complex = intrinsics::set1_epi8(-128);        // equals to 0x80 (0b1000'0000)
+
+    while (input < end - sizeof(intrinsics::m128i))
+    {
+        intrinsics::m128i batch = intrinsics::load_unaligned((intrinsics::m128i*) input);
+        intrinsics::m128i isControl = intrinsics::compare_less(batch, ControlCodeMax);
+        intrinsics::m128i isComplex = intrinsics::and128(batch, Complex);
+        // intrinsics::m128i isComplex = _mm_cmplt_epi8(batch, Complex);
+        intrinsics::m128i testPack = intrinsics::or128(isControl, isComplex);
+        if (int const check = intrinsics::movemask_epi8(testPack); check != 0)
+        {
+            int advance = countTrailingZeroBits(static_cast<unsigned>(check));
+            input += advance;
+            break;
+        }
+        input += sizeof(intrinsics::m128i);
+    }
+
+    while (input != end && is_ascii(*input))
+        ++input;
+
+    // if (static_cast<size_t>(distance(_buffer.data(), input)))
+    //     fmt::print(
+    //         "countAsciiTextChars: {} bytes: \"{}\"\n",
+    //         static_cast<size_t>(distance(_buffer.data(), input)),
+    //         (string_view(_buffer.data(), static_cast<size_t>(distance(_buffer.data(), input)))));
+
+    return static_cast<size_t>(std::distance(_buffer.data(), input));
+}
+
+void grapheme_line_segmenter::reset(std::string_view buffer, size_t maxWidth) noexcept
+{
+    _buffer = buffer;
+
+    _utf8 = {};
+    _lastCodepointHint = 0;
+
+    _maxWidth = maxWidth; // TODO: Not sure we want to do this here, nor need this at all.
+}
+
+void grapheme_line_segmenter::expand_buffer_by(size_t count) noexcept
+{
+    _buffer = std::string_view(_buffer.data(), _buffer.size() + count);
+}
+
+void grapheme_line_segmenter::move_forward_to(char const* pos) noexcept
+{
+    assert(_buffer.data() <= pos && pos <= _buffer.data() + _buffer.size());
+    auto const skippedBytesCount = static_cast<size_t>(pos - _buffer.data());
+    _buffer.remove_prefix(skippedBytesCount);
+}
+
+scan_result grapheme_line_segmenter::process(size_t maxWidth) noexcept
+{
+    if (_buffer.empty())
+        return scan_result { .text = _buffer.substr(0, 0), .width = 0 };
+
+    _maxWidth = maxWidth;
+
+    char const* start = _buffer.data();
+
+    // Number of bytes used in the current line.
+    size_t totalByteCountProcessed = 0;
+
+    // Number of width used in the current line.
+    size_t totalWidthProcessed = 0;
+
+    auto const makeResult = [&]() -> scan_result {
+        return scan_result { .text = std::string_view(start, totalByteCountProcessed),
+                             .width = totalWidthProcessed };
+    };
+
+    enum class State
+    {
+        ASCII,
+        ComplexUnicode,
+    };
+
+    State state = State::ASCII;
+
+    while (_maxWidth > 0 && !_buffer.empty())
+    {
+        state =
+            (_utf8.expectedLength != 0 || is_complex(_buffer.front())) ? State::ComplexUnicode : State::ASCII;
+
+        switch (state)
+        {
+            case State::ASCII: {
+                auto const count = process_ascii();
+                if (count == 0)
+                    return makeResult();
+                _events.on_ascii(_buffer.substr(0, count));
+                _maxWidth -= count;
+                totalWidthProcessed += count;
+                totalByteCountProcessed += count;
+                _buffer.remove_prefix(count);
+                state = State::ComplexUnicode;
+                break;
+            }
+            case State::ComplexUnicode: {
+                auto const sub = process_complex_unicode();
+                if (sub.width == 0)
+                    return makeResult();
+                totalWidthProcessed += sub.width;
+                totalByteCountProcessed += sub.text.size();
+                _buffer.remove_prefix(sub.text.size());
+                state = State::ASCII;
+                break;
+            }
+        }
+    }
+
+    return makeResult();
+}
+
+scan_result grapheme_line_segmenter::process_complex_unicode() noexcept
+{
+    size_t count = 0;
+
+    char const* start = _buffer.data();
+    char const* end = start + _buffer.size();
+    char const* input = start;
+    char const* clusterStart = start;
+    char const* lastCodepointStart = start;
+
+    unsigned byteCount = 0; // bytes consume for the current codepoint
+
+    // TODO: move currentClusterWidth to object's state?
+    size_t currentClusterWidth = 0; // current grapheme cluster's East Asian Width
+
+    char const* resultStart = _utf8.expectedLength ? start - _utf8.currentLength : start;
+    char const* resultEnd = resultStart;
+
+    while (input != end && count <= _maxWidth)
+    {
+        if (is_control(*input) || !is_complex(*input))
+        {
+            // ASCII control character or single US-ASCII text codepoint.
+
+            if (_utf8.expectedLength)
+            {
+                // Incomplete UTF-8 sequence hit. That's invalid as well.
+                ++count;
+                _events.on_invalid(std::string_view(input, input + 1));
+                _utf8 = {};
+            }
+
+            _lastCodepointHint = 0;
+            resultEnd = input;
+            break;
+        }
+
+        auto const result = from_utf8(_utf8, static_cast<uint8_t>(*input++));
+        ++byteCount;
+
+        if (holds_alternative<Incomplete>(result))
+            continue;
+
+        if (holds_alternative<Success>(result))
+        {
+            auto const prevCodepoint = _lastCodepointHint;
+            auto const nextCodepoint = std::get<Success>(result).value;
+            auto const nextWidth = std::max(currentClusterWidth, static_cast<size_t>(width(nextCodepoint)));
+            _lastCodepointHint = nextCodepoint;
+            if (grapheme_segmenter::breakable(prevCodepoint, nextCodepoint))
+            {
+                // Flush out current grapheme cluster's East Asian Width.
+                count += currentClusterWidth;
+                _maxWidth -= currentClusterWidth;
+
+                if (count + nextWidth > _maxWidth)
+                {
+                    // Currently scanned grapheme cluster won't fit. Break at start.
+                    currentClusterWidth = 0;
+                    input -= byteCount;
+                    break;
+                }
+
+                _events.on_grapheme_cluster(std::string_view(clusterStart, byteCount), currentClusterWidth);
+
+                // And start a new grapheme cluster.
+                currentClusterWidth = nextWidth;
+                clusterStart = lastCodepointStart;
+                lastCodepointStart = input - byteCount;
+                byteCount = 0;
+                resultEnd = input;
+            }
+            else
+            {
+                resultEnd = input;
+                // Increase width on VS16 but do not decrease on VS15.
+                if (nextCodepoint == 0xFE0F) // VS16
+                {
+                    currentClusterWidth = 2;
+                    if (count + currentClusterWidth > _maxWidth)
+                    {
+                        // Rewinding by {byteCount} bytes (overflow due to VS16).
+                        currentClusterWidth = 0;
+                        input = clusterStart;
+                        break;
+                    }
+                }
+
+                // Consumed {byteCount} bytes for grapheme cluster.
+                lastCodepointStart = input - byteCount;
+            }
+        }
+        else
+        {
+            assert(holds_alternative<Invalid>(result));
+            count++;
+            _events.on_invalid(std::string_view(clusterStart, byteCount));
+            currentClusterWidth = 0;
+            _lastCodepointHint = 0;
+            _utf8.expectedLength = 0;
+            byteCount = 0;
+            clusterStart = input;
+            resultEnd = input;
+        }
+    }
+    count += currentClusterWidth;
+
+    assert(resultStart <= resultEnd);
+
+    return scan_result { .text = std::string_view(resultStart,
+                                                  static_cast<size_t>(std::distance(resultStart, resultEnd))),
+                         .width = count };
+}
+
+ConvertResult grapheme_line_segmenter::process_single_byte(uint8_t byte) noexcept
+{
+    return from_utf8(_utf8, byte);
+}
+
+} // namespace unicode