Merge pull request #349 from funstory-ai/dev

feat: improve paragraph translation handling and unicode management in LLM processing

Author: awwaawwa

babeldoc/__init__.py

@@ -1 +1 @@
-__version__ = "0.3.41"
+__version__ = "0.3.42"

babeldoc/const.py

@@ -3,7 +3,7 @@
 import subprocess
 from pathlib import Path
 
-__version__ = "0.3.41"
+__version__ = "0.3.42"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

babeldoc/document_il/midend/il_translator_llm_only.py

@@ -224,6 +224,7 @@ def translate_paragraph(
         try:
             inputs = []
             llm_translate_trackers = []
+            paragraph_unicodes = []
             for i in range(len(batch_paragraph.paragraphs)):
                 paragraph = batch_paragraph.paragraphs[i]
                 tracker = batch_paragraph.trackers[i]
@@ -237,8 +238,16 @@ def translate_paragraph(
                 should_translate_paragraph.append(i)
                 llm_translate_trackers.append(llm_translate_tracker)
                 inputs.append(
-                    (text, translate_input, paragraph, tracker, llm_translate_tracker)
+                    (
+                        text,
+                        translate_input,
+                        paragraph,
+                        tracker,
+                        llm_translate_tracker,
+                        paragraph_unicodes,
+                    )
                 )
+                paragraph_unicodes.append(paragraph.unicode)
             if not inputs:
                 return
             json_format_input = []
@@ -454,6 +463,8 @@ def translate_paragraph(
                         paragraph_token_count = self.calc_token_count(
                             inputs[id_][2].unicode
                         )
+                        paragraph_unicodes = inputs[id_][5]
+                        inputs[id_][2].unicode = paragraph_unicodes[id_]
                         executor.submit(
                             self.il_translator.translate_paragraph,
                             inputs[id_][2],
@@ -473,6 +484,8 @@ def translate_paragraph(
             for llm_translate_tracker in llm_translate_trackers:
                 llm_translate_tracker.set_error_message(error_message)
                 llm_translate_tracker.set_fallback_to_translate()
+            for input_ in inputs:
+                input_[2].unicode = input_[5]
             if not should_translate_paragraph:
                 should_translate_paragraph = list(
                     range(len(batch_paragraph.paragraphs))

babeldoc/main.py

@@ -23,7 +23,7 @@
 from babeldoc.translation_config import WatermarkOutputMode
 
 logger = logging.getLogger(__name__)
-__version__ = "0.3.41"
+__version__ = "0.3.42"
 
 
 def create_parser():

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "BabelDOC"
-version = "0.3.41"
+version = "0.3.42"
 description = "Yet Another Document Translator"
 license = "AGPL-3.0"
 readme = "README.md"
@@ -150,7 +150,7 @@ pythonpath = [".", "src"]
 testpaths = ["tests"]
 
 [bumpver]
-current_version = "0.3.41"
+current_version = "0.3.42"
 version_pattern = "MAJOR.MINOR.PATCH[.PYTAGNUM]"
 
 [bumpver.file_patterns]