Merge pull request #173 from funstory-ai/pdf2zh-v2-rc

fix(high_level): implement null xref handling in PDF processing

Author: awwaawwa

babeldoc/__init__.py

@@ -1 +1 @@
-__version__ = "0.2.13"
+__version__ = "0.2.14"

babeldoc/const.py

@@ -3,7 +3,7 @@
 import subprocess
 from pathlib import Path
 
-__version__ = "0.2.13"
+__version__ = "0.2.14"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

babeldoc/document_il/backend/pdf_creater.py

@@ -730,7 +730,12 @@ def write(self, translation_config: TranslationConfig) -> TranslateResult:
                     draw_op.append(b" Tj ET Q \n")
                 for xobj in page.pdf_xobject:
                     draw_op = xobj_draw_ops[xobj.xobj_id]
-                    pdf.update_stream(xobj.xref_id, draw_op.tobytes())
+                    try:
+                        pdf.update_stream(xobj.xref_id, draw_op.tobytes())
+                    except Exception:
+                        logger.warning(
+                            f"update xref {xobj.xref_id} stream fail, continue"
+                        )
                     # pdf.update_stream(xobj.xref_id, b'')
                 for rect in page.pdf_rectangle:
                     self._debug_render_rectangle(page_op, rect)

babeldoc/high_level.py

@@ -190,10 +190,10 @@ def start_parse_il(
         #         box[y0:y1, x0:x1] = 0
         # layout[page.pageno] = box
         # 新建一个 xref 存放新指令流
-        page.page_xref = doc_zh.get_new_xref()  # hack 插入页面的新 xref
-        doc_zh.update_object(page.page_xref, "<<>>")
-        doc_zh.update_stream(page.page_xref, b"")
-        doc_zh[page.pageno].set_contents(page.page_xref)
+        # page.page_xref = doc_zh.get_new_xref()  # hack 插入页面的新 xref
+        # doc_zh.update_object(page.page_xref, "<<>>")
+        # doc_zh.update_stream(page.page_xref, b"")
+        # doc_zh[page.pageno].set_contents(page.page_xref)
         ops_base = interpreter.process_page(page)
         il_creater.on_page_base_operation(ops_base)
         il_creater.on_page_end()
@@ -353,6 +353,21 @@ def _monitor_memory_usage(self):
             time.sleep(self.interval)
 
 
+def fix_null_xref(doc: Document) -> None:
+    """Fix null xref in PDF file by replacing them with empty arrays.
+
+    Args:
+        doc: PyMuPDF Document object to fix
+    """
+    for i in range(1, doc.xref_length()):
+        try:
+            obj = doc.xref_object(i)
+            if obj == "null":
+                doc.update_object(i, "[]")
+        except Exception:
+            doc.update_object(i, "[]")
+
+
 def do_translate(
     pm: ProgressMonitor, translation_config: TranslationConfig
 ) -> TranslateResult:
@@ -507,12 +522,14 @@ def _do_translate_single(
     """Original translation logic for a single document or part"""
     translation_config.progress_monitor = pm
     original_pdf_path = translation_config.input_file
-    doc_input = Document(original_pdf_path)
     if translation_config.debug:
+        doc_input = Document(original_pdf_path)
         logger.debug("debug mode, save decompressed input pdf")
         output_path = translation_config.get_working_file_path(
             "input.decompressed.pdf",
         )
+        # Fix null xref in PDF file
+        fix_null_xref(doc_input)
         doc_input.save(output_path, expand=True, pretty=True)
 
     # Continue with original processing
@@ -521,29 +538,15 @@ def _do_translate_single(
     resfont = "china-ss"
 
     # Fix null xref in PDF file
-    for i in range(1, doc_pdf2zh.xref_length()):
-        try:
-            obj = doc_pdf2zh.xref_object(i)
-            if obj == "null":
-                ret = doc_pdf2zh.update_object(i, "[]")
-                if ret != 0:
-                    logger.warning(f"try fix1 xref {i} fail, continue")
-                else:
-                    logger.info(f"try fix1 xref {i} success")
-        except Exception:
-            ret = doc_pdf2zh.update_object(i, "[]")
-            if ret != 0:
-                logger.warning(f"try fix2 xref {i} fail, continue")
-            else:
-                logger.info(f"try fix2 xref {i} success")
+    fix_null_xref(doc_pdf2zh)
 
     for page in doc_pdf2zh:
         page.insert_font(resfont, None)
 
     resfont = None
     doc_pdf2zh.save(temp_pdf_path)
     il_creater = ILCreater(translation_config)
-    il_creater.mupdf = doc_input
+    il_creater.mupdf = doc_pdf2zh
     xml_converter = XMLConverter()
     logger.debug(f"start parse il from {temp_pdf_path}")
     with Path(temp_pdf_path).open("rb") as f:
@@ -578,7 +581,7 @@ def _do_translate_single(
 
     # Generate layouts for all pages
     logger.debug("start generating layouts")
-    docs = LayoutParser(translation_config).process(docs, doc_input)
+    docs = LayoutParser(translation_config).process(docs, doc_pdf2zh)
     logger.debug("finish generating layouts")
     if translation_config.debug:
         xml_converter.write_json(
@@ -634,7 +637,7 @@ def _do_translate_single(
 
     if translation_config.watermark_output_mode == WatermarkOutputMode.Both:
         mono_watermark_first_page_doc_bytes, dual_watermark_first_page_doc_bytes = (
-            generate_first_page_with_watermark(doc_input, translation_config, docs)
+            generate_first_page_with_watermark(doc_pdf2zh, translation_config, docs)
         )
 
     Typesetting(translation_config).typsetting_document(docs)

babeldoc/main.py

@@ -22,7 +22,7 @@
 from babeldoc.translation_config import WatermarkOutputMode
 
 logger = logging.getLogger(__name__)
-__version__ = "0.2.13"
+__version__ = "0.2.14"
 
 
 def create_parser():

babeldoc/pdfinterp.py

@@ -337,14 +337,14 @@ def process_page(self, page: PDFPage) -> None:
         ops_base = self.render_contents(page.resources, page.contents, ctm=ctm)
         self.device.fontid = self.fontid
         self.device.fontmap = self.fontmap
-        ops_new = self.device.end_page(page)
+        _ops_new = self.device.end_page(page)
         # 上面渲染的时候会根据 cropbox 减掉页面偏移得到真实坐标，这里输出的时候需要用 cm 把页面偏移加回来
-        self.obj_patch[page.page_xref] = (
-            # f"q {ops_base}Q 1 0 0 1 {x0} {y0} cm {ops_new}"  # ops_base 里可能有图，需要让 ops_new 里的文字覆盖在上面，使用 q/Q 重置位置矩阵
-            ""
-        )
-        for obj in page.contents:
-            self.obj_patch[obj.objid] = ""
+        # self.obj_patch[page.page_xref] = (
+        #     # f"q {ops_base}Q 1 0 0 1 {x0} {y0} cm {ops_new}"  # ops_base 里可能有图，需要让 ops_new 里的文字覆盖在上面，使用 q/Q 重置位置矩阵
+        #     ""
+        # )
+        # for obj in page.contents:
+        #     self.obj_patch[obj.objid] = ""
         return ops_base
 
     def render_contents(

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "BabelDOC"
-version = "0.2.13"
+version = "0.2.14"
 description = "Yet Another Document Translator"
 license = "AGPL-3.0"
 readme = "README.md"
@@ -147,7 +147,7 @@ pythonpath = [".", "src"]
 testpaths = ["tests"]
 
 [bumpver]
-current_version = "0.2.13"
+current_version = "0.2.14"
 version_pattern = "MAJOR.MINOR.PATCH[.PYTAGNUM]"
 
 [bumpver.file_patterns]