Merge remote-tracking branch 'origin/asciimath2' into asciimath2

# Conflicts:
#	llm_web_kit/extractor/html/recognizer/cc_math/render/mathjax.py
#	tests/llm_web_kit/extractor/assets/extractor_chain_input/good_data/html/math_test_ascii_delimiter.html
#	tests/llm_web_kit/extractor/assets/extractor_chain_input/good_data/html_data_input.jsonl
#	tests/llm_web_kit/extractor/test_extractor_chain.py

Author: 1041206149

llm_web_kit/extractor/html/extractor.py

@@ -90,6 +90,7 @@ def _do_extract(self, data_json: DataJson) -> DataJson:
         raw_html:str = data_json['html']
         base_url:str = data_json['url']
         main_html:str = data_json['main_html']
+        language:str = data_json.get('language', 'en')
         # page_layout_type:str = data_json.get('page_layout_type', HTMLPageLayoutType.LAYOUT_ARTICLE)  # 默认是文章类型
 
         # main_html, method, title = self._extract_main_html(raw_html, base_url, page_layout_type)
@@ -98,7 +99,7 @@ def _do_extract(self, data_json: DataJson) -> DataJson:
         for extract_func in [self._extract_code, self._extract_table, self._extract_math, self._extract_list,
                              self._extract_image,
                              self._extract_title, self._extract_paragraph]:
-            parsed_html = extract_func(base_url, parsed_html, raw_html)
+            parsed_html = extract_func(base_url, parsed_html, raw_html, language)
 
         # 过滤掉包含script和style标签的元素,在这里改，是因为math提取需要保留script标签
         filtered_parsed_html = []
@@ -111,7 +112,7 @@ def _do_extract(self, data_json: DataJson) -> DataJson:
         # data_json['title'] = title
         return data_json
 
-    def _extract_code(self, base_url:str, html_lst:List[Tuple[HtmlElement, HtmlElement]], raw_html:str) -> List[Tuple[HtmlElement,HtmlElement]]:
+    def _extract_code(self, base_url:str, html_lst:List[Tuple[HtmlElement, HtmlElement]], raw_html:str, language:str) -> List[Tuple[HtmlElement,HtmlElement]]:
         """从html文本中提取代码.
 
         Args:
@@ -121,10 +122,10 @@ def _extract_code(self, base_url:str, html_lst:List[Tuple[HtmlElement, HtmlEleme
         Returns:
         """
 
-        lst = self.__code_recognizer.recognize(base_url, html_lst, raw_html)
+        lst = self.__code_recognizer.recognize(base_url, html_lst, raw_html, language)
         return lst
 
-    def _extract_math(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str) -> List[Tuple[str,str]]:
+    def _extract_math(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str, language:str) -> List[Tuple[str,str]]:
         """从html文本中提取数学公式.
 
         Args:
@@ -135,10 +136,10 @@ def _extract_math(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:st
         Returns:
         """
 
-        lst = self.__math_recognizer.recognize(base_url, html_lst, raw_html)
+        lst = self.__math_recognizer.recognize(base_url, html_lst, raw_html, language)
         return lst
 
-    def _extract_image(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str) -> List[Tuple[str,str]]:
+    def _extract_image(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str, language:str) -> List[Tuple[str,str]]:
         """从html文本中提取图片.
 
         Args:
@@ -149,10 +150,10 @@ def _extract_image(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:s
         Returns:
         """
 
-        lst = self.__image_recognizer.recognize(base_url, html_lst, raw_html)
+        lst = self.__image_recognizer.recognize(base_url, html_lst, raw_html, language)
         return lst
 
-    def _extract_audio(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str) -> List[Tuple[str,str]]:
+    def _extract_audio(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str, language:str) -> List[Tuple[str,str]]:
         """从html文本中提取音频.
 
         Args:
@@ -163,10 +164,10 @@ def _extract_audio(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:s
         Returns:
         """
 
-        lst = self.__audio_recognizer.recognize(base_url, html_lst, raw_html)
+        lst = self.__audio_recognizer.recognize(base_url, html_lst, raw_html, language)
         return lst
 
-    def _extract_video(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str) -> List[Tuple[str,str]]:
+    def _extract_video(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str, language:str) -> List[Tuple[str,str]]:
         """从html文本中提取视频.
 
         Args:
@@ -177,10 +178,10 @@ def _extract_video(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:s
         Returns:
         """
 
-        lst = self.__video_recognizer.recognize(base_url, html_lst, raw_html)
+        lst = self.__video_recognizer.recognize(base_url, html_lst, raw_html, language)
         return lst
 
-    def _extract_table(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str) -> List[Tuple[str,str]]:
+    def _extract_table(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str, language:str) -> List[Tuple[str,str]]:
         """从html文本中提取表格.
 
         Args:
@@ -191,10 +192,10 @@ def _extract_table(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:s
         Returns:
         """
 
-        lst = self.__table_recognizer.recognize(base_url, html_lst, raw_html)
+        lst = self.__table_recognizer.recognize(base_url, html_lst, raw_html, language)
         return lst
 
-    def _extract_list(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str) -> List[Tuple[str,str]]:
+    def _extract_list(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str, language:str) -> List[Tuple[str,str]]:
         """从html文本中提取列表.
 
         Args:
@@ -205,10 +206,10 @@ def _extract_list(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:st
         Returns:
         """
 
-        lst = self.__list_recognizer.recognize(base_url, html_lst, raw_html)
+        lst = self.__list_recognizer.recognize(base_url, html_lst, raw_html, language)
         return lst
 
-    def _extract_title(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str) -> List[Tuple[str,str]]:
+    def _extract_title(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str, language:str) -> List[Tuple[str,str]]:
         """从html文本中提取标题.
 
         Args:
@@ -219,10 +220,10 @@ def _extract_title(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:s
         Returns:
         """
 
-        lst = self.__title_recognizer.recognize(base_url, html_lst, raw_html)
+        lst = self.__title_recognizer.recognize(base_url, html_lst, raw_html, language)
         return lst
 
-    def _extract_paragraph(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str) -> List[Tuple[str,str]]:
+    def _extract_paragraph(self, base_url:str, html_lst:List[Tuple[str,str]], raw_html:str, language:str) -> List[Tuple[str,str]]:
         """从html文本中提取段落.
 
         Args:
@@ -233,7 +234,7 @@ def _extract_paragraph(self, base_url:str, html_lst:List[Tuple[str,str]], raw_ht
         Returns:
         """
 
-        lst = self.__paragraph_recognizer.recognize(base_url, html_lst, raw_html)
+        lst = self.__paragraph_recognizer.recognize(base_url, html_lst, raw_html, language)
         return lst
 
     def __is_valid_node(self, node: dict) -> bool:

llm_web_kit/extractor/html/recognizer/audio.py

@@ -10,7 +10,7 @@
 class AudioRecognizer(BaseHTMLElementRecognizer):
     """解析音频元素."""
     @override
-    def recognize(self, base_url:str, main_html_lst: List[Tuple[HtmlElement,HtmlElement]], raw_html:str) -> List[Tuple[HtmlElement,HtmlElement]]:
+    def recognize(self, base_url:str, main_html_lst: List[Tuple[HtmlElement,HtmlElement]], raw_html:str, language:str = 'en') -> List[Tuple[HtmlElement,HtmlElement]]:
         """父类，解析音频元素.
 
         Args:

llm_web_kit/extractor/html/recognizer/cccode.py

@@ -27,7 +27,8 @@ def recognize(
         self,
         base_url: str,
         main_html_lst: List[Tuple[HtmlElement, HtmlElement]],
-        raw_html: str
+        raw_html: str,
+        language:str = 'en'
     ) -> List[Tuple[HtmlElement, HtmlElement]]:
         """父类，解析代码元素.
 

llm_web_kit/extractor/html/recognizer/ccmath.py

@@ -26,7 +26,7 @@ def __init__(self):
         self.cm = CCMATH()
 
     @override
-    def recognize(self, base_url: str, main_html_lst: List[Tuple[HtmlElement, HtmlElement]], raw_html: str) -> List[Tuple[HtmlElement, HtmlElement]]:
+    def recognize(self, base_url: str, main_html_lst: List[Tuple[HtmlElement, HtmlElement]], raw_html: str, language:str = 'en') -> List[Tuple[HtmlElement, HtmlElement]]:
         """父类，解析数学公式元素.
 
         Args:

llm_web_kit/extractor/html/recognizer/image.py

@@ -67,7 +67,7 @@ def __ccimg_to_content_list(self, raw_html_segment: str, html_obj: HtmlElement)
         return result
 
     @override
-    def recognize(self, base_url: str, main_html_lst: List[Tuple[HtmlElement, HtmlElement]], raw_html: str) -> List[
+    def recognize(self, base_url: str, main_html_lst: List[Tuple[HtmlElement, HtmlElement]], raw_html: str, language:str = 'en') -> List[
         Tuple[HtmlElement, HtmlElement]]:
         """父类，解析图片元素.
 

llm_web_kit/extractor/html/recognizer/list.py

@@ -48,7 +48,7 @@ def to_content_list_node(self, base_url: str, parsed_content: HtmlElement, raw_h
         return ele_node
 
     @override
-    def recognize(self, base_url: str, main_html_lst: List[Tuple[HtmlElement, HtmlElement]], raw_html: str) -> List[Tuple[HtmlElement, HtmlElement]]:
+    def recognize(self, base_url: str, main_html_lst: List[Tuple[HtmlElement, HtmlElement]], raw_html: str, language:str = 'en') -> List[Tuple[HtmlElement, HtmlElement]]:
         """父类，解析列表元素.
 
         Args:

llm_web_kit/extractor/html/recognizer/recognizer.py

@@ -29,7 +29,7 @@ class BaseHTMLElementRecognizer(ABC):
 
     """基本的元素解析类."""
     @abstractmethod
-    def recognize(self, base_url:str, main_html_lst: List[Tuple[HtmlElement, HtmlElement]], raw_html:str) -> List[Tuple[HtmlElement, HtmlElement]]:
+    def recognize(self, base_url:str, main_html_lst: List[Tuple[HtmlElement, HtmlElement]], raw_html:str, language:str) -> List[Tuple[HtmlElement, HtmlElement]]:
         """父类，解析html中的元素.
 
         Args:

llm_web_kit/extractor/html/recognizer/table.py

@@ -24,7 +24,8 @@ def __init__(self):
     def recognize(self,
                   base_url: str,
                   main_html_lst: List[Tuple[HtmlElement, HtmlElement]],
-                  raw_html: str) -> List[Tuple[HtmlElement, HtmlElement]]:
+                  raw_html: str,
+                  language:str = 'en') -> List[Tuple[HtmlElement, HtmlElement]]:
         """父类，解析表格元素.
 
         Args:
@@ -256,7 +257,8 @@ def __simplify_td_th_content(self, table_nest_level, elem: HtmlElement) -> None:
     def __get_table_body(self, table_type, table_nest_level, table_root):
         """获取并处理table body，返回处理后的HTML字符串。"""
         if table_type == 'empty':
-            return None
+            content = table_root.text_content()
+            return content
         allowed_attributes = ['colspan', 'rowspan']
         # 清理除了colspan和rowspan之外的属性
         if len(table_root.attrib) > 0:

llm_web_kit/extractor/html/recognizer/text.py

@@ -67,6 +67,9 @@
     'mjx-container', 'mjx-assistive-mml', 'strike', 'wbr', 'ins'
 }
 
+# 词间无分隔符的语言
+no_separation_language = ['zh', 'ja', 'ko', 'wuu', 'th', 'km', 'lo', 'bo', 'ii', 'jv']
+
 
 class TextParagraphRecognizer(BaseHTMLElementRecognizer):
     """解析文本段落元素."""
@@ -93,7 +96,7 @@ def to_content_list_node(self, base_url: str, parsed_content: HtmlElement, raw_h
         return node
 
     @override
-    def recognize(self, base_url:str, main_html_lst: List[Tuple[HtmlElement | str, HtmlElement | str]], raw_html:str) -> List[Tuple[HtmlElement, HtmlElement]]:
+    def recognize(self, base_url:str, main_html_lst: List[Tuple[HtmlElement | str, HtmlElement | str]], raw_html:str, language:str = 'en') -> List[Tuple[HtmlElement, HtmlElement]]:
         """父类，解析文本段落元素.
 
         Args:
@@ -111,11 +114,11 @@ def recognize(self, base_url:str, main_html_lst: List[Tuple[HtmlElement | str, H
                 new_html_lst.append((html_element, raw_html_element))
             else:
                 lst = list(self.__extract_paragraphs(html_element))
-                new_lst = self.__to_cctext_lst(lst)
+                new_lst = self.__to_cctext_lst(lst, language)
                 new_html_lst.extend(new_lst)
         return new_html_lst
 
-    def __to_cctext_lst(self, lst: List[Tuple[HtmlElement | str, HtmlElement | str]]) -> List[Tuple[HtmlElement, HtmlElement]]:
+    def __to_cctext_lst(self, lst: List[Tuple[HtmlElement | str, HtmlElement | str]], language:str) -> List[Tuple[HtmlElement, HtmlElement]]:
         """将lst[Element, raw_html] 进行处理. 提出Element里的文字，做成<<cctext>>标签.
 
         Args:
@@ -129,7 +132,7 @@ def __to_cctext_lst(self, lst: List[Tuple[HtmlElement | str, HtmlElement | str]]
             el_element = html_to_element(el) if isinstance(el, str) else el
             raw_html_element = html_to_element(raw_html) if isinstance(raw_html, str) else raw_html
 
-            para_text = self.__get_paragraph_text(el_element)
+            para_text = self.__get_paragraph_text(el_element, language)
             if para_text:
                 cctext_el = self._build_cc_element(CCTag.CC_TEXT, json.dumps(para_text, ensure_ascii=False, indent=4), '', html=element_to_html_unescaped(raw_html_element))
                 new_lst.append((cctext_el, raw_html_element))
@@ -185,20 +188,20 @@ def __combine_text(self, text1:str, text2:str, lang='en') -> str:
             lang: str: 语言  TODO 实现根据语言连接文本的不同方式, 还有就是一些特殊符号开头的连接不加空格。
         """
         text1 = text1.strip(' ') if text1 else ''
-        text2 = text2.strip(' ') if text2 else ''
-        if lang == 'zh':
+        text2 = text2.rstrip(' ') if text2 else ''
+        if lang in no_separation_language:
             txt = text1 + text2
             return self.replace_entities(txt.strip(), entities_map)
         else:
             # 根据text1的最后一个字符和text2的第一个字符判断两个text之间的连接
             if (text2[0] in string.punctuation) or (text2[0] in special_symbols) or (text2[0] in other_symbols) or (text1 and text1[-1] in other_symbols):
                 words_sep = ''
-            else :
+            else:
                 words_sep = ' '
             txt = text1 + words_sep + text2
             return self.replace_entities(txt.strip(), entities_map)
 
-    def __get_paragraph_text(self, root: HtmlElement) -> List[dict]:
+    def __get_paragraph_text(self, root: HtmlElement, language:str = 'en') -> List[dict]:
         """
         获取段落全部的文本.
         对于段落里的行内公式<equation-inline>需要特定处理，转换为段落格式：
@@ -235,7 +238,7 @@ def __get_paragraph_text_recusive(el: HtmlElement, text: str) -> str:
                 pass
             else:
                 if el.text and el.text.strip():
-                    text = self.__combine_text(text, el.text.strip())
+                    text = self.__combine_text(text, el.text.strip(), language)
                 for child in el:
                     text = __get_paragraph_text_recusive(child, text)
 
@@ -244,7 +247,8 @@ def __get_paragraph_text_recusive(el: HtmlElement, text: str) -> str:
                 if is_sub_sup:
                     text += el.tail
                 else:
-                    text = self.__combine_text(text, el.tail.strip())
+                    new_tail = f' {el.tail.strip()}' if el.tail.startswith(' ') and el.tail.strip()[0] in string.punctuation else el.tail.strip()
+                    text = self.__combine_text(text, new_tail, language)
 
             return text
 

llm_web_kit/extractor/html/recognizer/title.py

@@ -39,7 +39,7 @@ def to_content_list_node(self, base_url: str, parsed_content: HtmlElement, raw_h
         return cctitle_content_node
 
     @override
-    def recognize(self, base_url: str, main_html_lst: List[Tuple[HtmlElement, HtmlElement]], raw_html: str) -> List[Tuple[HtmlElement, HtmlElement]]:
+    def recognize(self, base_url: str, main_html_lst: List[Tuple[HtmlElement, HtmlElement]], raw_html: str, language:str = 'en') -> List[Tuple[HtmlElement, HtmlElement]]:
         """父类，解析标题元素.
 
         Args: