<fix>: fix table tag integrity (#497)

Author: papayalove

llm_web_kit/main_html_parser/parser/layout_batch_parser.py

@@ -154,18 +154,20 @@ def find_blocks_drop(self, element, depth, element_dict, parent_keyy, parent_lab
         # 匹配正文节点
         has_red = False
         layer_nodes_dict = dict()
+        layer_nodes_dict_drop_tail = dict()
         layer_norm_eles = {}
         # 构造当前层的候选映射字典
         for ele_keyy, ele_value in layer_nodes.items():
             ele_parent_keyy = self.normalize_key(ele_value[1])
             if ele_parent_keyy is not None:
                 ele_parent_keyy = tuple(ele_parent_keyy)
             ele_label = ele_value[0]
+            is_drop_tail = ele_value[3]
             norm_ele_keyy = self.normalize_key(ele_keyy[:3])
             if norm_ele_keyy in layer_norm_eles:
-                layer_norm_eles[norm_ele_keyy].append((ele_label, ele_keyy[:3], ele_parent_keyy))
+                layer_norm_eles[norm_ele_keyy].append((ele_label, ele_keyy[:3], ele_parent_keyy, is_drop_tail))
             else:
-                layer_norm_eles[norm_ele_keyy] = [(ele_label, ele_keyy[:3], ele_parent_keyy)]
+                layer_norm_eles[norm_ele_keyy] = [(ele_label, ele_keyy[:3], ele_parent_keyy, is_drop_tail)]
         # 尝试匹配当前层每个节点，判断是否存在至少一个红色节点
         for current_layer_key, current_layer_value in current_layer_keys.items():
             current_layer_ori_key = current_layer_value[0]
@@ -175,20 +177,22 @@ def find_blocks_drop(self, element, depth, element_dict, parent_keyy, parent_lab
                     if layer_norm_ele_value[2] != parent_keyy:
                         continue
                     node_label = layer_norm_ele_value[0]
-
+                    is_drop_tail = layer_norm_ele_value[3]
                     if current_layer_key in layer_nodes_dict:
                         layer_nodes_dict[current_layer_key].append(node_label)
+                        layer_nodes_dict_drop_tail[current_layer_key].append(is_drop_tail)
                     else:
                         layer_nodes_dict[current_layer_key] = [node_label]
+                        layer_nodes_dict_drop_tail[current_layer_key] = [is_drop_tail]
                     if node_label == 'red':
                         has_red = True
                         break
             # 动态id匹配逻辑
             elif self.dynamic_id_enable and current_layer_key[2]:
-                node_label, matched_ele_key = self.__match_tag_class(layer_nodes, current_layer_ori_key, parent_keyy,
+                node_label, matched_ele_key, is_drop_tail = self.__match_tag_class(layer_nodes, current_layer_ori_key, parent_keyy,
                                                                      node_html, template_doc)
                 if node_label is None and self.dynamic_classid_enable:
-                    node_label, matched_ele_key = self.__match_tag(layer_nodes, current_layer_ori_key, parent_keyy,
+                    node_label, matched_ele_key, is_drop_tail = self.__match_tag(layer_nodes, current_layer_ori_key, parent_keyy,
                                                                    node_html,
                                                                    template_doc, False, True)
                 if node_label is None:
@@ -199,13 +203,14 @@ def find_blocks_drop(self, element, depth, element_dict, parent_keyy, parent_lab
                     element.set('id', matched_ele_key[2])
                 if current_layer_key in layer_nodes_dict:
                     layer_nodes_dict[matched_ele_key].append(node_label)
+                    layer_nodes_dict_drop_tail[matched_ele_key].append(is_drop_tail)
                 else:
                     layer_nodes_dict[matched_ele_key] = [node_label]
-
+                    layer_nodes_dict_drop_tail[matched_ele_key] = [is_drop_tail]
                 if node_label == 'red':
                     has_red = True
             elif self.dynamic_id_enable and self.dynamic_classid_enable and current_layer_key[1]:
-                node_label, matched_ele_key = self.__match_tag(layer_nodes, current_layer_ori_key, parent_keyy,
+                node_label, matched_ele_key, is_drop_tail = self.__match_tag(layer_nodes, current_layer_ori_key, parent_keyy,
                                                                node_html,
                                                                template_doc, True, False)
                 if node_label is None:
@@ -216,11 +221,12 @@ def find_blocks_drop(self, element, depth, element_dict, parent_keyy, parent_lab
                     element.set('class', matched_ele_key[1])
                 if current_layer_key in layer_nodes_dict:
                     layer_nodes_dict[matched_ele_key].append(node_label)
+                    layer_nodes_dict_drop_tail[matched_ele_key].append(is_drop_tail)
                 else:
                     layer_nodes_dict[matched_ele_key] = [node_label]
+                    layer_nodes_dict_drop_tail[matched_ele_key] = [is_drop_tail]
                 if node_label == 'red':
                     has_red = True
-
         if not has_red and parent_label != 'red':
             parent = element.getparent()
             if parent is not None:
@@ -244,6 +250,10 @@ def find_blocks_drop(self, element, depth, element_dict, parent_keyy, parent_lab
                     return
             else:
                 label = 'red'
+                # 正文节点情况下还需要判断tail是否是正文
+                if False not in layer_nodes_dict_drop_tail[keyy]:
+                    if element.tail:
+                        element.tail = None
         elif length > 0 or length_tail > 0 or tag in ['figure', 'img']:
             return
 
@@ -321,6 +331,7 @@ def __match_tag_class(self, layer_nodes, current_layer_key, parent_key, node_htm
             if ele_parent_keyy is not None:
                 ele_parent_keyy = tuple(ele_parent_keyy)
             ele_label = ele_value[0]
+            is_drop_tail = ele_value[3]
             norm_ele_keyy = self.normalize_key((ele_keyy[0], ele_keyy[1], None))
             norm_ele_keyy_parent = (norm_ele_keyy, ele_parent_keyy)
             if current_norm_key == norm_ele_keyy_parent:
@@ -335,10 +346,10 @@ def __match_tag_class(self, layer_nodes, current_layer_key, parent_key, node_htm
                 template_sim = similarity(feature1, feature2, layer_n=3)
 
                 if template_sim > DYNAMIC_ID_SIM_THRESHOLD:
-                    return ele_label, self.normalize_key(ele_keyy[0:3])
+                    return ele_label, self.normalize_key(ele_keyy[0:3]), is_drop_tail
                 # else:
                 # logger.info(f'{current_layer_key} and {ele_keyy} similarity is {template_sim}')
-        return None, None
+        return None, None, None
 
     def __match_tag(self, layer_nodes, current_layer_key, parent_key, node_html, template_doc, class_must=False,
                     id_exist=False):
@@ -357,6 +368,7 @@ def __match_tag(self, layer_nodes, current_layer_key, parent_key, node_html, tem
             if ele_parent_keyy is not None:
                 ele_parent_keyy = tuple(ele_parent_keyy)
             ele_label = ele_value[0]
+            is_drop_tail = ele_value[3]
             norm_ele_keyy = self.normalize_key((ele_keyy[0], None, None))
             norm_ele_keyy_parent = (norm_ele_keyy, ele_parent_keyy)
             # 相似度方案
@@ -370,14 +382,14 @@ def __match_tag(self, layer_nodes, current_layer_key, parent_key, node_html, tem
                     continue
                 template_sim = similarity(feature1, feature2, layer_n=3)
                 if template_sim >= self.dynamic_classid_similarity_threshold:
-                    return ele_label, self.normalize_key(ele_keyy[0:3])
+                    return ele_label, self.normalize_key(ele_keyy[0:3]), is_drop_tail
             # first class方案
             norm_ele_keyy_with_first_class = self.normalize_key((ele_keyy[0], ele_keyy[1].strip().split(' ')[0], None))
             norm_ele_keyy_parent_with_first_class = (norm_ele_keyy_with_first_class, ele_parent_keyy)
             if current_norm_key_with_first_class == norm_ele_keyy_parent_with_first_class:
-                return ele_label, self.normalize_key(ele_keyy[0:3])
+                return ele_label, self.normalize_key(ele_keyy[0:3]), is_drop_tail
 
-        return None, None
+        return None, None, None
 
     def __is_natural_language(self, text, min_words=3):
         """判断文本是否像自然语言.

llm_web_kit/main_html_parser/parser/tag_mapping.py

@@ -124,27 +124,36 @@ def deal_element_direct(self, item_id, test_root):
         elements = test_root.xpath(f'//*[@_item_id="{item_id}"]')
         deal_element = elements[0]
         deal_element.set('magic_main_html', 'True')
+        for ele in deal_element:
+            ele.set('magic_main_html', 'True')
 
     def find_affected_element_after_drop(self, element):
         prev_sibling = element.getprevious()
         parent = element.getparent()
-
+        is_main = bool(element.get('magic_main_html', None))
         # 包裹子节点的情况返回element父节点
         if len(element) > 0:
-            if element.get('magic_main_html', None):
+            if is_main:
                 for ele in element:
                     ele.set('magic_main_html', 'True')
 
             element.drop_tag()
-            return parent
+            # 如果包含子tag并且还有text，text有可能是兄弟节点的tail
+            if element.text and element.text.strip():
+                if prev_sibling is not None:
+                    # 兄弟节点是否drop text， 是否drop tail
+                    return prev_sibling, False, not is_main
+                else:
+                    return parent, not is_main, False
+            return parent, False, False
 
         # 只有文本的情况，返回element前面的兄弟节点或者父节点
         element.drop_tag()
 
         if prev_sibling is not None:
-            return prev_sibling
+            return prev_sibling, False, not is_main
         else:
-            return parent
+            return parent, not is_main, False
 
     def process_element(self, element):
         # 前序遍历元素树（先处理子元素）
@@ -154,9 +163,13 @@ def process_element(self, element):
         # 如果是cc-alg-uc-text标签，用drop_tag()删除标签但保留子元素
         if element.tag == 'cc-alg-uc-text':
             is_main = element.get('magic_main_html', None)
-            affected = self.find_affected_element_after_drop(element)
+            affected, drop_text, drop_tail = self.find_affected_element_after_drop(element)
             if is_main:
                 affected.set('magic_main_html', 'True')
+            if drop_text:
+                affected.set('drop_text', 'True')
+            if drop_tail:
+                affected.set('drop_tail', 'True')
 
         return
 
@@ -234,6 +247,7 @@ def process_main_tree(self, element, depth, layer_index_counter, all_dict, all_s
             all_dict[depth] = {}
             all_set[depth] = {}
         is_main_html = element.get('magic_main_html', None)
+        is_drop_tail = element.get('drop_tail', None)
         current_dict = all_dict[depth]
         current_set = all_set[depth]
         tag = element.tag
@@ -260,10 +274,10 @@ def process_main_tree(self, element, depth, layer_index_counter, all_dict, all_s
         # 写入该层元素key，如果有重复的green节点，只保留一个
         if keyy_for_sim in current_set:
             if is_main_html and current_set[keyy_for_sim][0] == 'green':
-                current_dict[keyy] = ('red', parent_keyy, xpath)
+                current_dict[keyy] = ('red', parent_keyy, xpath, bool(is_drop_tail))
                 current_set[keyy_for_sim] = ('red', parent_keyy)
         else:
-            current_dict[keyy] = (color, parent_keyy, xpath)
+            current_dict[keyy] = (color, parent_keyy, xpath, bool(is_drop_tail))
             current_set[keyy_for_sim] = (color, parent_keyy)
 
         for ele in element: