<improv>: modify typical main html similarity threshold to 0.92 (#504)

Author: papayalove

llm_web_kit/input/pre_data_json.py

@@ -30,6 +30,8 @@ class PreDataJsonKey:
     TYPICAL_MAIN_HTML = 'typical_main_html'
     # 模版网页提取正文成功标签, bool类型
     TYPICAL_MAIN_HTML_SUCCESS = 'typical_main_html_success'
+    # similarity between typical main html and html
+    TYPICAL_MAIN_HTML_SIM = 'typical_main_html_sim'
     # 用于生成element dict的html
     TYPICAL_DICT_HTML = 'typical_dict_html'
     # 动态id开关
@@ -44,6 +46,8 @@ class PreDataJsonKey:
     HTML_SOURCE = 'html_source'
     # 推广网页提取正文成功标签, bool类型
     MAIN_HTML_SUCCESS = 'main_html_success'
+    # similarity between main html and typical main html
+    MAIN_HTML_SIM = 'main_html_sim'
     # 推广网页提取正文文本
     MAIN_HTML = 'main_html'
     # 推广网页提取正文树

llm_web_kit/main_html_parser/parser/layout_batch_parser.py

@@ -86,6 +86,7 @@ def parse(self, pre_data: PreDataJson) -> PreDataJson:
             sim = None
             if feature1 is not None and feature2 is not None:
                 sim = similarity(feature1, feature2, layer_n=layer)
+                pre_data[PreDataJsonKey.MAIN_HTML_SIM] = sim
             if sim is None or sim < SIMILARITY_THRESHOLD:
                 pre_data[PreDataJsonKey.MAIN_HTML_SUCCESS] = False
             else:

llm_web_kit/main_html_parser/parser/tag_mapping.py

@@ -7,7 +7,7 @@
     LayoutBatchParser
 from llm_web_kit.main_html_parser.parser.parser import BaseMainHtmlParser
 
-SIMILAR_THRESHOLD = 0.9
+SIMILAR_THRESHOLD = 0.92
 
 
 class MapItemToHtmlTagsParser(BaseMainHtmlParser):
@@ -74,8 +74,9 @@ def parse(self, pre_data: PreDataJson) -> PreDataJson:
             template_sim = None
             if feature1 is not None and feature2 is not None:
                 template_sim = similarity(feature1, feature2, layer_n=layer)
+                pre_data[PreDataJsonKey.TYPICAL_MAIN_HTML_SIM] = template_sim
 
-            # 比较模版正文html与原html相似度
+                # 比较模版正文html与原html相似度
             if template_sim is None or template_sim > SIMILAR_THRESHOLD:
                 pre_data[PreDataJsonKey.TYPICAL_MAIN_HTML_SUCCESS] = False
             else:

tests/llm_web_kit/input/test_pre_data_json.py

@@ -394,6 +394,8 @@ def test_pre_data_json_key_constants(self):
         assert hasattr(PreDataJsonKey, 'DYNAMIC_CLASSID_ENABLE')
         assert hasattr(PreDataJsonKey, 'DYNAMIC_CLASSID_SIM_THRESH')
         assert hasattr(PreDataJsonKey, 'MORE_NOISE_ENABLE')
+        assert hasattr(PreDataJsonKey, 'TYPICAL_MAIN_HTML_SIM')
+        assert hasattr(PreDataJsonKey, 'MAIN_HTML_SIM')
 
         # Check actual values
         assert PreDataJsonKey.DOMAIN_NAME == 'domain_name'