new_tiku_structure_v3_sci/utils/html_again_parse_old.py

# -*- coding: utf-8 -*-
import re
from pprint import pprint

from pyquery import PyQuery as pq


#
# pattern = re.compile(r"\[来源.*?\]|www\..*?com")
#
# filter_words = ["学科网", "高考资源网", "Ziyuanku.com", "WWW.ziyuanku.com",
#                 "高☆考♂资♀源€网", "w.w.w.k.s.5.u.c.o.m", "本资料由《七彩教育网》www.7caiedu.cn 提供!",
#                 "本资料来源于《七彩教育网》http://www.7caiedu.cn", "本资料由《七彩教育网》www.7caiedu.cn 提",
#                 "高考试题来源:"]
#
#
# def filter_word(txt_list):
#     new_txt_list = []
#     for word in txt_list:
#         if not word.strip():
#             continue
#         new_word = re.sub(pattern, "", word)
#         for keys in filter_words:
#             if keys in new_word:
#                 new_word = new_word.replace(keys, "")
#         new_txt_list.append(new_word)
#     return new_txt_list


def filter_data(x):
    if not str(x).replace(" ", "").strip():
        pass
    else:
        return str(x)


def replace_k(con):
    # con = str(con).replace(" ", "+")
    # con = str(con).replace(" ", "+")
    con = re.sub(r'\s(?!(src="http|_src="http|class="tiankong"|data-num=))', "+", str(con))
    return pq(con, parser="html")


def again_parse(content):
    # todo add 9-4
    content = re.sub(r'<font\s+style="color: red">', "", str(content))
    content = re.sub(r'<span\s+style="color: red">', "", str(content))
    content = re.sub(r'<span\s+style="color: blue">', "", str(content))
    content = re.sub(r'<font\s+style="color: blue">', "", str(content))
    content = str(content).replace("</font >", "").replace("</font>", "")
    content = str(content).replace("</span >", "").replace("</span>", "")
    content = str(content).replace('<p style="height: 0;">&nbsp;</p>', "\n").replace('<p><br/></p>', "\n")
    # parm = False
    # if "<article>" not in str(content):
    #     parm = True

    html = pq(content, parser="html")
    a = []
    if html.children():
        for line in html.children().items():

            if str(line).startswith("<p") and line.text().strip():

                if '<img src="http://zsytk2.zhixinhuixue.com/static/images' in str(line) or "text-decoration: underline" in str(line) or "border-bottom:" in str(line) or "text - decoration: underline" in str(line):
                    a.append(line.html().replace("<br />", "\n").replace("<br/>", "\n").replace("<br>", "\n") + "\n")
                else:
                    line = replace_k(line)
                    if "<br>" in str(line) or "<br/>" in str(line) or "<br />" in str(line):
                        line = str(line).replace("<br/>", "###").replace("<br>", "###").replace("<br />", "###")
                        line = pq(line)
                        new_line = list(map(lambda x: str(x).replace("+", "\xa0") + "\n", line.text().split("###")))
                        a.extend(new_line)
                    else:
                        if line.text().strip():
                            a.append(line.text().replace("+", "\xa0") + "\n")

            elif "<article>" in str(line) and "</article>" in str(line):
                line = re.sub(r'<p.*?>', "", str(line.html()))
                b = line.replace('</p>', ""). \
                    replace("<br>", "\n"). \
                    replace("<br/>", "\n"). \
                    replace("<br />", "\n"). \
                    replace('<p style="height: 0;">&nbsp;</p>', "\n"). \
                    replace('<p style="height: 0;"> </p>', "\n")
                b_list = b.split("\n")
                # b_list = list(filter(lambda x: str(x), b_list))
                b_list = list(filter(filter_data, b_list))
                b_list = list(map(lambda x: str(x) + "\n", b_list))
                a.extend(b_list)

            elif str(line).startswith("<ul"):
                a.append(line.text() + "\n")
            elif line.attr("class") == "slave-datas":
                for j, ss in enumerate(line.children().items()):
                    if ss(".read-list-title").text():
                        a.append(str(j + 1) + "." + ss(".read-list-title").text() + "\n")
                    if ss(".read-list-opt").text():
                        a.append(ss(".read-list-opt").text().replace("\n", "\t") + "\n")
                    if str(ss).startswith("<ul"):
                        a.append(ss.text() + "\n")
            elif str(line).startswith("<table"):
                a.append(str(line) + "\n")
            elif str(line).startswith("<ol"):
                for i, ss in enumerate(line.children().items()):
                    a.append(str(i + 1) + "." + ss.text() + "\n")
            else:
                if line.text().strip():
                    line = replace_k(line)
                    a.append(line.text().replace("+", "\xa0") + "\n")
    else:
        if html.text().strip():
            a.append(html.text() + "\n")
    new_a = list(filter(lambda x: x.strip(), a))
    # if parm:
    #     new_a[0] = "\xa0" * 4 + new_a[0]
    return new_a


if __name__ == '__main__':
    cons = '''<p>____ prize&nbsp;for&nbsp;the&nbsp;winner&nbsp;of&nbsp;the&nbsp;competition&nbsp;is&nbsp;____&nbsp;two-week&nbsp;holiday&nbsp;in&nbsp;Paris.</p><ul class="option-datas list-paddingleft-2"><li><p><span class="lis-opt">A.</span>The;&nbsp;/</p></li><li><p><span class="lis-opt">B.</span>A;&nbsp;/</p></li><li><p><span class="lis-opt">C.</span>A;&nbsp;the</p></li><li><p><span class="lis-opt">D.</span>The;&nbsp;a</p></li></ul>
<p>【答案】D</p>
<p>【解析】考查冠词，第一个空格表示对比赛获胜者的奖励，是特指，故用the，第二个空格泛指一次为期两周的假期，用不定冠词a，故选D项。句意，这次比赛给获胜者的奖励是在巴黎度假两周。</p>'''

#     cons = '''<p style="margin-top:0;margin-right:0;margin-bottom:0;margin-left:0;text-align:justify;text-justify:inter-ideograph;line-height:150%"><span style="line-height: 150%;font-size: 14px">Unbelievable</span><span style=";font-family:宋体;line-height:150%;font-size:14px"><span style="font-family:宋体">！</span></span><span style="line-height: 150%;font-size: 14px">Oh..., _____ you don&#39;t mind, I&#39;ll stop and take a deep breath.</span></p><p><br/></p>
# <p>【答案】<p style="margin-top:0;margin-right:0;margin-bottom:0;margin-left:0;text-align:justify;text-justify:inter-ideograph;line-height:150%"><span style=";font-family:宋体;line-height:150%;font-size:14px">1</span><span style="line-height: 150%;font-size: 14px">.if</span><span style=";font-family:宋体;line-height:150%;font-size:14px"><span style="font-family:宋体">。考查</span></span><span style="line-height: 150%;font-size: 14px">if</span><span style=";font-family:宋体;line-height:150%;font-size:14px"><span style="font-family:宋体">引导的条件状语从句。根据句意可知。</span></span></p><p><br/></p></p>
# <p>【解析】</p>'''

    # pprint(cons)
    # print(again_parse(cons))
    # print(again_parse(cons))
    # 2021_04_14_09_38_55
    path2 = r"F:\zwj\Text_Structure\fail_files\2021_04_14_09_38_55.html"
    html = open(path2, "r", encoding="utf-8").read()
    # print(html)
    print(list(map(lambda x: str(x).replace("     ", " "), again_parse(html))))