new_tiku_structure_v3_sci/utils/html_again_parse.py

# -*- coding: utf-8 -*-
import re
from pprint import pprint

from pyquery import PyQuery as pq


#
# pattern = re.compile(r"\[来源.*?\]|www\..*?com")
#
# filter_words = ["学科网", "高考资源网", "Ziyuanku.com", "WWW.ziyuanku.com",
#                 "高☆考♂资♀源€网", "w.w.w.k.s.5.u.c.o.m", "本资料由《七彩教育网》www.7caiedu.cn 提供!",
#                 "本资料来源于《七彩教育网》http://www.7caiedu.cn", "本资料由《七彩教育网》www.7caiedu.cn 提",
#                 "高考试题来源:"]
#
#
# def filter_word(txt_list):
#     new_txt_list = []
#     for word in txt_list:
#         if not word.strip():
#             continue
#         new_word = re.sub(pattern, "", word)
#         for keys in filter_words:
#             if keys in new_word:
#                 new_word = new_word.replace(keys, "")
#         new_txt_list.append(new_word)
#     return new_txt_list


def filter_data(x):
    if not str(x).replace(" ", "").strip():
        pass
    else:
        return str(x)


def replace_k(con):
    # con = str(con).replace(" ", "+")
    # con = str(con).replace(" ", "+")
    con = re.sub(r'\s(?!(src="http|_src="http|class="tiankong"|data-num=))', "#+#", str(con))
    return pq(con, parser="html")


def css_label_wash(content):
    # todo add 9-4
    """
    清洗文本中带有的css标签
    :param content:
    :return:
    """
    # temp_con = re.sub('</?p(\s*|\s+style=.*?")?>', "", str(content))
    if re.search('</?(span|font|article|ul|ol|div)(\s*|\s+style=.*?")>|text\s*-\s*decoration: underline|border\s*-\s*bottom:', str(content)) is None:
        # content = re.sub('</p\s*>', "\n", content).strip().replace("\n\n", "\n").replace("\n", "<br/>").
        content = re.sub('<p(\s*|\s+style=.*?")>', "<p>", content)
        content = re.sub('</p><p>', "</p>\n<p>", content)
        return content
    else:
        content = re.sub(r'<font\s+style="color: red">', "", str(content))
        content = re.sub(r'<span\s+style="color: red">', "", str(content))
        content = re.sub(r'<span\s+style="color: blue">', "", str(content))
        content = re.sub(r'<font\s+style="color: blue">', "", str(content))
        content = content.replace("</font >", "").replace("</font>", "")
        # content = content.replace("</span >", "").replace("</span>", "")
        content = content.replace('<p style="height: 0;">&nbsp;</p>', "\n").replace('<p><br/></p>', "\n")
        # content = content.replace("</p>", "</p>\n")  # 2022-4-25
        content = re.sub(r"</p>\s*(?!\n)", "</p>\n", content)
        content = re.sub(r"<br\s*/?>", "<p>", content)  # 2022/1/6
        # content = re.sub('<p( style=.*?")?>', "", content)
        # content = re.sub('<br\s*/?>', "\n", content)
        # parm = False
        # if "<article>" not in str(content):
        #     parm = True
        subs2img = {}
        if re.search('<img.*? src="',content) or "</table>" in content:
            all_imgs = re.findall('<img.*? src=.*?[\s/"]>|<table.*?>.*?</table>',
                                  content, flags=re.S)
            for k, img in enumerate(all_imgs):
                content = content.replace(img, "&{}&".format(k))
                subs2img["&{}&".format(k)] = img
        content = re.sub(r"<(su[bp])>(.*?)</(su[bp])>", r"【\1】\2【/\3】", content)
        content = content.replace("&lt;", "【#lt;】")

        html = pq(content, parser="html")
        a = []

        if html.children():
            for line in html.children().items():  # <p>.*?</p>里面的内容可能会被过滤掉

                test = line.text()
                # 保留下划线及着重符标签   <span style="text-decoration: underline;">
                # 波浪线：<span style="text-decoration: underline wavy;">
                # pq会将多个空格换成一个
                if '<span style="text-decoration: underline' in str(line) or '<span class="dots"' in str(line) \
                        or '<p style="text-indent:' in str(line) or '<p style="text-align:' in str(line):
                    line = re.sub(r'<span style="text-decoration: underline(.*?">.+?)</span>', r"【1#\1##】", str(line))
                    line = re.sub(r'<span class="dots">(.+?)</span>', r"【2#\1##】", str(line))
                    line = re.sub(r'<(p style="text-(indent|align):.*?">.+?)</p>', r"【\1##3】", str(line))
                    line = line.replace(" ", "【+】")
                    line = pq(line)
                    new_line = list(map(lambda x: str(x).replace("【1#", '<span style="text-decoration: underline')
                                        .replace("##】", "</span>").replace("【2#", '<span class="dots">')
                                        .replace("【p【+】style=", "<p style=").replace("##3】", "</p>").replace("【+】",
                                                                                                             " "),
                                        line.text().split("\n")))
                    a.extend(new_line)
                elif str(line).startswith("<p") and line.text().strip():
                    if '<img src="http://' in str(line) or "text-decoration: underline" in str(line) or "border-bottom:" in str(line) or "text - decoration: underline" in str(line):
                        # a.append(line.html().replace("<br />", "\n").replace("<br/>", "\n").replace("<br>", "\n") + "\n")
                        a.append(line.html())
                    else:
                        line = replace_k(line)
                        if re.search(r"\n|<br\s*/?>", str(line), flags=re.S):
                            line = re.sub(r"\n|<br\s*/?>", "#*#", str(line), flags=re.S)
                            line = pq(line)
                            new_line = list(map(lambda x: str(x).replace("#+#", " "), line.text().split("#*#")))
                            a.extend(new_line)
                        else:
                            if line.text().strip():
                                a.append(line.text().replace("#+#", " "))

                elif "<article>" in str(line) and "</article>" in str(line):
                    line = re.sub(r'<p.*?>', "", str(line.html()))
                    b = line.replace('</p>', ""). \
                        replace("<br>", "\n"). \
                        replace("<br/>", "\n"). \
                        replace("<br />", "\n"). \
                        replace('<p style="height: 0;">&nbsp;</p>', "\n"). \
                        replace('<p style="height: 0;"> </p>', "\n")
                    b_list = b.split("\n")
                    # b_list = list(filter(lambda x: str(x), b_list))
                    b_list = list(filter(filter_data, b_list))
                    b_list = list(map(lambda x: str(x), b_list))
                    a.extend(b_list)
                elif str(line).startswith("<ul"):
                    a.append(line.text())
                elif line.attr("class") == "slave-datas":
                    for j, ss in enumerate(line.children().items()):
                        if ss(".read-list-title").text():
                            a.append(str(j + 1) + "." + ss(".read-list-title").text())
                        if ss(".read-list-opt").text():
                            a.append(ss(".read-list-opt").text().replace("\n", "\t"))
                        if str(ss).startswith("<ul"):
                            a.append(ss.text())
                elif str(line).startswith("<table"):
                    a.append(str(line))
                elif str(line).startswith("<ol"):
                    for i, ss in enumerate(line.children().items()):
                        a.append(str(i + 1) + "." + ss.text())
                else:
                    # print('test:',line.text())  # 自动去掉了图片
                    if line.text().strip():
                        line = replace_k(line)
                        # a.append(line.text().replace("+", "\xa0") + "\n")
                        a.append(line.text().replace("#+#", " "))

        else:
            # if html.text().strip():  把换行\n 都去掉了
            #     a.append(html.text())
            # a.append(str(html)) # 会自动带上</?p> ；&变为&amp；
            a.append(content.strip())

        new_a = "\n".join(list(map(lambda x: str(x).strip(), a)))
        if subs2img:
            new_a = re.sub("|".join(subs2img.keys()), lambda x: subs2img[x.group()], new_a)
        new_a = "<p>" + new_a.replace("\n\n", "\n").replace("\n", "</p>\n<p>") + "</p>"

        # for sb, img in subs2img.items():  # 2021
        #     new_a = new_a.replace(sb, img)

        # if parm:
        #     new_a[0] = "\xa0" * 4 + new_a[0]
        new_a = re.sub(r"【(/?su[bp])】", r"<\1>", new_a).replace("【#lt;】", "&lt;")
        return new_a


if __name__ == '__main__':
    cons = r'''
    <div>(1)求货物和重物的质量关系;</div>
(2)要使货物运送到B端,若采用在重物下方挖坑的方法,求至少挖多深的坑;<br>(3)要使货物运送到B端,若采用配重落地时传送带立刻顺时针转动的方法(启动时间可<br>忽略),求传送带速度大小的范围和货物从A端传送到B端所用时间的范围(结果保留三位有<br>效数字)。<br><br><br><br>物理参考答案<br>
<div>1.C</div>
<div>【解析】绝大多数α粒子沿直线穿过,偏转角很</div>
<div>小,说明原子核很小;A项错误;少数α粒子穿过金箱</div>
<div>后发生较犬角度的偏转是由于少数α粒子穿过金箱</div>
<div>时距离金原子核较近,受到的库仑斥力较大,B项错</div>
<div>误;极少数α粒子被弹回,说明原子核是一个体积小、</div>
    '''
    cons1 = '''
    9 . 中国古代的政治权力由“传贤”转变为“传子”，“家天下”制度开始形成于<table name=\"optionsTable\" style=\"width:100%;table-layout:fixed;\" cols=\"4\"><tr><td>A．夏朝</td><td>B．商朝</td><td>C．周朝</td><td>D．秦朝</td></tr></table>
    '''

    # pprint(cons)
    # print(again_parse(cons))
    # print(again_parse(cons))
    # print(list(map(lambda x: str(x).replace("     ", " "), again_parse(cons))))
    # con1 = r'<p>解：A．研究跨栏动作时，刘翔的大小和形状不能忽略，不能看作质点，故A错误；<br/>B．选取不同的参考系，物体的运动状态是不相同的，故B错误；<br/>C．出租车收费是按路程收费的，故C错误；<br/>D．第<img src="http://192.168.1.145:10811/static/physical_formulas_imgs/16184553930794225.png" data-latex="${4 \rm{s} }$" width="13",height="11" />是指<img src="http://192.168.1.145:10811/static/physical_formulas_imgs/16184553931930702.png" data-latex="${1 \rm{s} }$" width="12",height="11" />的时间，是指从<img src="http://192.168.1.145:10811/static/physical_formulas_imgs/16184553930220437.png" data-latex="${3 \rm{s} }$" width="13",height="11" />末到<img src="http://192.168.1.145:10811/static/physical_formulas_imgs/16184553930794225.png" data-latex="${4 \rm{s} }$" width="13",height="11" />末这一段时间，故D正确；<br/>故选：D．</p>'

    cons = css_label_wash(cons1)

    print(cons)