new_tiku_structure_2021/utils/html_again_parse.py

# -*- coding: utf-8 -*-
import re
from pprint import pprint

from pyquery import PyQuery as pq


#
# pattern = re.compile(r"\[来源.*?\]|www\..*?com")
#
# filter_words = ["学科网", "高考资源网", "Ziyuanku.com", "WWW.ziyuanku.com",
#                 "高☆考♂资♀源€网", "w.w.w.k.s.5.u.c.o.m", "本资料由《七彩教育网》www.7caiedu.cn 提供!",
#                 "本资料来源于《七彩教育网》http://www.7caiedu.cn", "本资料由《七彩教育网》www.7caiedu.cn 提",
#                 "高考试题来源:"]
#
#
# def filter_word(txt_list):
#     new_txt_list = []
#     for word in txt_list:
#         if not word.strip():
#             continue
#         new_word = re.sub(pattern, "", word)
#         for keys in filter_words:
#             if keys in new_word:
#                 new_word = new_word.replace(keys, "")
#         new_txt_list.append(new_word)
#     return new_txt_list


def filter_data(x):
    if not str(x).replace(" ", "").strip():
        pass
    else:
        return str(x)


def replace_k(con):
    # con = str(con).replace(" ", "+")
    # con = str(con).replace(" ", "+")
    con = re.sub(r'\s(?!(src="http|_src="http|class="tiankong"|data-num=))', "#+#", str(con))
    return pq(con, parser="html")


def css_label_wash(content):
    # todo add 9-4
    """
    清洗文本中带有的css标签
    :param content:
    :return:
    """
    # temp_con = re.sub('</?p(\s*|\s+style=.*?")?>', "", str(content))
    if re.search('</?(span|font|article|ul|ol|div)(\s*|\s+style=.*?")>|text\s*-\s*decoration: underline|border\s*-\s*bottom:', str(content)) is None:
        # content = re.sub('</p\s*>', "\n", content).strip().replace("\n\n", "\n").replace("\n", "<br/>")
        content = re.sub('<p(\s*|\s+style=.*?")>', "<p>", content)
        content = re.sub('</p><p>', "</p>\n<p>", content)
        return content
    else:
        content = re.sub(r'<font\s+style="color: red">', "", str(content))
        content = re.sub(r'<span\s+style="color: red">', "", str(content))
        content = re.sub(r'<span\s+style="color: blue">', "", str(content))
        content = re.sub(r'<font\s+style="color: blue">', "", str(content))
        content = content.replace("</font >", "").replace("</font>", "")
        content = content.replace("</span >", "").replace("</span>", "")
        content = content.replace('<p style="height: 0;">&nbsp;</p>', "\n").replace('<p><br/></p>', "\n")
        content = content.replace("</p>", "\n")
        # content = re.sub('<p( style=.*?")?>', "", content)
        # content = re.sub('<br\s*/?>', "\n", content)
        # parm = False
        # if "<article>" not in str(content):
        #     parm = True
        subs2img = {}
        if re.search('<img.*? src="',content) or "<table/>" in content:
            all_imgs = re.findall('<img.*? src=.*?[\s/"]>|<table.*?>.*?</table>',
                                  content, flags=re.S)
            for k, img in enumerate(all_imgs):
                content = content.replace(img, "&{}&".format(k))
                subs2img["&{}&".format(k)] = img
        content = re.sub(r"<(su[bp])>(.*?)</(su[bp])>", r"【\1】\2【/\3】", content)

        html = pq(content, parser="html")
        a = []

        if html.children():
            for line in html.children().items():  # <p>.*?</p>里面的内容可能会被过滤掉

                test = line.text()
                if str(line).startswith("<p") and line.text().strip():

                    if '<img src="http://' in str(line) or "text-decoration: underline" in str(line) or "border-bottom:" in str(line) or "text - decoration: underline" in str(line):
                        # a.append(line.html().replace("<br />", "\n").replace("<br/>", "\n").replace("<br>", "\n") + "\n")
                        a.append(line.html())
                    else:
                        line = replace_k(line)
                        if "<br>" in str(line) or "<br/>" in str(line) or "<br />" in str(line):
                            line = str(line).replace("<br/>", "###").replace("<br>", "###").replace("<br />", "###")
                            line = pq(line)
                            new_line = list(map(lambda x: str(x).replace("#+#", " "), line.text().split("###")))
                            a.extend(new_line)
                        else:
                            if line.text().strip():
                                a.append(line.text().replace("#+#", " "))

                elif "<article>" in str(line) and "</article>" in str(line):
                    line = re.sub(r'<p.*?>', "", str(line.html()))
                    b = line.replace('</p>', ""). \
                        replace("<br>", "\n"). \
                        replace("<br/>", "\n"). \
                        replace("<br />", "\n"). \
                        replace('<p style="height: 0;">&nbsp;</p>', "\n"). \
                        replace('<p style="height: 0;"> </p>', "\n")
                    b_list = b.split("\n")
                    # b_list = list(filter(lambda x: str(x), b_list))
                    b_list = list(filter(filter_data, b_list))
                    b_list = list(map(lambda x: str(x), b_list))
                    a.extend(b_list)

                elif str(line).startswith("<ul"):
                    a.append(line.text())
                elif line.attr("class") == "slave-datas":
                    for j, ss in enumerate(line.children().items()):
                        if ss(".read-list-title").text():
                            a.append(str(j + 1) + "." + ss(".read-list-title").text())
                        if ss(".read-list-opt").text():
                            a.append(ss(".read-list-opt").text().replace("\n", "\t"))
                        if str(ss).startswith("<ul"):
                            a.append(ss.text())
                elif str(line).startswith("<table"):
                    a.append(str(line))
                elif str(line).startswith("<ol"):
                    for i, ss in enumerate(line.children().items()):
                        a.append(str(i + 1) + "." + ss.text())
                else:
                    # print('test:',line.text())  # 自动去掉了图片
                    if line.text().strip():
                        line = replace_k(line)
                        # a.append(line.text().replace("+", "\xa0") + "\n")
                        a.append(line.text().replace("#+#", " "))

        else:
            # if html.text().strip():  把换行\n 都去掉了
            #     a.append(html.text())
            # a.append(str(html)) # 会自动带上</?p> ；&变为&amp；
            a.append(content.strip())

        new_a = "\n".join(list(filter(lambda x: x.strip(), a)))
        if subs2img:
            new_a = re.sub("|".join(subs2img.keys()), lambda x: subs2img[x.group()], new_a)
        new_a = "<p>" + new_a.replace("\n\n", "\n").replace("\n", "</p>\n<p>") + "</p>"

        # for sb, img in subs2img.items():  # 2021
        #     new_a = new_a.replace(sb, img)

        # if parm:
        #     new_a[0] = "\xa0" * 4 + new_a[0]
        new_a = re.sub(r"【(/?su[bp])】", r"<\1>", new_a)
        return new_a


if __name__ == '__main__':
    cons = r'''
    <div class="stem-wraper" data-v-0b6ea9b4="">
<article>
<p>Nancycarrots are good for eyes.Eat some,please.</p>
</article>
</div>
<div class="topic-analysis" data-v-0b6ea9b4="">
<ul class="topic-analysis-items" data-v-0b6ea9b4="">
<li class="topic-analysis-content" data-v-0b6ea9b4=""><span class="analysis-prefix" data-v-0b6ea9b4="">【答案】</span><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAOQAAABOCAYAAAAw0LoFAAAJY0lEQVR4Ae2dz2sVVxTH+z+5ciF0U7HiRkTciOLOhZuuBVdSshYLrkKDxlU3iqUEqdKGSFMJqU1TaKshJmpNGpvGNs0PSU755vX7cuZ6Z97MvJD7XuZ7Ybi/77n3O+cz581L4H1g+5Smp6ft2LFjdujQIbt9+3au1c3NTbt69erOOIzldfjwYbt3755tb2/nzr127Vp7/MTERHvc5OSknTlzpt3HNS9evGizs7PtcSisra3Z5cuXd8aePn3arl+/bjdu3Mi90I9xWBPzMP+gJq/N+fPnbXl5ufZRZ2Zm7MSJE3uiW57P4J4cPXrURkZGon7z7NkzO3nyZNsvzp49aw8fPrSNjY3ouXDeoaEhO378eHvOlStXbHV1dWc8fPPOnTt26tQpm5qaaq+BNuzlyJEjNjAwEPWlCxcu7Iz5oD1rHwpbW1v29OlTW1hYKLQ2Nja2s+lbt27Zo0eP7PHjx7a0tFQ4B50PHjxoHzYEDf1wKIiKCzcxltB+9+7dnXWQ543j3KrjOa8fc3/W4eHhtiPWOcvi4qINDg6W1rmTDfjV+Ph45pqfny+8fwBodHR050GP/RQ97L19+PHLly93QH/z5o3vsvX1dVtZWcm03b9/386dO2eXLl2yubm5TB8r9N19BZLGlUsBKRBXQEDGdVGrFEiigIBMIruMSoG4AgIyrotapUASBQRkEtllVArEFRCQcV3UKgWSKCAgk8guo1IgroCAjOuiVimQRAEBmUR2GZUCcQUEZFwXtUqBJAoIyCSyy6gUiCsgIOO6qFUKJFFAQCaRXUalQFwBARnXRa1SIIkCAjKJ7DIqBeIKCMi4LmqVAkkUEJBJZJdRKRBXQEDGdVGrFEiigIBMIruMSoG4AgIyrotapUASBQRkEtllVArEFRCQcV3UKgWSKCAgk8guo1IgroCAjOuiVimQRAEBmUR2GZUCcQUEZFwXtUqBJAoIyCSyy6gUiCsgIOO6qFUKJFFAQCaRXUalQFwBARnXRa1SIIkCAjKJ7DIqBeIKCMi4LmqVAkkU6CsgB741+2jI7MNBXWU0gFbQrGoae25284nZ55O6ymgAraDZXqS+ARKO9fFNgVgGRD8GmlWBEo41/KNALAOiHwPN9gLKvgFSkbH+wwjalU2KjPUfRtCu29Q3QPqnvsrV4SzrKP6pr3J1OMvqnDdOQDbkfTTPAcJ2QVgdQq9ZqGfVuoAUkBmf8c6lcnU4M2LWqAhIAZlxG0FYHUKvWUbMGhUBKSAzbuOdS+XqcGbErFERkAIy4zaCsDqEXrOMmDUqAlJAZtzGO5fK1eHMiFmj0igghybNfl1q/clg4pUZLvwJ5ctfzGaX3/9Twidfmf2xmq/q6qbZp9/szkMZ48O2v9bMYDv25xrsB315tkIbsTXKtOWfItuzFxD+8LvZ0r8tZ3711gwX1sVZl9fed/KR38xwzry0uWU2+nx3HsoYzzbkGOPT1rYZ9gG74Xh/Rtr2431/1bLfQ52ygOwAJED1gNH5AVDYB7ABOcYXORjGYR2MwxpYCxcc14Mbs0H7VfOyzlHVAWPj6wAJUAmYXxPAhH3QiYmw+3Eov92IA0kAOR85HhTeZjdlv26dcmOAhOMzARbchFiE9NESQJSNkBj700ILLIAWRkoChH3QBveDHFEUV5iw19gDgeuVzcN18+rdOCPmAh4mRC1ESkLjI6QvxyDhGsh9hCTsHkDYBcywjXbf5/eDtRb+ycLa7XnD+X7fdcqNARKOG35kpWAr661IBVAAEuDC+KIIFfZhLi+sUSZCwil9RCwLV51xPGunPHSwOnVCg7k+mq2/a0GDc0MfgIMxHqDQXtgXAobzwAbamaMNda6F/cA2oMV6PnpyzF7lnfTt1N9oIH2EhFCIXnR2H1E7iQgAX//dGoXyZ+P5EZLr8+GAPeACzHnJPyQ4v2qet3bYvheOGQLpIyTseVhigIV7Yh0QT71uwQyoAdbMn61ev2Z4Bj4U8J7482L+w9I/JMI1yta517p544B8t7V7Qz2QHkY6u3/HQxug4RxESNxoRjgfMTEPHz/zoiQh9DcN0GE92KB9vybb6ubeVlG5rOMVjQOQcP6ij6x+vv+4iXbARojDiIax4Rc4GMs5KHMu1sJ43AdESOwLEKOOMm0Vwez3WaZcpG2ZvsYAyYhH8AgFQWO7d3iAlQcVxAXcBBLrMcEhCBj6pxZ2YfZ2YcvXuUeugxz2+/Edkk7uAQE0bPfOHYPMa+C/ASVgPkLCBtYNE23PrbR0xNzwI3Denvz+qpTDPVStNwZIOD8/JoYgICoRSB8VfRlziiIk+nlhHKGM3RBAyLGEkAAflAjZ6c8egAMQARBfhvMz2qFcJkLCFh5cWIcQEiLA+P2L3X6M4Vjaij0kOL9qHrvfVdoaDyQA4Jc6gATQ4qYOP2nduE5i+iiJtZjwjStgi0VIOBwARIJtAko4uQbyfouQcODYOyTOzC91OAYfH/FOGH4E9ednmVGS8OJdkO+QyNEfJmhH4AlhGBHDelUAw/HhHqrWGw0kxAIYX8/sRkhAxWjJKMYcfYSHbcwxBxeiKt4fv5jOB9KvgTLrmA8bWBNlJuwR75O0VSfnWp3y0MHq1EMgYRNA4AsYRqMiENCHh2LMdjgvHBtGSKzBqMhIiTlcO1yP7XXzTvp26m8MkHB6pJjzAyI4DNN387sRjG1FuQeG0IZfyHgbhA5gEUgPIGxhTLhGHRA5p2j/vq+uI3IegEAiUACQAAAMHw1f/P9u5+0XlXGPsB7WgT2UkWgLbUVAYh/8KM39CsgixQv66FhNyvsRSDp6U/MCFy7V1ZgI2SSQY2ct5Q22+1GuqUB1e+6yOueNE5Du29GYIx+UtjwHCNu7dcimzw/1rFoXkAIy4zNNB6rb82fErFERkAIy4zbdOmTT52fErFERkAIy4zZNB6rb82fErFERkAIy4zbdOmTT52fErFERkAIy4zZNB6rb82fErFHpGyD1UwL1/1NHPyWwP3/OadRPCejHduoBqR/b2R8YG/djO4j+gFKRsjyY0KrKL1/xE5Z+jq4axI38OTo6i3IpcJAV6Jt3yIN8E3Q2KUAFBCSVUC4FekABAdkDN0FbkAJUQEBSCeVSoAcUEJA9cBO0BSlABQQklVAuBXpAAQHZAzdBW5ACVEBAUgnlUqAHFBCQPXATtAUpQAUEJJVQLgV6QAEB2QM3QVuQAlTgP96ZXxLDlEz6AAAAAElFTkSuQmCC" alt=""></li>
<li class="topic-analysis-content" data-v-0b6ea9b4=""><span class="analysis-prefix" data-v-0b6ea9b4="">【解析】</span></li>
</ul>
</div>
    '''
#     cons = '''<p style="margin-top:0;margin-right:0;margin-bottom:0;margin-left:0;text-align:justify;text-justify:inter-ideograph;line-height:150%"><span style="line-height: 150%;font-size: 14px">Unbelievable</span><span style=";font-family:宋体;line-height:150%;font-size:14px"><span style="font-family:宋体">！</span></span><span style="line-height: 150%;font-size: 14px">Oh..., _____ you don&#39;t mind, I&#39;ll stop and take a deep breath.</span></p><p><br/></p>
# <p>【答案】<p style="margin-top:0;margin-right:0;margin-bottom:0;margin-left:0;text-align:justify;text-justify:inter-ideograph;line-height:150%"><span style=";font-family:宋体;line-height:150%;font-size:14px">1</span><span style="line-height: 150%;font-size: 14px">.if</span><span style=";font-family:宋体;line-height:150%;font-size:14px"><span style="font-family:宋体">。考查</span></span><span style="line-height: 150%;font-size: 14px">if</span><span style=";font-family:宋体;line-height:150%;font-size:14px"><span style="font-family:宋体">引导的条件状语从句。根据句意可知。</span></span></p><p><br/></p></p>
# <p>【解析】</p>'''

    # pprint(cons)
    # print(again_parse(cons))
    # print(again_parse(cons))
    # print(list(map(lambda x: str(x).replace("     ", " "), again_parse(cons))))
    # con1 = r'<p>解：A．研究跨栏动作时，刘翔的大小和形状不能忽略，不能看作质点，故A错误；<br/>B．选取不同的参考系，物体的运动状态是不相同的，故B错误；<br/>C．出租车收费是按路程收费的，故C错误；<br/>D．第<img src="http://192.168.1.145:10811/static/physical_formulas_imgs/16184553930794225.png" data-latex="${4 \rm{s} }$" width="13",height="11" />是指<img src="http://192.168.1.145:10811/static/physical_formulas_imgs/16184553931930702.png" data-latex="${1 \rm{s} }$" width="12",height="11" />的时间，是指从<img src="http://192.168.1.145:10811/static/physical_formulas_imgs/16184553930220437.png" data-latex="${3 \rm{s} }$" width="13",height="11" />末到<img src="http://192.168.1.145:10811/static/physical_formulas_imgs/16184553930794225.png" data-latex="${4 \rm{s} }$" width="13",height="11" />末这一段时间，故D正确；<br/>故选：D．</p>'

    cons = css_label_wash(cons)

    print(cons)