word_parse_of_eng/英语word解析代码说明/get_ans_structure.txt

# -*- coding: utf-8 -*-
"""
答案提取原则：按行提取，按不同类型答案先后提取
              1.提取听力材料（可只能出现在答案开头或答案末尾）
			     特征：开始关键词：听力（原文|材料|文稿）等 结尾关键词:Text 10
			  2.提取客观题答案（答案是ABCD这样的）
"""







import os
from collections import Counter, OrderedDict
from pprint import pprint

import jieba
import operator
import Levenshtein

from flow.get_item_type import type_keyw_judge, item_type_classify
from util import *

chinese_content = []

def get_obj_ans(one_line):
    """
    judge one line is objective answers,that is all answers is ABCDEFG
    1.exist ACBD ([ABCDEFG](2,7))
    2.may has English words,eg:shanghai test_paper:Cloze:21-25 BADCB << Only start of line has English Word
    :param one_line:
    :return: if return {},not obj,no ABCD, if return dict,that is {id:ans}, if return [],only has ans,but no id,need further process
    """
    if re.search(r"[A-K]",one_line) and re.search(r"[A-K][a-z]",one_line) is None:
        one_line = re.sub(r'\d+分', '分数', one_line)
        one_line = re.sub(r'l\s*[\.\-—~]\s*(\d)', r"1-\1", one_line)
        one_line = re.sub(r'(\d)l\s*[\.\-—~]',r"\1犇1-",one_line)
        one_line = re.sub(r'l(\d)\s*[\.\-—~]', r"1\1", one_line)
        one_line = re.sub(r'[Ss](\d)\s*[\.\-—~]', r"5\1", one_line)
        one_line = re.sub(r'(\d)[Ss]\s*[\.\-—~]', r"\1犇5-", one_line)
        one_line = re.sub(r'[Ss]\s*[\.\-—~]\s*(\d)', r"5-\1", one_line)
        one_line = re.sub(r'(\d)G\s*[\.\-—~]', r"\1犇6-" , one_line)
        one_line = re.sub(r'G(\d)\s*[\.\-—~]', r"6\1", one_line)
        one_line = re.sub(r'G[\.\-—~]\s*(\d)', r"6-\1", one_line)
        one_line = re.sub(r'(\d)[oO]\s*[\.\-—~]', r"\1犇0-", one_line)

        one_line = one_line.replace("犇","")
        ###答案是成块给出还是一一对应
        block_ans = re.search(r"\d+([-—]+|~)\d+",one_line) or  len(re.findall(r"[A-D]{2,}",one_line))> 0

        one_line_list = [i for i in re.split(r'\s+', one_line)]  # if re.search(r'[\u4e00-\u9fa5]', i) == None]
        ids = []
        ABCD = []
        id_list = []  # 连续的题号序列
        if block_ans == False: #可能是一个题号写一个答案,答案和题号一一对应
            id_list = [i for i in re.findall('\d+', one_line) if int(i) <= 120]
            one_line = re.sub(r"[\s\n]","",one_line)
            if len(one_line) > len("".join(map(str,id_list)))+2*len(id_list)+6:  #一一对应的话，提取出来的数字和答案是的长度关系，乘以2是可能有.加6最多加句首可能的描述性语言
                id_list = []
            ids = id_list  #下面很多地方用到了ids
        else:
            ids = sum([ [int(i[0]),int(i[-1])] for i in re.findall(r"(\d+)[^\d]{0,3}(\d+)",one_line)], [])
            if len(ids):
                id_list = [str(i) for i in range(min(ids), max(ids) + 1)]
        if len(id_list):
            one_line = one_line[one_line.index(id_list[0]):]
            ABCD = [i for i in english_alpha_extract(one_line)]

        if len(id_list) == len("".join(ABCD)):  #题号编码和答案个数相等
            return dict(zip(id_list,list("".join(ABCD))))
        else:
            obj_id_ans = OrderedDict()
            if len(ABCD) >= 1 and len(re.findall('[a-z]{2,15}', one_line)) <= 3:  # Keys: 1-5 BCABA
                sids = sorted([int(i) for i in ids])
                int_ids = [int(i) for i in ids]
                if len(set(ids)) < len(ids) or operator.eq(sids, int_ids) == False:
                    one_line = ' '.join(one_line_list)  # 听力(30分)     1-5 CCAAB     6-10 ABBCB    11-15 CBCAB    16-20 CACAB
                    int_ids = [int(i) for i in ids]
                    one_line = one_line.replace(' ', '')
                    ABCD = english_alpha_extract(one_line)
                    len_ABCD = sum([1 for i in ABCD if len(i) > 1])
                    if len_ABCD != len(ABCD):
                        ABCD = []
                if len(ids) == 2 * len(ABCD):  # 说明id是两端点的题号,且题号没有丢失
                    for k, v in enumerate(ABCD):
                        lid = int_ids[k * 2]
                        rid = int_ids[k * 2 + 1]
                        if len(v) == rid - lid + 1:  # ABCD none deletion
                            sub_id = [str(iid) for iid in range(lid, rid + 1)]
                            sub_v = [i for i in v]
                            if len(sub_id) == len(sub_v):
                                obj_id_ans.update(dict(zip(sub_id, sub_v)))
                else:
                    if len(int_ids) > 0:
                        all_ids = [str(i) for i in range(min(int_ids), max(int_ids) + 1)]
                        opts = [i for i in ''.join(ABCD) if i != '']
                        if len(all_ids) == len(opts):
                            obj_id_ans.update(dict(zip(all_ids, opts)))
                        elif len(all_ids) > len(opts):  # 61-64 63-66: CDBD << deletion有选项丢失,every part to get
                            for i, iid in enumerate(ids):
                                if i % 2 != 0:
                                    st = one_line.index(str(iid))
                                    ed = 0
                                    if i < len(int_ids) - 1:
                                        # print('one_line:{},索引值是:{}'.format(one_line,str(int_ids[i+1])))
                                        ed = one_line.index(str(ids[i])) + 1
                                    else:
                                        ed = len(one_line)
                                    str_part = one_line[st:ed]
                                    sub_ABCD = re.findall(r'[ABCDEFGHIJK]{2,15}', str_part)
                                    if len(sub_ABCD) > 0:
                                        sub_ids = [str(i) for i in range(int_ids[i - 1], int(iid) + 1)]
                                        sub_A_B_C_D = [i.strip() for i in ''.join(sub_ABCD) if i.strip() != '']
                                        if len(sub_A_B_C_D) == len(sub_ids):
                                            obj_id_ans.update(dict(zip(sub_ids, sub_A_B_C_D)))

                    elif len(ids) <= len(ABCD) and len(ABCD) > 1:
                        return ABCD

            else:
                A_B_C_D = re.findall('[ABCDEFGHJIK]\s*', one_line)
                if re.search('[a-z]{2,15}', one_line) == None and len(A_B_C_D) >= 3:  # 答案是分散的
                    ids = [i for i in re.findall('\d+', one_line) if int(i) < 120]
                    if len(ids) == len(A_B_C_D):
                        obj_id_ans.update(dict(zip(ids, [i.strip() for i in A_B_C_D])))
            return obj_id_ans
    else:
        return {}


def get_word_ans(one_line):
    """
    answer is English word(one or word group)  no other symbols like → no Chinese,may have \、
    语法填空,单词拼写,课文填空,短文填空,根据中文完成下面的短语,翻译句子都可以提取
    use
    :param one_line:
    :return:
    """
    if get_obj_ans(one_line) == {}:
        word_id_ans = OrderedDict()
        if is_parse(one_line) == False:
            ids = [i for i in re.findall('\d+', one_line) if int(i) < 120]
            ids = [iid for iid in ids if
                   len(re.findall(r'[a-z]{1,15}', one_line[one_line.index(iid):], re.I)) > 0]  ##id后面一定有单词
            if len(ids) > 0:#若第一个id前面有三个单词,或者第一个id前面有超过10个字符,就认为不是答案
                if len(ids)==1: #对于等于1的,很有可能就不是答案,而是句子中有一个数字,提取错了
                    if len(re.findall(r'[a-zA-Z]{1,15}', one_line[:one_line.index(ids[0])])) >= 1: #只要数字前面有至少一个单词,就认为不是答案
                        ids = []
                else:
                    if len(re.findall(r'[a-zA-Z]{1,15}', one_line[:one_line.index(ids[0])])) >= 3 or one_line.index(
                            ids[0]) >= 10:
                        ids = []
                #        word = re.findall('[a-z]{2,15}', one_line[one_line.index(ids[0]):])
            ids = del_outlier(ids)
            if len(ids) > 0:  # re.search(r'[,，!！→]|[\u4e00-\u9fa5]', one_line) == None and
                # 曾遇到一种情况，１. branches    2. predict，，正则知道第一个是1,但字符串不知道,用index取不到，，字体不一样
                try:
                    nword = [strip_point(i) for i in re.split('|'.join(ids), one_line[one_line.index(ids[0]):]) if re.search(r'[A-Za-z]+', i) != None]
                except:
                    nword = [strip_point(i) for i in re.split('|'.join(ids), one_line) if re.search(r'[A-Za-z]+', i) != None]
                b = True
                if len(nword) == len(ids):
                    for k, w in enumerate(nword):
                        try:
                            if w not in one_line[one_line.index(ids[k]):]:
                                b = False
                                break
                        except:
                            if w not in one_line:
                                b = False
                                break
                    if b:
                        word_id_ans.update(dict(zip(ids, nword)))
                else:
                    nids = del_outlier(ids)  # 57.词组 (10分)   1) dress up      2) keep her word  3) getting away with  4) cut down
                    if len(nids) == len(nword):
                        word_id_ans.update(dict(zip(nids, nword)))  # 38.dress up 39.keep her word 40.$25.3
                    else:
                        nword = [' '.join(re.findall(r'[A-Za-z]+', i)) for i in
                                 re.split('|'.join(nids), one_line[one_line.index(nids[0]):]) if
                                 len(re.findall(r'[A-Za-z]+', i)) > 0]
                        if len(nword) == len(nids):
                            word_id_ans.update(dict(zip(nids, nword)))
        else: #判断是解析:
            may_id = re.search(r"(\d+)",one_line[:5])
            if may_id:
                word_id_ans["parse_{}".format(may_id.group(1))] = one_line
            else:
                if re.search(r"改[为成]|删[除掉]|[增添]加",one_line) is None:  #否则会把短文改错提断
                    word_id_ans["parse_0"] = one_line
            return word_id_ans
        return word_id_ans


def chinese_start_line(line):
    """
    judge this line is ans_description
    :param line:
    :return:
    """
    line = re.sub("[\s\n]","",line)
    # chinese = re.match(r'[\u4e00-\u9fa5VI]',line)
    # chinese = re.findall(r'[\u4e00-\u9fa5]', line)
    # english = re.findall(r'[a-zA-Z]{2,15}', line)
    if re.search(r'(篇章|语篇)解析|[答考试真\d大小本该]题',line[:8]) and re.search(r'[a-zA-Z]',line) == None:
        return True
    elif re.match(r'[ⅫⅪⅩⅨⅧⅦⅥⅤⅣⅢⅡⅠ]', line) != None :
        return True
    elif re.match(r'(IV|VI|III|V|VII|VIII)\.', line) != None:
        return True
    elif re.search(r'读后续写|书面表达|单词拼写|写作|作文|七选五|改错|阅读(理解|表达)|语法填空|完[型形]|任务型阅读|One possible version|Writing version|Translation', line,re.I) :#
        #and len(re.findall(r'[a-z]{1,15}', line, re.I)) <= 5:
        return True
    elif re.search(r'第.(部分|小?节|卷)',line):
        return True
    elif len(re.findall(r"[评给总打满\d]分|档次|分数",line)) > 1:
        return True
    else:
        return False

# def get_essay_ans(ans_list, ans_result, chinese_index):
def get_essay_ans(ans_result):
    """
    :param ans_list:
    :return:
    """

    chinese_index = [k for k, v in enumerate(ans_result) if
                     v == 'chinese' or isinstance(v, dict) or isinstance(v, OrderedDict)]
    # chinese_index = chinese_index.reverse()

    for i, ci in enumerate(chinese_index):
        start = ci
        end = 0
        if i < len(chinese_index) -1 :
            end = chinese_index[i + 1]
        else:
            end = len(ans_result)
        if end - start >= 3 and end - start <= 20:
            part = "\n".join(ans_result[start + 1:end])
            ###把最后面没有英文字母的连续文字去掉,否则可能出现,最后一行是噪声汉字,但由于汉字较多,影响了下文的判断
            count = -30
            while count < 0:
                if re.search(r"[\u4e00-\u9fa5]",part[count:]) is not None and re.search(r"[a-zA-Z]",part[count:]) is not None:
                    count += 1
                else:
                    if count == -30:
                        count = len(part)
                    break
            part = part[:count]

            if len(set(list_ele_type(part))) == 1 and 'str' in list_ele_type(part):
                english_word = re.findall(r'[a-z]{2,15}', part)
                chinese_word = re.findall(r'[\u4e00-\u9fa5]', part)
                nums = del_outlier(re.findall(r'\d+', part))
                if len(chinese_word) <= 4 and len(english_word) >= 80 and len(nums) < end - start:
                    nans_result = []
                    for k, v in enumerate(ans_result):
                        if k <= start or k >= end:
                            nans_result.append(v)
                    #####利用答案前面的说明做题型分类
                    ty = "短文"
                    if start > 2:
                        exp = ""
                        for i in range(1,start-2):
                            line_exp = ans_result[start-i]
                            if isinstance(line_exp,str):
                                if line_exp == "chinese":
                                    exp += chinese_content[ans_result[:start].count("chinese")-1]
                                else:
                                    exp += line_exp
                            else:
                                break
                        if exp != "":
                            ty = type_keyw_judge(exp)
                            if ty is None:
                                ty = "短文"
                    nans_result.insert(start + 1, {ty: part})
                    ans_result = nans_result
                    return get_essay_ans(ans_result)
                else:
                    if end == len(ans_result):
                        return ans_result
                    else:
                        pass
        else:
            if end == len(ans_result):
                return ans_result
            else:
                pass


def get_listening_ans(ans_lines):
    """
    :param ans_lines: list,all ans lines
    :return:
    """
    start = -1
    endL = 0
    endR = 0
    for k, i in enumerate(ans_lines):
        if re.search(r'听力材料|听力原文|录音原文|听力录音稿|听力录音材料|听力部分录音稿|听力文稿|Text\s*1(?![0o])|Text\s+[Oo0]ne', i) != None:
            start = k
            break
    if start > 5:  # that is : listening is the last ans
        listening = {'听力原文': ''.join(ans_lines[start:])}
        ans_lines = ans_lines[:start]
        ans_lines.append(listening)
        return ans_lines
    else:
        b1 = re.search(r'Text\s+Ten|Text\s+(10|l0|1o)|听录音，根据短文内容完成', ' '.join(ans_lines)) != None
        if b1:
            for k, i in enumerate(ans_lines):
                if re.search(r'Text\s+Ten|Text\s+(10|l0|1o)|听录音，根据短文内容完成', i) != None:
                    endL = k
                    break
            for k2, i2 in enumerate(ans_lines[endL + 1:]):
                if k2 > endL + 1 and re.match(r'[\u4e00-\u9fa5ⅫⅪⅩⅨⅧⅦⅥⅤⅣⅢⅡⅠ]', i2) != None or get_obj_ans(i2) != {}:
                    endR = endL + k2
                    break
        else:
            #####没有结尾标识,并且start<=5或许还等于-1
            if start != -1:
                for k3, i3 in enumerate(ans_lines[start + 1:]):
                    if start != -1 and k > 20 + start and \
                            re.match(r'[\u4e00-\u9fa5ⅫⅪⅩⅨⅧⅦⅥⅤⅣⅢⅡⅠ]',i) != None or get_obj_ans(i) != {}:
                        endR = k3 + start
        if endR != 0:
            if start == -1:
                start = 0
            nans_lines = ans_lines[:start]
            nans_lines.insert(start, {'听力原文': '\n'.join(ans_lines[start:endR])})
            nans_lines.extend(ans_lines[endR + 1:])
            return nans_lines
        else:
            return ans_lines


def error_correct(ans_result):
    """
    提取短文改错
    :return:
    """
    # chinese_index = [k for k, v in enumerate(ans_result) if
    #                  v == 'chinese' or isinstance(v, dict) or isinstance(v, OrderedDict) or (
    #                  k > 5 and v.startswith("Dear"))]
    chinese_index = [k for k, v in enumerate(ans_result) if
                     v == 'chinese' or isinstance(v, dict) or isinstance(v, OrderedDict)]# or (
                             # k > 5 and "".join(v).startswith("Dear"))]  #2019-7-9把Dear行当chinese,会使得答案里少这行的内容

    # chinese_index = chinese_index.reverse()
    for i, ci in enumerate(chinese_index):
        start = ci
        end = 0
        if i < len(chinese_index) - 1:
            end = chinese_index[i + 1]
        else:
            end = len(ans_result)
        # if end - start > 5 and end - start < 30:
        part = ans_result[start + 1:end]
        if len(set(list_ele_type(part))) == 1 and 'str' in list_ele_type(part):
            each_line_word = ['0' if len(re.findall(r'[a-z]{2,15}', ep)) < 5 else '1' for ep in part]
            chinese_word = re.findall(r'[\u4e00-\u9fa5]', ''.join(part))
            b1 = len(re.findall(r'添加|加上|去掉|插入|删掉|删除|∧|︿|改为', ''.join(part))) > 0
            b2 = len(re.findall(r'[a-z]{2,15}', ''.join(part))) > 90 and len(chinese_word) < 20
            b3 = '101010' in ''.join(each_line_word)
            b4 = False
            if re.search(r'\d+', ''.join(part)) == None:
                part_str = ''.join(part).replace('  ', '#')
                partL = [strip_point(i) for i in re.split(r'[^a-zA-Z0-9\s]', part_str) if strip_point(i) != '']
                editd = []
                if len(partL) % 2 == 0:
                    for i in range(0, len(partL)):
                        if i % 2 == 0:
                            editd.append(Levenshtein.jaro(partL[i], partL[i + 1]))
                    if sum(editd) / (len(editd) + 0.0001) > 0.6:
                        b4 = True

            if (b2 and b3) or b1 or b4:
                nans_result = []
                for k, v in enumerate(ans_result):
                    if k <= start or k >= end:
                        nans_result.append(v)
                # nans_result = [ans_result[i] for i in range(len(ans_result)) if i<=ci or i>=chinese_index[i+1]]
                # dd = len(nans_result)
                if len(re.findall(r'[a-zA-Z]{2,}','\n'.join(part))) >= 15:  #10个题目,英语单词至少得有15
                    nans_result.insert(start + 1, {'短文改错': '\n'.join(part)})
                    ans_result = nans_result
            else:
                continue
        else:
            continue

    return ans_result


def ans_structure(ans_list):
    """
    anslist:del empty line first
    :param ans_list:
    :return:
    """
    ans_list = [i for i in ans_list if strip_point(i) != '' and re.search(r"共.[1,5]页|第.[1,5]页",i) is None]
    ans_result = []
    # chinese_index = []
    ans_list = get_listening_ans(ans_list)  #提取听力材料
    for line in ans_list:
        if isinstance(line, dict) :
            ans_result.append(line)
        else:
            obj_ans = get_obj_ans(line) #提取客观题答案
            if obj_ans != {}:
                ans_result.append(obj_ans)
            else:
                word_ans = get_word_ans(line) #提取英语单词类答案
                if word_ans != {}:
                    ans_result.append(word_ans)
                else:
                    if chinese_start_line(line) == True:
                        ans_result.append('chinese')
                        chinese_content.append(line)

                        # chinese_index.append(ans_list.index(line))
                    else:
                        #                    ans_result.append(None)
                        ans_result.append(line)

    if 'chinese' in ans_result and 'str' in list_ele_type(ans_result):
        ans_result = error_correct(ans_result) #提取短文改错
    # if 'chinese' in ans_result and 'str' in list_ele_type(ans_result):
    if 'str' in list_ele_type(ans_result):
        ans_result = get_essay_ans(ans_result)  #提取短文类答案

    ans_result = double_check_ans_structure(ans_result)
    new_ans_result = repeated_id_partandtype(ans_result, ans_list)
    parse_0_correct(new_ans_result)
    parse_extract(new_ans_result) #都是原地修改的,不用重新赋值
    return new_ans_result


def double_check_ans_structure(ans_result):
    """
    after ans_structure(ans_result),the result is :ans_result,according to the characteristics of forward and backward answers,
    re-extract the answers that were not extracted for the first time
    :param ans_result:
    :return: new ans_result
    """
    try:
        ty = list_ele_type(ans_result)
        ans_str_ind = []
        for k, t in enumerate(ty):
            if (t == 'str' and ans_result[k] != 'chinese'):  # list:obj ans but no id
                if re.search(r'[a-z]{1,15}', ans_result[k], re.I) != None:
                    ans_str_ind.append(k)

        if len(ans_str_ind) > 0:
            for astr in ans_str_ind:
                # 没有提取出来的ABCD
                # forward:
                f_id = -1
                b_id = -1
                if len([int(i) for i in re.findall(r'\d+', ans_result[astr]) if int(i) < 120]) > 0:
                    for d1 in range(astr - 1, 0, -1):
                        if isinstance(ans_result[d1], dict) or isinstance(ans_result[d1], OrderedDict):
                            keyL1 = list(ans_result[d1].keys())
                            str_keyL1 = list(map(str, keyL1))
                            if (len(keyL1) > 0) and re.search(r'[\u4e00-\u9fa5]|parse', ''.join(str_keyL1)) == None:
                                f_id = max([int(i) for i in list(keyL1)])
                                break
                    for d2 in range(astr + 1, len(ans_result)):
                        if isinstance(ans_result[d2], dict) or isinstance(ans_result[d2], OrderedDict):
                            keyL2 = list(ans_result[d2].keys())
                            str_keyL2 = list(map(str, keyL2))
                            if (len(keyL2) > 0) and re.search(r'[\u4e00-\u9fa5]|parse', ''.join(str_keyL2)) == None:
                                b_id = min([int(i) for i in list(keyL2)])
                                break
                    if f_id != -1 and b_id != -1:
                        split_ind = [str(i) for i in range(f_id + 1, b_id)]
                        if len(split_ind) > 0 and split_ind[0] in ans_result[astr]:
                            ct = [strip_point(i) for i in
                                  re.split('|'.join(split_ind), (ans_result[astr])[ans_result[astr].index(split_ind[0]):])
                                  if strip_point(i) != '']
                            if len(ct) == len(split_ind):
                                ans_result[astr] = dict(zip(split_ind, ct))
                            elif len(ct) == 1:  # first id in line
                                kk = -1
                                for k, v in enumerate(ans_result[astr:astr + 2 * len(split_ind)]):
                                    if (isinstance(v, str) and v == 'chinese') or \
                                        isinstance(v, dict) or isinstance(v,OrderedDict) or isinstance( v, list):
                                        kk = k
                                        break
                                split_content = ''.join(ans_result[astr:astr + kk])
                                ct = [strip_point(i) for i in re.split('|'.join(split_ind), split_content) if
                                      strip_point(i) != '']
                                if len(ct) == len(split_ind):
                                    ans_result[astr] = dict(zip(split_ind, ct))
                                    for i in range(1, kk):
                                        ans_result[astr + i] = '已提取'
                    elif f_id != -1 and b_id == -1:
                        # 提取出来的改错【10】live前加to 自成一行而没有提出来
                        if str(f_id + 1) in ans_result[astr]:
                            content = (ans_result[astr])[ans_result[astr].index(str(f_id + 1)):]
                            ct = [strip_point(i) for i in content.split(str(f_id + 1)) if strip_point(i) != '']
                            if len(ct) == 1:
                                ans_result[astr] = {str(f_id + 1): ct[0]}
                    elif f_id == -1 and b_id != -1:
                        if str(b_id - 1) in ans_result[astr]:
                            content = (ans_result[astr])[ans_result[astr].index(str(b_id - 1)):]
                            ct = [strip_point(i) for i in content.split(str(b_id - 1)) if strip_point(i) != '']
                            if len(ct) == 1:
                                ans_result[astr] = {str(b_id - 1): ct[0]}
            return [i for i in ans_result if i != '已提取']
        else:
            return ans_result
    except:
        print("【答案提取】:double_check_ans_structure出错")
        return ans_result


def repeated_id_partandtype(ans_result, ans_orig_list):
    """
    将重复id的答案分块,认为：id重复,一定有题目说明,不是在前一行,就是在本行开头,拿到答案说明,去匹配题型,按题型分答案
    有可能,前面一行也是答案,但由于某种原因没有提取出来而变成str,
    eg:1:for effort/ hard work,2: No.2,3: Three
       {'4': 'How Parents Should Praise Their Kids/ Why Praise Can Be Bad For Kids'}
    1.前面一行是str,不是chinese,2.在本行前面也没提取到汉字,3.本行key又不是从1开始 >>> 很大可能前一行也是答案,而没有提取出来,所以用1-该行最小key值范围的数字,去split前一行
    答案提取出来变成dict,怎么拿到原始数据,从而提取到该行首部的汉字:用dict中的value,每个key与原始数据每行定位,拿到index,然后最多的index,就是该dict的原始数据
    <<可能某个key,存在于多行中,所以不能只用一个value去定位
    """
    try:
        last_line_max_id = 0
        last_parse_id = 0
        for i, v in enumerate(ans_result):
            if isinstance(v, dict) :
                kl = list(v.keys())
                if len(v) == 1 and re.search(r'[^\d+]+',kl[0]):
                    pass
                else:
                    this_line_min_id = min(map(int, v.keys()))
                    if this_line_min_id > eval(str(last_line_max_id)):
                        last_line_max_id = max(map(int, v.keys()))
                    else:
                        # 解析先编码
                        vvalue = ' '.join(list(v.values()))
                        chinese_words = []
                        for w in get_chinese_char(vvalue):
                            chinese_words.extend(list(jieba.cut(w)))
                        if re.search(r'考查|故选|根据|因此|可知|所以|解析', vvalue) != None:  # 解析
                            if last_parse_id == 0:
                                ans_result[i] = {'parse_{}'.format(kl[0]): vvalue}
                                last_parse_id = int(kl[0])
                            elif int(kl[0]) > last_parse_id and int(
                                    kl[0]) < last_parse_id + 10:  # 和上一个parse id在一定范围内的,认为是正常提取的解析
                                ans_result[i] = {'parse_{}'.format(kl[0]): vvalue}
                                last_parse_id = int(kl[0])
                            else:
                                ans_result[i] = '已提取'  # 提取有问题的解析,不要了
                         # /(len(re.findall(r'[a-zA-Z]{1-15}',vvalue))+0.0001)> 1:
                        elif len([ii for ii in chinese_words if ii not in ['改为', '去掉', '删除', '删掉', '添加','加上']]) > 5:
                            if last_parse_id == 0:
                                ans_result[i] = {'parse_{}'.format(kl[0]): vvalue}
                                last_parse_id = int(kl[0])
                            elif int(kl[0]) > last_parse_id and int(kl[0]) < last_parse_id + 10:
                                ans_result[i] = {'parse_{}'.format(kl[0]): vvalue}
                                last_parse_id = int(kl[0])
                            else:
                                ans_result[i] = '已提取'  # 提取有问题的解析,不要了
                        # 不是解析,编码又重复了,肯定是一块新题,要分块,且拿到题目说明
                        else:
                            next = -1
                            type_content = ''
                            fc = ans_result[i - 1]
                            if fc == 'chinese':
                                type_content = chinese_content[appear_times(ans_result, 'chinese')[i - 1] - 1]
                            else:
                                may_ids = []
                                for iv in list(v.values()):
                                    sm = [Levenshtein.ratio(strip_point(origc), iv) for origc in ans_orig_list]
                                    may_ids.append(sm.index(max(sm)))  # where this line ans is in orig ans list
                                may_id = Counter(may_ids).most_common()[0][0]
                                chinesew = get_chinese_char(ans_orig_list[may_id])  # 提取句首的汉字
                                if len(chinesew) > 0:
                                    type_content = chinesew[0]
                                elif kl[0] != '1' and isinstance(fc, str):
                                    split_num = [n for n in re.findall('\d+', fc) if int(n) in range(1, int(kl[0]))]
                                    rr = re.split('(' + '|'.join(split_num) + r')(\.|、|．)', fc)
                                    num = [strip_point(nm) for nm in rr if re.match(r'\d+$', strip_point(nm)) != None]
                                    aa = [strip_point(a) for a in rr if re.search(r'[a-zA-Z]', strip_point(a)) != None]
                                    if len(num) == len(aa):
                                        v.update(dict(zip(num, aa)))
                                        ans_result[i - 1] = '已提取'
                                        if i > 2 and ans_result[i - 2] == 'chinese':
                                            type_content = chinese_content[appear_times(ans_result, 'chinese')[i - 2] - 1]
                            for ii, vv in enumerate(ans_result[i + 1:]):
                                if isinstance(vv, dict) and list(vv.keys())[0] not in ['短文', '短文改错', '听力原文']:
                                    if min(map(int, list(vv.keys()))) < last_line_max_id:
                                        v.update(vv)
                                        ans_result[i + 1 + ii] = '已提取'
                                    else:
                                        break
                                else:
                                    break
                            tylable = item_type_classify([([type_content], [str(v)])])[0].replace("写作","短文")
                            if "单" not in tylable :
                                ans_result[i] = {tylable: v}
            else:
                pass
                # nkeys = [str(last_line_max_id) + '_'+ v for v in list(v.keys())]
                # last_line_max_id = nkeys[-1]
                # ans_result[i] = dict(zip(nkeys,v.values()))
        return [i for i in ans_result if i != '已提取']
    except:
        print("【答案提取】:repeated_id_partandtype出错")
        return ans_result


def is_parse(text:str):
    """
    判断该行内容是否为解析
    :param text:
    :return:
    """
    if chinese_start_line(text) == False:
        if len(jieba.lcut(text)) > 5:
            if re.search(r'(?<!(篇章|语篇))解析|(?<![答考试真小大\d本该]){2,6}题',text[:20]) :
                return True
            elif re.search(r'答案为|考查|故选|根据|因此|可[知得]|可以?[猜推]出|所以|[A-G](选?项)?应?该?[是为](对|正确)|故[A-D]选?项',text) :  # 如果没有题号,那就要求在开头要有"解析"两个字
                return True
    return False


def parse_extract(ans_result):
    if "str" in list_ele_type(ans_result):
        try:
            idd = ""
            parse_index = []
            for k, ss in enumerate(ans_result):
                # 如果没有题号,那就要求在开头要有"解析"两个字
                #有解析的,可能是语篇解析(用里面是否有ABCD来判断),语篇解析暂时不要,语篇解析容易提取成一个小题解析
                if k > 0 and isinstance(ss, str) and is_parse(ss):
                    id_find = re.search(r'(\d+)', ss[:4])
                    if id_find == None:
                        before_one_line = ans_result[k - 1]
                        if isinstance(before_one_line, dict) and len(before_one_line) > 0:  # 答案已提取了
                            may_id = re.search(r"(\d+)",list(before_one_line.keys())[-1])
                            if may_id:
                                idd = may_id.group(1)  # 字典里最后一个题号,作为这个解析的题号
                        elif isinstance(before_one_line, str):
                            may_id = re.search(r'(\d+)', before_one_line[:4])
                            if may_id:
                                idd = may_id.group(1)
                    elif id_find != None:  # 拿到id了
                        idd = id_find.group(1) if eval(id_find.group(1)) < 120 else "0"  #把大于120的id置为0
                    ###########题号获取结束
                    if idd != "":
                        ans_result[k] = {'parse_{}'.format(idd): ss}
                        parse_index.append(k)  #不全,还有之前提取的解析索引没有在里面
            for k, v in enumerate(ans_result):
                if k not in parse_index and isinstance(v, dict) and "parse" in str(v.keys()):
                    parse_index.append(k)
            parse_index.sort()
            len_ppc = len(parse_index.copy())
            count = 0
            while len(parse_index) > 0 and count < 2*len_ppc :  #因为还在不停添加parse_index,所以循环多设置几次,以免没合并完成
                pi = parse_index[0]
                    ####两个parse之间一行还有未提取的东西,就把这一行归到上一个parse里去
                if pi < len(ans_result)-1:
                    next_ans = ans_result[pi+1]
                    this_id = list(ans_result[pi].keys())[0]
                    if isinstance(next_ans,str) and next_ans!="chinese" :
                        ans_result[pi+1] = {this_id:list(ans_result[pi].values())[0]+"\n"+next_ans}
                        if pi+1 not in parse_index:
                            parse_index.insert(1,pi+1) #pi行下面可能有多行str或多行parse，如果不添加,这个parse只会处理紧挨着的一行
                        ans_result[pi] = "chinese"
                    elif isinstance(next_ans,dict):
                        next_id = list(next_ans.keys())[0]
                        if re.search(r"\d",next_id) and eval("{}-{}".format(next_id.replace("parse_",""),this_id.replace("parse_",""))) <= 0:  #都是一个题的解析,将答案合并
                            ans_result[pi+1] = {this_id:list(ans_result[pi].values())[0]+"\n"+list(ans_result[pi+1].values())[0]}
                            if pi + 1 not in parse_index:
                                parse_index.insert(1,pi+1)
                            ans_result[pi] = "chinese"
                del parse_index[0]
                count += 1
            # return ans_result
        except:
            print("【答案提取】:parse_extract出错")
            pass


def parse_0_correct(ans_result):
    """
    在get_word_ans中,对于上一行是答案，下一行是解析,且解析前没有题号的情况，会提取成parse_0
    1.如果上一行已经提取为dict了,就把上一行的题号给这一行
    2.如果上一行还是str,就找这一行靠前的数字作为这一题的题号
    :param ans_result:
    :return:
    """

    if "parse_0" in str(ans_result):
        parse_0_index = [k for k,v in enumerate(ans_result) if k > 0 and isinstance(v,dict) and "parse_0" in v]

        while len(parse_0_index) > 0 :
            parse = ans_result[parse_0_index[0]]["parse_0"]
            last_line = ans_result[parse_0_index[0]-1]
            if isinstance(last_line,dict):
                may_id = list(last_line.keys())[-1]
            elif isinstance(last_line,str):
                may_id = last_line[:6]
            new_id = re.search(r"(\d+)",may_id).group(1) if re.search(r"(\d+)",may_id) else "0"
            if new_id != "0":
                ans_result[parse_0_index[0]] = {"parse_{}".format(new_id):parse}
            del parse_0_index[0]


if __name__ == '__main__':

    # b = get_obj_ans("school is over at 5 o'clock. After school, we often play basketball or do some other sport on the")
    # print(b)
    # b2 = get_word_ans("21.B推断题。通读全文得出,第2项赛事的起跑点和终点之间落差最大，因此该赛事的下坡跑最著名。由该")
    # print(b2)
# #    a = ['21.D\n', '21.根据第2段可知，文章推荐了一些免费观看喜剧电影的网站。\n', '22.C\n', '22.根据Comedy Movies at Crackle部分中的You can browse by year or title可知，你可以通过搜索电影标题来找到自己最喜欢的电影。\n', '23.\n', "23.C根据Hulu's Free Comedy Movies部分中的“Hulu has more diversities of free comedy movies than any other place.”可知答案。\n", '24.C\n', '24.根据文章第1段中He is an unconventional man 和第2段中Everything about Eliza Doolittle seems to challenge any conventional concept可知，他们都反对传统观念。\n', '25.C\n', '25.根据第1段第2句“..and uses all manners of recording and...understandable units.”可知答案。\n', '26.D\n', '26.根据文章第2段末句可知，独立自主的个性使Eliza Doolittle获得了尊重。\n', '27.A\n', '27.文章分析的是戏剧《卖花女》中的人物性格特征，所以我们很可能在文学杂志中读到这样的文章。\n', '28.B\n', '28.根据文章第1段第2、3句可知，科学家发现改变植物基因会使植物的光合作用速度加快，将来会增加全球植物的产量。\n', '29.B\n', '29.根据文章第4段第2、3、4句可知，作者提到镜片是为了说明植物的保护系统的工作原理与镜片相同。\n', '30.D\n', '30.根据文章最后一句可知答案。\n', '31.B\n', '31.根据文章内容可知，科学家改变植物的基因，加快光合作用的速度，使植物叶子生长迅速，从而提高植物产量。\n', '32.B\n', '32.根据文中列举的提高记忆力的3种方法可知，有些充分利用思维的秘诀，一旦我们了解了，就简单易行。\n', '33.D\n', '33.根据第1段最后一句可知，本文主要介绍提高记忆力的几种方法。\n', '34.C\n', '34.根据文章最后一段第1句可知，图像联系记忆是那些记忆力比赛获胜者常用的方法。\n', '35.D\n', '35.根据文章的主题一—提高记忆力的方法可知，这篇文章与思维有关。\n', '36～40 FADCE\n', '41.B\n', '41.根据上文可知，父亲正急切地等着作者的回答。\n', '42.C\n', '42.作者仔细地看了看花园，然后回答“非常好！”。\n', '43.A\n', '43.然后，作者列举了她发现的园子里的所有变化，父亲满意地笑了。\n', '44.D\n', '44.根据下文可知，作者的母亲在一次车祸中去世了，留下父亲抚养三个年幼的女儿。\n', '45.C\n', '46.A\n', '46.根据下文可知，一开始，生活并不顺利。\n', '47.D\n', '47.根据下文可知，父亲在鼓励我们。\n', '48.B\n', '48.同时，父亲尽自己最大的努力去证明那个信念。\n', '49.\n', '49.A1972年，父亲在Okaloosa Island开垦了一片被废弃的土地。\n', '50.A\n', '50.根据下文可知，这里指在每年的初春。\n', '51.B\n', '51.经过父亲不断的辛勤劳作，这片土地逐渐变成了美丽的花园。\n', '52.C\n', '52.根据下文可知，作者有时候和父亲一起在他的花园里愉快地劳动。\n', '53.B\n', '53.根据上文可知，作者会邀请她的朋友们参观花园。\n', '54.D\n', '54.根据下文可知，父亲把他的花园打理得非常不错。\n', '55.A\n', '55.根据下文可知，多年以来，每当作者心烦的时候，就会去父亲的花园。\n', '56.D\n', '56.因为它会使作者想起父亲的信念。\n', '57.B\n', '57.根据下文可知，正是父亲和他的花园给了作者力量。\n', '58.C\n', '58.这力量让作者能够继续生活下去并且战胜生活中的挑战。\n', '59.A\n', '59.根据下文可知，父亲说他不再继续打理这个花园。\n', '60.D\n', '60.根据下文可知，我们沉默地坐着。\n', '语法填空\n', '61.earlier 62.what 63.probably 64.ways 65.a 66.in 67.grown 68.was brought 69.that 70.was\n', '短文改错\n', 'Being responsible is actually not that difficult.I used to thinking that it was hard to grow up into a respon-sible member of the society.An incident which happened in a rainy Sunday afternoon changed my attitude.I was on my way to a bookstore and was waiting for the green light when a girl knocked down by a passing car，that drove off quickly.A man immediately gave her first aid and I had joined in without hesitation.Soon many help was given to the girl.Because we sent her to the nearest hospitals in time，she was able to receive properly treatment.Not badly injured，I expressed gratitude to those giving help.Comparing with the escaped driver，I was proud of what I had done.\n', '71. thinking—think in—on 在girl 和knocked 中间加was that-- which had去掉 many--much hospitals-hospital properly--proper I--she Comparing-- Compared\n', '书面表达\n', "One possible version：Dear John，I'm glad you are concerned about my school life.You asked me what amateur activities we had recently.Do you like stage drama？It's fun to watch and do.On Nov.22，students in Grade 2 from our school acted out their own stage drama Red Crag.It's from the famous novel with the same name.Students acted and directed the drama by themselves.They are all members of the drama club.Through the drama，students said they had a better understanding of the novel.If you have time you can surf the Internet for this story and exchange your idea with me.\n", 'Looking forward to your reply.\n', 'Yours，LiHua\n']
    ans_list = []
    for root, dirs, files in os.walk (r'E:\online_old\ans_txt'):
       files_list = [files]
       for file in files:
           print("成功:{}".format(file))
           with open(os.path.join(root,file)) as f:
               a = f.readlines()
               a2 = ans_structure(a)
               ans_list.append( (file,a2) )

    import json
    with open (r'./structure_ans.json','w') as ff:
        ff.write(json.dumps({"ans":ans_list}))

 # with open(r"../ans_txt/[tiku.gaokao.com]吉林省九校联合体届高三第二次摸底考试英语试题.txt",'r') as f:
 #     a = f.readlines()
 #     a2 = ans_structure(a)
 #     b = repeated_id_partandtype(a2, a)
 #     # pprint(b)
 #     parse_extract(b)
 #     pprint(b)
 #    print(chinese_start_line("2.C解析]推理判断题。根据第四个项目的第三句话，可知EF Ciep Yer可以让学生自主地设定学习时间"))

######山东省桓台第二中学2017-2018学年高一下学期4月月考英语试题Word版含答案.txt
    # with open(r"E:\online_old\ans_txt\四川省资阳市2018届高三第二次诊断性考试试题英语Word版含答案.txt","r") as f:
    #     ans_list = f.readlines()
    #     res = ans_structure(ans_list)
    #     pprint(res)

    # print(is_parse("第三节（共5小题；每小题1分，共5分"))