1.客户端上传图像不一致问题服务端对应修改(choice-ocr不用先写再读啦)

2.solve 推断补全；
3.某个choice_m题号识别错误不会在影响其他；
4.base64写法修改。

segment/server.py

@@ -45,16 +45,20 @@ def convert_pil_to_jpeg(raw_img):
     return img, open_cv_image
 
 
-def opencv2base64(image, to_pil=True):
+def opencv2base64(image, to_pil=False):
     # image = cv2.imencode('.jpg', img)[1]
     # base64_data = str(base64.b64encode(image))[2:-1]
 
     if to_pil:
         image = Image.fromarray(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
-    output_buffer = BytesIO()
-    image.save(output_buffer, format='JPEG')
-    byte_data = output_buffer.getvalue()
-    base64_data = base64.b64encode(byte_data)
+        output_buffer = BytesIO()
+        image.save(output_buffer, format='JPEG')
+        byte_data = output_buffer.getvalue()
+        base64_data = base64.b64encode(byte_data)
+
+    else:
+        data = cv2.imencode('.jpg', image)[1]
+        base64_data = base64.b64encode(data.tostring()).decode('utf8')
 
     return base64_data
 

segment/sheet_resolve/analysis/choice/choice_line_box.py

@@ -343,15 +343,19 @@ def choice_m_row_col0(left, top, image, choice_bbox_list, xml_path, img, choice_
 
 def choice_bbox_vague(choice_m_list0, x_y_interval_ave, singe_box_width_height_ave, direction, image_size):
     img_width = image_size[0]
+    img_height = image_size[1]
     choice_list_temp1 = []
     choice_list_temp2 = []
     index_list1 = []
     index_list2 = []
     for index, ele in enumerate(choice_m_list0):
-        if ele[0] < img_width // 2:
+        if int(img_width) < int(img_height):
             choice_list_temp1.append(ele)
             index_list1.append(index)
-        else:
+        elif int(img_width) >= int(img_height) and ele[0] < img_width // 2:
+            choice_list_temp1.append(ele)
+            index_list1.append(index)
+        elif int(img_width) >= int(img_height) and ele[0] >= img_width // 2:
             choice_list_temp2.append(ele)
             index_list2.append(index)
     if index_list2 == []:
@@ -370,7 +374,7 @@ def choice_bbox_vague(choice_m_list0, x_y_interval_ave, singe_box_width_height_a
 
         if direction == 180:     # vertical
             x_diff = x_y_interval_ave[0]
-            s_width = singe_box_width_height_ave[1]
+            s_width = singe_box_width_height_ave[0]
             choice_bbox = (np.hstack((np.array([min(xmin0) - x_diff - 3 * s_width, min(ymin0)]), np.array([max(xmax0), max(ymax0)])))).tolist()
             choice_bbox_with_index_list = (choice_bbox, choice_m_list1[1])
             choice_bbox_all.append(choice_bbox_with_index_list)

segment/sheet_resolve/analysis/choice/get_title_number_by_choice_m.py

@@ -474,33 +474,45 @@ def get_title_number(choice_bbox, choice_region, choice_m_box_dict, direction):
             choice_m_s.update({'title_number': title_number_list})
             all_list_new.append(choice_m_s)
         else:
-            s_box = sorted(s_box, key=lambda k: k[1])
-            s_box_row = get_one_line_box(s_box, single_bbox_width_height[1])
-            row_box_ = []
-
-            index_list = []  # 存在的index
-            for index0, ele0 in enumerate(s_box_row):
-                for index1, ele1 in enumerate(title_number):
-                    title_number_bbox = [ele1['location']['left'], ele1['location']['top'],
-                                         ele1['location']['left'] + ele1['location']['width'],
-                                         ele1['location']['top'] + ele1['location']['height']]
-                    row_box = {}
-                    if title_number_bbox[1] - single_bbox_width_height[1] < ele0[0][1] < title_number_bbox[1] + \
-                            single_bbox_width_height[1] \
-                            and title_number_bbox[3] - single_bbox_width_height[1] < ele0[0][3] < \
-                            title_number_bbox[3] + single_bbox_width_height[1]:
-                        row_box['title_number'] = ele1
-                        row_box['row_box'] = ele0
-                        row_box_.append(row_box)
-                        index_list.append(index0)
-            index0 = list(set([i for i in range(0, row_and_col[0])]) - set(index_list))  # lack index
-            number0 = choice_m_s['number']
-            for index, exist_index in enumerate(index_list):
-                number_char = choice_m_s['title_number'][index]['char']
-                number0[exist_index] = int(number_char)
-            new_number_list = utils.infer_number(number0)
-            choice_m_s.update({'title_number': new_number_list})
-            all_list_new.append(choice_m_s)
+            try:
+                s_box = sorted(s_box, key=lambda k: k[1])
+                s_box_row = get_one_line_box(s_box, single_bbox_width_height[1])
+                row_box_ = []
+
+                index_list = []  # 存在的index
+                for index0, ele0 in enumerate(s_box_row):
+                    for index1, ele1 in enumerate(title_number):
+                        title_number_bbox = [ele1['location']['left'], ele1['location']['top'],
+                                             ele1['location']['left'] + ele1['location']['width'],
+                                             ele1['location']['top'] + ele1['location']['height']]
+                        row_box = {}
+                        if title_number_bbox[1] - single_bbox_width_height[1] < ele0[0][1] < title_number_bbox[1] + \
+                                single_bbox_width_height[1] \
+                                and title_number_bbox[3] - single_bbox_width_height[1] < ele0[0][3] < \
+                                title_number_bbox[3] + single_bbox_width_height[1]:
+                            row_box['title_number'] = ele1
+                            row_box['row_box'] = ele0
+                            row_box_.append(row_box)
+                            index_list.append(index0)
+                        elif title_number_bbox[0] - single_bbox_width_height[0] < ele0[0][0] < title_number_bbox[0] + \
+                                single_bbox_width_height[0] \
+                                and title_number_bbox[2] - single_bbox_width_height[0] < ele0[0][2] < \
+                                title_number_bbox[2] + single_bbox_width_height[0]:
+                            row_box['title_number'] = ele1
+                            row_box['row_box'] = ele0
+                            row_box_.append(row_box)
+                            index_list.append(index0)
+                index_list = sorted(list(set(index_list)))
+                index0 = list(set([i for i in range(0, row_and_col[0])]) - set(index_list))  # lack index
+                number0 = choice_m_s['number']
+                for index, exist_index in enumerate(index_list):
+                    number_char = choice_m_s['title_number'][index]['char']
+                    number0[exist_index] = int(number_char)
+                new_number_list = utils.infer_number(number0)
+                choice_m_s.update({'title_number': new_number_list})
+                all_list_new.append(choice_m_s)
+            except Exception as e:
+                all_list_new.append(choice_m_s)
     # print(all_list_new)
     title_number_by_choice_m_list = []     # sort change coordinate
     for index, single_choice_m in enumerate(all_list_new):

segment/sheet_resolve/analysis/resolve.py

@@ -22,9 +22,10 @@ from segment.sheet_resolve.tools.tf_sess import TfSess
 from segment.sheet_resolve.tools.tf_settings import xml_template_path, model_dict
 from segment.sheet_resolve.tools.utils import read_single_img, read_xml_to_json, create_xml
 from segment.sheet_resolve.analysis.sheet.sheet_adjust import adjust_item_edge_by_gray_image
-from segment.sheet_resolve.analysis.sheet.sheet_infer import infer_bar_code, box_infer_and_complete
+from segment.sheet_resolve.analysis.sheet.sheet_infer import infer_bar_code, box_infer_and_complete, infer_solve
 from segment.sheet_resolve.analysis.sheet.sheet_infer import infer_exam_number, adjust_exam_number, exam_number_infer_by_s
 from segment.sheet_resolve.analysis.sheet.choice_infer import infer_choice_m
+from segment.sheet_resolve.analysis.sheet.ocr_sheet import tell_columns, sheet_sorted
 
 logger = logging.getLogger(settings.LOGGING_TYPE)
 
@@ -32,7 +33,8 @@ logger = logging.getLogger(settings.LOGGING_TYPE)
 sheet_infer_dict = dict(bar_code=True,
                         choice_m=True,
                         exam_number=True,
-                        common_sheet=False)
+                        common_sheet=False,
+                        solve=True)
 infer_choice_m_flag = False
 
 
@@ -59,6 +61,9 @@ def sheet(series_number, image_path, image, conf_thresh, mns_thresh, subject, sh
         traceback.print_exc()
         logger.info('试卷：{} 自适应边框失败: {}'.format(image_path, e))
 
+    # 分栏
+    col_split_x = tell_columns(image, regions)
+
     if sheet_infer_dict['bar_code']:
         try:
             if ('bar_code' not in fetched_class) and ocr:
@@ -95,7 +100,7 @@ def sheet(series_number, image_path, image, conf_thresh, mns_thresh, subject, sh
     if sheet_infer_dict['choice_m']:
 
         try:
-            choice_m_list = infer_choice_m(image, regions, ocr)
+            choice_m_list = infer_choice_m(image, regions, col_split_x, ocr)
             #remain_choice_m = []
             if len(choice_m_list) > 0:
                 choice_m_old_list = [ele for ele in regions if 'choice_m' == ele['class_name']]
@@ -128,6 +133,42 @@ def sheet(series_number, image_path, image, conf_thresh, mns_thresh, subject, sh
             traceback.print_exc()
             logger.info('试卷：{} 选择题推断失败: {}'.format(image_path, e))
 
+    if sheet_infer_dict['solve']:
+        try:
+            include_class = ['info_title',
+                             'bar_code',
+                             'choice_m',
+                             'cloze',
+                             'cloze_s',
+                             'exam_number',
+                             'solve',
+                             'composition',
+                             'correction'
+                             ]
+            regions_subset = [ele for ele in regions if ele['class_name'] in include_class]
+            col_regions = sheet_sorted(regions_subset, col_split_x)
+
+            top = min([ele['bounding_box']['ymin'] for ele in regions])
+            bottom = max([ele['bounding_box']['ymax'] for ele in regions])
+
+            seal_area = [ele for ele in regions if 'seal' in ele['class_name']]
+            if len(seal_area) > 0:
+                right, left = w, 1
+                for ele in seal_area:
+                    if ele['bounding_box']['xmax'] > w // 2:
+                        right = ele['bounding_box']['xmin']
+                    if ele['bounding_box']['xmax'] < w // 2:
+                        left = ele['bounding_box']['xmax']
+            else:
+                left = min([ele['bounding_box']['xmin'] for ele in regions])
+                right = max([ele['bounding_box']['xmax'] for ele in regions])
+
+            solve_regions = infer_solve(regions, left, right, top, bottom, col_regions, col_split_x)
+            regions.append(solve_regions)
+        except Exception as e:
+            traceback.print_exc()
+            logger.info('试卷：{} 解答题补全推断失败: {}'.format(image_path, e))
+
     if sheet_infer_dict['common_sheet']:
 
         try:

segment/sheet_resolve/analysis/sheet/choice_infer.py

@@ -120,8 +120,12 @@ def find_digital(ocr_raw_list):
                          if index not in digital_index_detail_list and char['char'] not in ['.', ',', '。', '、']]
 
         chars_list += current_chars
-    d_mean_height = sum(height_list) // len(height_list)
-    d_mean_width = sum(width_list) // len(width_list)
+
+    if not height_list or not width_list:
+        d_mean_height, d_mean_width = 0, 0
+    else:
+        d_mean_height = sum(height_list) // len(height_list)
+        d_mean_width = sum(width_list) // len(width_list)
 
     # mean_height = max(height_list)
     # mean_width = max(width_list)
@@ -669,8 +673,8 @@ def cluster_and_anti_abnormal(image, xml_path, digital_list, chars_list,
     return choice_m_list
 
 
-def infer_choice_m(image, tf_sheet, ocr, xml=None):
-    infer_box_list = ocr2sheet(image, tf_sheet, ocr, xml)
+def infer_choice_m(image, tf_sheet, col_split_x, ocr, xml=None):
+    infer_box_list = ocr2sheet(image, col_split_x, ocr, xml)
     # print(sheet_region_list)
     choice_m_list = []
 
@@ -706,22 +710,26 @@ def infer_choice_m(image, tf_sheet, ocr, xml=None):
 
         if choice_flag:
             infer_image = utils.crop_region_direct(image, loc)
-            try:
-                save_dir = os.path.join(settings.MEDIA_ROOT, 'tmp')
-                if not os.path.exists(save_dir):
-                    os.makedirs(save_dir)
-                save_path = os.path.join(save_dir, 'choice.jpeg')
-                cv2.imwrite(save_path, infer_image)
-                img_tmp = utils.read_single_img(save_path)
-                os.remove(save_path)
-                ocr = brain_api.get_ocr_text_and_coordinate(img_tmp, 'accurate', 'CHN_ENG')
-            except Exception as e:
-                print('write choice and ocr failed')
-                traceback.print_exc()
-                ocr = brain_api.get_ocr_text_and_coordinate(infer_image, 'accurate', 'CHN_ENG')
+            ocr = brain_api.get_ocr_text_and_coordinate(infer_image, 'accurate', 'CHN_ENG')
+            # try:
+            #     save_dir = os.path.join(settings.MEDIA_ROOT, 'tmp')
+            #     if not os.path.exists(save_dir):
+            #         os.makedirs(save_dir)
+            #     save_path = os.path.join(save_dir, 'choice.jpeg')
+            #     cv2.imwrite(save_path, infer_image)
+            #     img_tmp = utils.read_single_img(save_path)
+            #     os.remove(save_path)
+            #     ocr = brain_api.get_ocr_text_and_coordinate(img_tmp, 'accurate', 'CHN_ENG')
+            # except Exception as e:
+            #     print('write choice and ocr failed')
+            #     traceback.print_exc()
+            #     ocr = brain_api.get_ocr_text_and_coordinate(infer_image, 'accurate', 'CHN_ENG')
 
             try:
                 digital_list, chars_list, digital_mean_h, digital_mean_w = find_digital(ocr)
+                if not digital_list:
+                    continue
+
                 choice_m = cluster_and_anti_abnormal(image, xml, digital_list, chars_list,
                                                      digital_mean_h, digital_mean_w,
                                                      choice_s_height, choice_s_width, loc)
@@ -735,17 +743,18 @@ def infer_choice_m(image, tf_sheet, ocr, xml=None):
     # print(choice_m_list)
     # tf_choice_sheet = [ele for ele in tf_sheet if ele['class_name'] == 'choice_m']
 
-    sheet_tmp = choice_m_list.copy()
-    remove_index = []
-    for i, region in enumerate(sheet_tmp):
-        if i not in remove_index:
-            box = region['bounding_box']
-            for j, region_in in enumerate(sheet_tmp):
-                box_in = region_in['bounding_box']
-                iou = utils.cal_iou(box, box_in)
-                if iou[0] > 0.85 and i != j:
-                    choice_m_list.remove(region)
-                    remove_index.append(j)
-                    break
+    if choice_m_list:
+        sheet_tmp = choice_m_list.copy()
+        remove_index = []
+        for i, region in enumerate(sheet_tmp):
+            if i not in remove_index:
+                box = region['bounding_box']
+                for j, region_in in enumerate(sheet_tmp):
+                    box_in = region_in['bounding_box']
+                    iou = utils.cal_iou(box, box_in)
+                    if iou[0] > 0.85 and i != j:
+                        choice_m_list.remove(region)
+                        remove_index.append(j)
+                        break
 
     return choice_m_list

segment/sheet_resolve/analysis/sheet/ocr_sheet.py

@@ -102,9 +102,9 @@ def tell_columns(image, sheet_dict):
     return split_x
 
 
-def ocr2sheet(image, sheet_dict, raw_ocr, xml_path=None):
+def ocr2sheet(image, col_split_list, raw_ocr, xml_path=None):
     image_y, image_x = image.shape[0], image.shape[1]
-    col_split_list = tell_columns(image, sheet_dict)
+    # col_split_list = tell_columns(image, sheet_dict)
     digital_p = r'\d'
     eng_char_p = '[\u0041-\u005a|\u0061-\u007a]'  # english
     chn_char_p = '[\u4e00-\u9fa5]'  # chinese
@@ -225,3 +225,63 @@ def ocr2sheet(image, sheet_dict, raw_ocr, xml_path=None):
     # tree.write(xml_path)
 
     return block_list
+
+
+def sheet_sorted(regions, split_x):
+    """
+    自底向上复杂度太高。
+    自顶向下排序，答题卡依次分块，每块区域内排序：第一次分栏，第二次栏内分块
+    """
+
+    region_contain_set = [('choice_m', 'solve'), ('choice_m', 'cloze'),]
+    region_contain_set_ = [(ele[1], ele[0]) for ele in region_contain_set]
+    region_contain_set.extend(region_contain_set_)
+
+    # split_x = tell_columns(image, regions)
+    regions = sorted(regions, key=lambda x: x['bounding_box']['xmin'])
+    x_min_list = [ele['bounding_box']['xmin'] for ele in regions]
+
+    # 分栏
+    col_list = []
+    for split in split_x:
+        x_min_list.append(split)
+        x_min_list = sorted(x_min_list)
+        split_index = x_min_list.index(split)
+        col_list.append(regions[:split_index])
+        regions = regions[split_index:]
+        x_min_list = x_min_list[split_index + 1:]
+
+    col_list.append(regions)
+
+    sorted_regions = []  # 栏中排序
+    for col_regions in col_list:
+        col_regions = sorted(col_regions, key=lambda x: x['bounding_box']['ymin'])
+        col_regions_num = len(col_regions)
+
+        sorted_cols = []
+        while len(col_regions) > 0:
+            row_regions = []
+            region = col_regions[0]  # 根据box确定行
+            name = region['class_name']
+            ymin = region['bounding_box']['ymin']
+            ymax = region['bounding_box']['ymax']
+            row_regions.append(region)
+            for j, region_ in enumerate(col_regions):
+                name_ = region_['class_name']
+                y_min_ = region_['bounding_box']['ymin']
+                if j > 0:
+                    # 栏内分块
+                    if ymin <= y_min_ < ymax and (name, name_) not in region_contain_set:
+                        row_regions.append(region_)
+                    if ymax <= y_min_:
+                        break
+
+            # 块内排序
+            row_regions = sorted(row_regions, key=lambda x: x['bounding_box']['xmin'])
+            col_regions = [ele for ele in col_regions if ele not in row_regions]
+
+            sorted_cols.extend(row_regions)
+
+        sorted_regions.append(sorted_cols)
+
+    return sorted_regions

segment/sheet_resolve/analysis/sheet/sheet_infer.py

@@ -1071,124 +1071,111 @@ def box_infer_and_complete(image, sheet_region_dict, ocr=''):
     return sheet_region_dict
 
 
-# 选择题区域补全
-def _get_split_index(sorted_list, spilt_value):
-    y_dif_list = np.array(sorted_list[1:]) - np.array(sorted_list[:-1])
-    y_split_index = [index for index, ele in enumerate(y_dif_list) if ele >= spilt_value]
-
-    y_split_index = [ele + 1 for ele in y_split_index]  # 索引值扩大
-    y_split_index.insert(0, 0)
-    y_split_index.insert(-1, len(sorted_list))
-    y_split_index = sorted(list(set(y_split_index)))
-
-    return y_split_index
-
-
-def get_letter_group(letter, location_list):
-    y_list = sorted([ele['location']['top'] for ele in location_list])
-    height = np.mean(np.array([ele['location']['height'] for ele in location_list]))
-    width = np.mean(np.array([ele['location']['width'] for ele in location_list]))
-    y_split_dif, x_split_dif = height * 1.5, width * 1.5
-
-    y_split_index = _get_split_index(y_list, y_split_dif)
-
-    letter_group_list = []
-    letter_group_location_list = []
-    for i, split in enumerate(y_split_index[1:]):
-        one_group_location_list = location_list[y_split_index[i]:y_split_index[i + 1]]
-        one_group_x_list = sorted([ele['location']['top'] for ele in one_group_location_list])
-        one_group_x_split_index = _get_split_index(one_group_x_list, x_split_dif)
-
-        block = []
-        block_location = []
-        for i_i, s_split in enumerate(one_group_x_split_index[1:]):
-            letter_group = one_group_location_list[one_group_x_split_index[i_i]:
-                                                   one_group_x_split_index[i_i + 1]]
-            letter_group = sorted(letter_group, key=lambda k: k.get('location')['top'])
-
-            xmin = min([ele['location']['left'] for ele in letter_group])
-            ymin = min([ele['location']['top'] for ele in letter_group])
-            xmax = max([ele['location']['left'] for ele in letter_group]) + width
-            ymax = max([ele['location']['top'] for ele in letter_group]) + height
-            middle_x, middle_y = (xmax - xmin) / 2 + xmin, (ymax - ymin) / 2 + ymin
-            block_location.append((xmin, ymin, xmax, ymax, middle_x, middle_y))
-            block.append(letter_group)
-
-        letter_group_list.append(block)
-        letter_group_location_list.append(block_location)
-
-    res_dict = {'letter': letter,
-                'letter_group': letter_group_list,
-                'letter_group_location': letter_group_location_list,
-                'width': width, 'height': height}
-
-    return res_dict
-
-
-def get_letter_group_h(letter, location_list):
-    location_list = sorted(location_list, key=lambda k: k.get('location')['left'])
-    x_list = sorted([ele['location']['left'] for ele in location_list])
-    height = np.mean(np.array([ele['location']['height'] for ele in location_list]), dtype=np.uint)
-    width = np.mean(np.array([ele['location']['width'] for ele in location_list]), dtype=np.uint)
-    print('h, w: ', height, width)
-    y_split_dif, x_split_dif = height * 1.5, width * 1.5
-
-    x_split_index = _get_split_index(x_list, x_split_dif)
-
-    letter_group_location_list = []
-    for i, split in enumerate(x_split_index[1:]):
-        one_group_location_list = location_list[x_split_index[i]:x_split_index[i + 1]]
-        one_group_location_list = sorted(one_group_location_list, key=lambda k: k.get('location')['top'])
-        xmin = min([ele['location']['left'] for ele in one_group_location_list])
-        ymin = one_group_location_list[0]['location']['top']
-        xmax = xmin + width
-        ymax = one_group_location_list[-1]['location']['top'] + 2*one_group_location_list[-1]['location']['height']
-        letter_group_location_list.append((xmin - 2*width, ymin,
-                                           xmax + 2*width, ymax))
-
-    return {'letter': letter, 'group_location': letter_group_location_list}
-
-
-def infer_choice_m_by_ocr(ocr_dict_list):
-    # 若字母识别漏掉结果太多, 此方法不能使用
-    a_e = 'ABCDEF'
-    pattern = '[ABCDEF]'
-    a_e_dict = {k: [] for k in a_e}
-    block_num = 1  # default
-    for i, ele in enumerate(ocr_dict_list):
-        words = ele['words']
-        cal_num = max([words.upper().count(char) for char in a_e])
-        if cal_num > 0:
-            words = words.replace(' ', '').upper()  # 去除空格，baidu_api bug
-            abcd_words_m = re.finditer(pattern, words)
-            abcd_index_list = [(m.group(), m.span()) for m in abcd_words_m if m]
-            for letter_info in abcd_index_list:
-                letter = letter_info[0]
-                a_e_dict[letter].append(ele['chars'][letter_info[1][0]])
-
-    letter_group_list = []
-    for k, v in a_e_dict.items():
-        if v:
-            letter_group = get_letter_group_h(k, v)
-            block_num = max(block_num, len(letter_group['group_location']))
-            print(letter_group)
-            letter_group_list.append(letter_group)
-
-    choice_m_list = []
-    for i in range(0, block_num):
-        block = []
-        for letter_group in letter_group_list:
-            if len(letter_group['group_location']) > i:
-                block.append(letter_group['group_location'][i])
-
-        if block:
-            block_array = np.asarray(block)
-            b_min = np.min(block_array, axis=0)
-            b_max = np.max(block_array, axis=0)
-            choice_m_dict = {'class_name': 'choice_m',
-                             'location': {'xmin': b_min[0], 'ymin': b_min[1],
-                                          'xmax': b_max[2], 'ymax': b_max[3]}}
-            choice_m_list.append(choice_m_dict)
-
-    # print(choice_m_list)
-    return choice_m_list
+def infer_solve(sheet_dict_list, left, right, top, bottom, col_regions, col_split):
+    if len(col_split) == 1:
+        col_split.insert(0, left)
+    else:
+        col_split.insert(0, left)
+        col_split.append(right)
+
+    boundary_list = [(split, top, col_split[i+1]-1, bottom) for i, split in enumerate(col_split[:-1])]
+
+    infer_polygon = []
+    tmp = []
+    for i, col in enumerate(col_regions):
+        if i == 0:
+            bottom_box = col[-1]
+            bottom_box_ymax = bottom_box['bounding_box']['ymax']
+            infer_loc = {'xmin': boundary_list[i][0], 'ymin': bottom_box_ymax+5,
+                         'xmax': boundary_list[i][2], 'ymax': boundary_list[i][3]}
+            box_polygon = Polygon([(infer_loc['xmin'], infer_loc['ymin']),
+                                   (infer_loc['xmax'], infer_loc['ymin']),
+                                   (infer_loc['xmax'], infer_loc['ymax']),
+                                   (infer_loc['xmin'], infer_loc['ymax']),])
+            infer_polygon.append(box_polygon)
+            tmp.append(infer_loc)
+
+        else:
+            top_box = col[0]
+            bottom_box_ymin = top_box['bounding_box']['ymin']
+            infer_loc = {'xmin': boundary_list[i][0], 'ymin': boundary_list[i][1],
+                         'xmax': boundary_list[i][2], 'ymax': bottom_box_ymin-5}
+            box_polygon = Polygon([(infer_loc['xmin'], infer_loc['ymin']),
+                                   (infer_loc['xmax'], infer_loc['ymin']),
+                                   (infer_loc['xmax'], infer_loc['ymax']),
+                                   (infer_loc['xmin'], infer_loc['ymax']),])
+            infer_polygon.append(box_polygon)
+            tmp.append(infer_loc)
+
+            bottom_box = col[-1]
+            bottom_box_ymax = bottom_box['bounding_box']['ymax']
+            infer_loc = {'xmin': boundary_list[i][0], 'ymin': bottom_box_ymax,
+                         'xmax': boundary_list[i][2], 'ymax': boundary_list[i][3]}
+            box_polygon = Polygon([(infer_loc['xmin'], infer_loc['ymin']),
+                                   (infer_loc['xmax'], infer_loc['ymin']),
+                                   (infer_loc['xmax'], infer_loc['ymax']),
+                                   (infer_loc['xmin'], infer_loc['ymax']),])
+            infer_polygon.append(box_polygon)
+            tmp.append(infer_loc)
+
+    res = []
+    all_type_score_polygon = []
+    for region_box in sheet_dict_list:
+        if region_box['class_name'] in ['type_score']:
+            coordinates = region_box['bounding_box']
+            xmin = coordinates['xmin']
+            ymin = coordinates['ymin']
+            xmax = coordinates['xmax']
+            ymax = coordinates['ymax']
+            box_polygon = Polygon([(xmin, ymin), (xmax, ymin), (xmax, ymax), (xmin, ymax)])
+            all_type_score_polygon.append(box_polygon)
+
+    for poly in infer_polygon.copy():  # infer type_score solve
+        p_xmin, p_ymin, p_xmax, p_ymax = poly.bounds
+        type_score_num = 0
+        type_score_ymin = []
+        for type_score_polygon in all_type_score_polygon:
+            cond1 = type_score_polygon.within(poly)
+            cond2 = False
+            cond3 = type_score_polygon.overlaps(poly)
+            if cond3:
+                intersection_poly = type_score_polygon.intersection(poly)
+                d1 = intersection_poly.area / type_score_polygon.area >= TYPE_SCORE_MNS
+                print('type_score:', intersection_poly.area / type_score_polygon.area)
+                d2 = type_score_polygon.area < 0.5 * poly.area
+                cond2 = d1 and d2
+
+            if cond1 or cond2:
+                t_xmin, t_ymin, t_xmax, t_ymax = type_score_polygon.bounds
+                x_dif = t_xmin - p_xmin
+                type_score_num += 1
+                type_score_ymin.append(t_ymin)
+                t_height = t_ymax - t_ymin
+                if t_ymin - p_ymin > 3 * t_height:
+                    type_score_num += 1
+                    type_score_ymin.append(p_ymin)
+
+        if type_score_num == 1:
+            solve_box = {'class_name': 'solve',
+                         'bounding_box': {'xmin': int(p_xmin), 'ymin': int(p_ymin),
+                                          'xmax': int(p_xmax), 'ymax': int(p_ymax)}}
+
+            sheet_dict_list.append(solve_box)
+            infer_polygon.remove(poly)
+            res.append(solve_box)
+        if type_score_num > 1:  # 多type_score
+            type_score_ymin = sorted(type_score_ymin)
+            type_score_ymin[0] = min(p_ymin, type_score_ymin[0])
+            type_score_ymin.append(p_ymax)
+            for i in range(0, len(type_score_ymin) - 1):
+                w = p_xmax - p_xmin
+                h = type_score_ymin[i + 1] - type_score_ymin[i]
+                if max(w / h, h / w) < ASPECT_FLAG:
+                    solve_box = {'class_name': 'solve',
+                                 'bounding_box': {'xmin': int(p_xmin), 'ymin': int(type_score_ymin[i]-5),
+                                                  'xmax': int(p_xmax), 'ymax': int(type_score_ymin[i + 1])}}
+                    sheet_dict_list.append(solve_box)
+                    res.append(solve_box)
+            infer_polygon.remove(poly)
+
+    return res

segment/sheet_resolve/analysis/sheet/sheet_points.py

@@ -81,6 +81,97 @@ def change_box(cloze_s_res, cloze_s_region):
     return words_result
 
 
+def get_cloze_number_and_value(cloze_s_box_with_content):
+    print(cloze_s_box_with_content)
+    list_of_cloze_s = []
+    number_value = []
+    for words_index, words_str in enumerate(cloze_s_box_with_content):
+        above_content = words_str['above_content']
+        above_content = sorted(above_content, key=lambda k: k['location']['left'])
+        words_list = [chars['words'] for chars in above_content]
+        words = ','.join(words_list)
+        pattern1 = re.compile(
+            '^\d+[,、.]?[\u4e00-\u9fa5]?[(（]?\d+分+[)）]?[(（]?\d+[)）]?|^\d+[,、.]?[\u4e00-\u9fa5]?[(（]?\d+分+[)）]?')
+        result1 = re.findall(pattern1, words)
+
+        pattern11 = re.compile('^\d+[,、.]?[\u4e00-\u9fa5]?[(（]?\d+[)）]?')
+        result11 = re.findall(pattern11, words)
+
+        pattern2 = re.compile('[(（]?\d+[)）]?')
+        result2 = re.findall(pattern2, words)
+
+        pattern3 = re.compile('[(（]?\d?分+[)）]?')
+        result3 = re.findall(pattern3, words)
+
+        pattern5 = re.compile('[\u4e00-\u9fa5]')
+        result5 = re.findall(pattern5, words)
+        # if result1:
+        #     print('ok')
+        if result11 and result2 and not result3 and not result5:
+            title_number = int(result2[0])
+            total_score = -1
+            words_str.update({'title_number': title_number, 'total_score': int(total_score)})
+            list_of_cloze_s.append(words_str)
+            number_value.append(title_number)
+            number_value.append(total_score)
+        if result11 and result3 and result5:
+            title_number1 = re.search('\d+', result11[0])
+            title_number = title_number1.group()
+            total_score_str = result3[0]
+            digital_total_score = re.search('\d+', total_score_str)
+            total_score = digital_total_score.group()
+            words_str.update({'title_number': int(title_number), 'total_score': int(total_score)})
+            list_of_cloze_s.append(words_str)
+            number_value.append(int(title_number))
+            number_value.append(total_score)
+        if not result11 and not result1 and result3 and result5:
+            title_number = -1
+            total_score_str = result3[0]
+            digital_total_score = re.search('\d+', total_score_str)
+            if digital_total_score == None:
+                total_score = -1
+            else:
+                total_score = digital_total_score.group()
+            words_str.update({'title_number': title_number, 'total_score': int(total_score)})
+            list_of_cloze_s.append(words_str)
+            number_value.append(title_number)
+            number_value.append(total_score)
+        if result11 and not result3 and not result5:
+            title_number1 = re.search('\d+', result11[0])
+            title_number = title_number1.group()
+            total_score = -1
+            words_str.update({'title_number': int(title_number), 'total_score': total_score})
+            list_of_cloze_s.append(words_str)
+            number_value.append(int(title_number))
+            number_value.append(total_score)
+        if not result11 and result2 and not result3 and not result5:
+            if len(result2) == 1:
+                title_number1 = re.search('\d+', result2[0])
+                title_number = title_number1.group()
+            elif len(result2) == 2:
+                title_number_str = re.search('\d+', result2[0])
+                title_number = int(title_number_str.group())
+            else:
+                title_number = -1
+            total_score = -1
+            words_str.update({'title_number': title_number, 'total_score': int(total_score)})
+            list_of_cloze_s.append(words_str)
+            number_value.append(title_number)
+            number_value.append(total_score)
+        elif len(number_value) == 0:
+            title_number = -1
+            total_score = -1
+            words_str.update({'title_number': title_number, 'total_score': int(total_score)})
+            list_of_cloze_s.append(words_str)
+    list_of_cloze_s_tmp = list_of_cloze_s.copy()
+    cloze_s_list = []
+    for cloze_s_s in list_of_cloze_s_tmp:
+        cloze_s_s_tmp = cloze_s_s.copy()
+        cloze_s_s_tmp.pop('above_content')
+        cloze_s_list.append(cloze_s_s)
+    return cloze_s_list
+
+
 def get_total_title_quantity_and_value(box_with_content):
     list_of_all = []
 
@@ -376,28 +467,33 @@ def get_sheet_points(sheet_dict_list):
             for one_col_box in cloze_or_solve0_box_list_front:
 
                 for ele_box in one_col_box:
-                    big_box = {}
-                    words_list = []
-                    for words_res in words_result_front:
-                        xmin = words_res['location']['left']
-                        ymin = words_res['location']['top']
-                        xmax = words_res['location']['left'] + words_res['location']['width']
-                        ymax = words_res['location']['top'] + words_res['location']['height']
-                        words_bbox = [xmin, ymin, xmax, ymax]
-                        if utils.decide_coordinate_contains1(words_bbox,
-                                                             [ele_box['bounding_box']['xmin'],
-                                                              ele_box['bounding_box']['ymin'],
-                                                              ele_box['bounding_box']['xmax'],
-                                                              ele_box['bounding_box']['ymax']]):
-                            words_list.append(words_res)
-                    big_box['class_name'] = ele_box['class_name']
-                    big_box['bounding_box'] = ele_box['bounding_box']
-                    big_box['above_content'] = words_list
-                    box_with_content_front.append(big_box)
-            box_with_content_front1 = box_with_content_front + cloze_s_box_with_content
-            list_of_front = get_total_title_quantity_and_value(box_with_content_front1)
+                    if ele_box['class_name'] == 'cloze_s':
+                        continue
+                    else:
+                        big_box = {}
+                        words_list = []
+                        for words_res in words_result_front:
+                            xmin = words_res['location']['left']
+                            ymin = words_res['location']['top']
+                            xmax = words_res['location']['left'] + words_res['location']['width']
+                            ymax = words_res['location']['top'] + words_res['location']['height']
+                            words_bbox = [xmin, ymin, xmax, ymax]
+                            if utils.decide_coordinate_contains1(words_bbox,
+                                                                 [ele_box['bounding_box']['xmin'],
+                                                                  ele_box['bounding_box']['ymin'],
+                                                                  ele_box['bounding_box']['xmax'],
+                                                                  ele_box['bounding_box']['ymax']]):
+                                words_list.append(words_res)
+                        big_box['class_name'] = ele_box['class_name']
+                        big_box['bounding_box'] = ele_box['bounding_box']
+                        big_box['above_content'] = words_list
+                        box_with_content_front.append(big_box)
+            # box_with_content_front1 = cloze_s_box_with_content + box_with_content_front
+            list_of_front1 = get_total_title_quantity_and_value(box_with_content_front)
+            list_of_cloze_s = get_cloze_number_and_value(cloze_s_box_with_content)
+            list_of_front = list_of_front1 + list_of_cloze_s
             sheet_dict_of_front.append(list_of_front)
-
+    # print(sheet_dict_of_front)
     sheet_dict_of_back = []
     sheet_dict_of_back_without_solve_cloze = []
     for single_sheet_dict in region_dict_back:
@@ -434,33 +530,40 @@ def get_sheet_points(sheet_dict_list):
             box_with_content_back = []
             for one_col_box in cloze_or_solve0_box_list_back:
                 for ele_box in one_col_box:
-                    words_list = []
-                    big_box = {}
-                    for words_res in words_result_back:
-                        xmin = words_res['location']['left']
-                        ymin = words_res['location']['top']
-                        xmax = words_res['location']['left'] + words_res['location']['width']
-                        ymax = words_res['location']['top'] + words_res['location']['height']
-                        words_bbox = [xmin, ymin, xmax, ymax]
-                        if utils.decide_coordinate_contains1(words_bbox,
-                                                             [ele_box['bounding_box']['xmin'],
-                                                              ele_box['bounding_box']['ymin'],
-                                                              ele_box['bounding_box']['xmax'],
-                                                              ele_box['bounding_box']['ymax']]):
-                            words_list.append(words_res)
-                    big_box['class_name'] = ele_box['class_name']
-                    big_box['bounding_box'] = ele_box['bounding_box']
-                    big_box['above_content'] = words_list
-                    box_with_content_back.append(big_box)
+                    if ele_box['class_name'] == 'cloze_s':
+                        continue
+                    else:
+                        words_list = []
+                        big_box = {}
+                        for words_res in words_result_back:
+                            xmin = words_res['location']['left']
+                            ymin = words_res['location']['top']
+                            xmax = words_res['location']['left'] + words_res['location']['width']
+                            ymax = words_res['location']['top'] + words_res['location']['height']
+                            words_bbox = [xmin, ymin, xmax, ymax]
+                            if utils.decide_coordinate_contains1(words_bbox,
+                                                                 [ele_box['bounding_box']['xmin'],
+                                                                  ele_box['bounding_box']['ymin'],
+                                                                  ele_box['bounding_box']['xmax'],
+                                                                  ele_box['bounding_box']['ymax']]):
+                                words_list.append(words_res)
+                        big_box['class_name'] = ele_box['class_name']
+                        big_box['bounding_box'] = ele_box['bounding_box']
+                        big_box['above_content'] = words_list
+                        box_with_content_back.append(big_box)
             # print(box_with_content_back)
 
-            box_with_content_back1 = box_with_content_back + cloze_s_box_with_content
+            # box_with_content_back1 = box_with_content_back + cloze_s_box_with_content
             # box_with_content_back1 = sorted(box_with_content_back1, key=lambda k: k.get('above_content'))
-            list_of_back = get_total_title_quantity_and_value(box_with_content_back1)
+            list_of_back1 = get_total_title_quantity_and_value(box_with_content_back)
+            list_of_cloze_s = get_cloze_number_and_value(cloze_s_box_with_content)
+            list_of_back = list_of_back1 + list_of_cloze_s
+
             sheet_dict_of_back.append(list_of_back)
 
     if sheet_dict_of_front != [] and sheet_dict_of_front_without_solve_cloze == []:
-        sheet_dict_of_front = check_classes(sheet_dict_of_front)
+        # sheet_dict_of_front = check_classes(sheet_dict_of_front)     raw
+        sheet_dict_of_front = sheet_dict_of_front
     else:
         sheet_dict_of_front = []
     if sheet_dict_of_back != [] and sheet_dict_of_back_without_solve_cloze == []:
@@ -492,11 +595,9 @@ def get_sheet_points(sheet_dict_list):
                                 ele13['bounding_box']:
                             title_number = ele13['title_number']
                             total_score = ele13['total_score']
-                            total_title_quantity = ele13['total_title_quantity']
-                            title_with_value = ele13['title_with_value']
-                            single_classes.update({'number': title_number, 'default_points': total_score,
-                                                   'total_title_quantity': total_title_quantity,
-                                                   'title_with_value': title_with_value})
+                            # total_title_quantity = ele13['total_title_quantity']
+                            # title_with_value = ele13['title_with_value']
+                            single_classes.update({'number': title_number, 'default_points': total_score})
                         else:
                             continue
     else:
@@ -512,12 +613,10 @@ def get_sheet_points(sheet_dict_list):
                                 single_class_back['bounding_box'] == regions_all['bounding_box']:
                             title_number0 = regions_all['title_number']
                             total_score0 = regions_all['total_score']
-                            total_title_quantity0 = regions_all['total_title_quantity']
-                            title_with_value0 = regions_all['title_with_value']
+                            # total_title_quantity0 = regions_all['total_title_quantity']
+                            # title_with_value0 = regions_all['title_with_value']
                             single_class_back.update({'number': title_number0,
-                                                      'default_points': total_score0,
-                                                      'total_title_quantity': total_title_quantity0,
-                                                      'title_with_value': title_with_value0})
+                                                      'default_points': total_score0})
                         else:
                             continue
     else:

segment/sheet_resolve/analysis/sheet/sheet_points_by_nlp.py

@@ -5,7 +5,6 @@
 
 import ast, cv2, re
 from segment.sheet_resolve.analysis.sheet.tag_parse import TagParse
-import CRFPP
 from segment.sheet_resolve.tools.utils import crop_region
 
 try:
@@ -189,6 +188,7 @@ def analyse_cloze_result1(result, cloze_and_cloze_s_list):
 
 
 def get_sheet_points_by_nlp(sheet_dict):
+    import CRFPP
     # json_path = r'C:\Users\Administrator\Desktop\type_score_nlp\type_score_info\example\english\33.json'
     # file = open(json_path, 'r', encoding='gbk').read()
     # json_file = ast.literal_eval(file)

segment/sheet_resolve/analysis/sheet/sheet_points_total.py

@@ -233,27 +233,27 @@ def model_type_score(all_type_score_one, choice_box, cloze_box, solve_box,compos
 
     if min_choice < min_cloze and min_choice < min_solve and min_choice < min_composition:
         # 建立相互关联的关系。 即表示该type_score对应于选择题
-        if test_result1 < 400:
+        if min_choice < 600:
             test_result1 = min_choice_dict
         else:
             test_result1 = -1
 
     elif min_cloze < min_choice and min_cloze < min_solve and min_cloze < min_composition:
         # 建立相互关联的关系。 即表示该type_score对应于填空题
-        if test_result1 < 400:
+        if min_cloze < 600:
             test_result1 = min_cloze_dict
         else:
             test_result1 = -1
 
     elif min_solve < min_cloze and min_solve < min_choice and min_solve < min_composition:
         # 建立相互关联的关系。 即表示该type_score对应于解答题
-        if test_result1 < 300:
+        if min_solve < 400:
             test_result1 = min_solve_dict
         else:
             test_result1 = -1
     elif min_composition < min_cloze and min_composition < min_choice and min_composition < min_solve:
         # 建立相互关联的关系。 即表示该type_score对应于解答题
-        if test_result1 < 300:
+        if min_composition < 400:
             test_result1 = min_composition_dict
         else:
             test_result1 = -1
@@ -795,12 +795,11 @@ def get_sheet_number_total(answer_sheet, res, img0):
                         if j == len(answer_sheet['regions']) - 1 and j_temp !=[]:
                             try:
                                 for index, jj in enumerate(j_temp):
-                                    num_score_m_infer = round(float(Score_last[i]['score'] / count_choice_m), 2)
-                                    num_score_m = [str(num_score_m_infer), int(num_score_m_infer)][int(num_score_m_infer) == num_score_m_infer]
+                                    num_score_m = round(float(Score_last[i]['score'] / count_choice_m), 2)
                                     answer_sheet['regions'][jj]['default_points'] = len(answer_sheet['regions'][jj]['number']) * [num_score_m]
                                 break
                             except Exception:
-                                pass
+                                print('choice_m_error')
                 elif Score_last[i]['label'] == 'cloze':
                     for j in range(len(answer_sheet['regions'])):
                         if answer_sheet['regions'][j]['class_name'] == 'cloze_s':
@@ -830,12 +829,11 @@ def get_sheet_number_total(answer_sheet, res, img0):
                         if j == len(answer_sheet['regions']) - 1 and j_temp !=[]:
                             try:
                                 for index, jj in enumerate(j_temp):
-                                    num_score_m_infer = round(float(Score_last[i]['score'] / count_choice_m), 2)
-                                    num_score_m = [str(num_score_m_infer), int(num_score_m_infer)][int(num_score_m_infer) == num_score_m_infer]
+                                    num_score_m = round(float(Score_last[i]['score'] / count_choice_m), 2)
                                     answer_sheet['regions'][jj]['default_points'] = len(answer_sheet['regions'][jj]['number']) * [num_score_m]
                                 break
                             except Exception:
-                                pass
+                                print('choice_m_error')
 
                 elif Score_last[i]['label'] == 'cloze':
                     for j in range(len(answer_sheet['regions'])):

segment/sheet_resolve/tools/brain_api.py

@@ -8,7 +8,8 @@ from urllib import parse, request
 import cv2
 import time
 import numpy as np
-
+from io import BytesIO
+from PIL import Image
 import pytesseract
 from segment.server import ocr_login
 from segment.sheet_resolve.tools import utils
@@ -54,9 +55,21 @@ def preprocess(img):
     return img
 
 
-def opecv2base64(img):
-    image = cv2.imencode('.jpg', img)[1]
-    base64_data = str(base64.b64encode(image))[2:-1]
+def opencv2base64(image, to_pil=False):
+    # image = cv2.imencode('.jpg', img)[1]
+    # base64_data = str(base64.b64encode(image))[2:-1]
+
+    if to_pil:
+        image = Image.fromarray(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
+        output_buffer = BytesIO()
+        image.save(output_buffer, format='JPEG')
+        byte_data = output_buffer.getvalue()
+        base64_data = base64.b64encode(byte_data)
+
+    else:
+        data = cv2.imencode('.jpg', image)[1]
+        base64_data = base64.b64encode(data.tostring()).decode('utf8')
+
     return base64_data
 
 
@@ -72,7 +85,7 @@ def get_ocr_raw_result(img, ocr_accuracy=OCR_ACCURACY, language_type='CHN_ENG'):
     group_id = 'group001'
     user_id = 'usr001'
 
-    image = opecv2base64(img)
+    image = opencv2base64(img)
 
     data = {
         'image_type': image_type,
@@ -110,7 +123,7 @@ def get_ocr_text_and_coordinate(img, ocr_accuracy=OCR_ACCURACY, language_type='C
     group_id = 'group001'
     user_id = 'usr001'
 
-    image = opecv2base64(img)
+    image = opencv2base64(img)
 
     data = {
         'image_type': image_type,
@@ -150,7 +163,7 @@ def get_ocr_text_and_coordinate0(img, ocr_accuracy=OCR_ACCURACY, language_type='
     group_id = 'group001'
     user_id = 'usr001'
 
-    image = opecv2base64(img)
+    image = opencv2base64(img)
 
     data = {
         'image_type': image_type,
@@ -190,7 +203,7 @@ def get_ocr_text_and_coordinate_direction(img, ocr_accuracy=OCR_ACCURACY, langua
     group_id = 'group001'
     user_id = 'usr001'
 
-    image = opecv2base64(img)
+    image = opencv2base64(img)
 
     data = {
         'image_type': image_type,
@@ -238,7 +251,7 @@ def get_ocr_text_and_coordinate_in_google_format(img, ocr_accuracy=OCR_ACCURACY,
     group_id = 'group001'
     user_id = 'usr001'
 
-    image = opecv2base64(img)
+    image = opencv2base64(img)
 
     data = {
         'image_type': image_type,
@@ -302,7 +315,7 @@ def get_handwriting_ocr_text_and_coordinate_in_google_format(img, words_type='wo
 
     headers = {'Content-Type': 'application/x-www-form-urlencoded'}
 
-    image = opecv2base64(img)
+    image = opencv2base64(img)
 
     data = {
         'image': image,

segment/sheet_server.py

@@ -132,6 +132,7 @@ def save_raw_image_without_segment(subject, datetime, img_file, analysis_type):
     file_name = '{}_{}.{}'.format(raw_name, uuid.uuid4().hex[:10], 'jpg')
 
     raw_img = Image.open(img_file)  # 读取上传的网络图像
+
     save_dir = os.path.join(settings.MEDIA_ROOT, analysis_type, subject, datetime)
     if not os.path.exists(save_dir):
         os.makedirs(save_dir)