|
@@ -7,10 +7,14 @@ import cn.sino.common.base.PageResult;
|
|
import cn.sino.common.exception.BizException;
|
|
import cn.sino.common.exception.BizException;
|
|
import cn.sino.common.util.SnowflakeIdFactory;
|
|
import cn.sino.common.util.SnowflakeIdFactory;
|
|
import cn.sino.mark.common.constants.Constant;
|
|
import cn.sino.mark.common.constants.Constant;
|
|
-import cn.sino.mark.common.enums.*;
|
|
|
|
|
|
+import cn.sino.mark.common.enums.IsOrNotEnum;
|
|
|
|
+import cn.sino.mark.common.enums.ResourceAssignStatusEnum;
|
|
|
|
+import cn.sino.mark.common.enums.ResourceTypeEnum;
|
|
|
|
+import cn.sino.mark.common.enums.SysUserStatusEnum;
|
|
import cn.sino.mark.common.log.Log;
|
|
import cn.sino.mark.common.log.Log;
|
|
import cn.sino.mark.common.log.SysLogEventPublisher;
|
|
import cn.sino.mark.common.log.SysLogEventPublisher;
|
|
import cn.sino.mark.common.model.bo.ResourceBO;
|
|
import cn.sino.mark.common.model.bo.ResourceBO;
|
|
|
|
+import cn.sino.mark.common.model.bo.ResourceDataBO;
|
|
import cn.sino.mark.common.model.dto.AddResourceDTO;
|
|
import cn.sino.mark.common.model.dto.AddResourceDTO;
|
|
import cn.sino.mark.common.model.dto.ImportResourceDTO;
|
|
import cn.sino.mark.common.model.dto.ImportResourceDTO;
|
|
import cn.sino.mark.common.model.dto.QueryResourceDTO;
|
|
import cn.sino.mark.common.model.dto.QueryResourceDTO;
|
|
@@ -28,13 +32,24 @@ import cn.sino.mark.service.convert.ResourceMapStruct;
|
|
import com.github.pagehelper.Page;
|
|
import com.github.pagehelper.Page;
|
|
import lombok.RequiredArgsConstructor;
|
|
import lombok.RequiredArgsConstructor;
|
|
import lombok.extern.slf4j.Slf4j;
|
|
import lombok.extern.slf4j.Slf4j;
|
|
|
|
+import org.apache.commons.lang3.StringUtils;
|
|
|
|
+import org.apache.poi.xssf.usermodel.XSSFCell;
|
|
|
|
+import org.apache.poi.xssf.usermodel.XSSFRow;
|
|
|
|
+import org.apache.poi.xssf.usermodel.XSSFSheet;
|
|
|
|
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
|
import org.springframework.stereotype.Service;
|
|
import org.springframework.stereotype.Service;
|
|
import org.springframework.transaction.annotation.Transactional;
|
|
import org.springframework.transaction.annotation.Transactional;
|
|
import org.springframework.web.multipart.MultipartFile;
|
|
import org.springframework.web.multipart.MultipartFile;
|
|
|
|
|
|
-import java.io.*;
|
|
|
|
|
|
+import java.io.BufferedReader;
|
|
|
|
+import java.io.ByteArrayInputStream;
|
|
|
|
+import java.io.IOException;
|
|
|
|
+import java.io.InputStreamReader;
|
|
import java.time.LocalDateTime;
|
|
import java.time.LocalDateTime;
|
|
-import java.util.*;
|
|
|
|
|
|
+import java.util.ArrayList;
|
|
|
|
+import java.util.List;
|
|
|
|
+import java.util.Map;
|
|
|
|
+import java.util.Objects;
|
|
import java.util.regex.Matcher;
|
|
import java.util.regex.Matcher;
|
|
import java.util.regex.Pattern;
|
|
import java.util.regex.Pattern;
|
|
import java.util.stream.Collectors;
|
|
import java.util.stream.Collectors;
|
|
@@ -142,12 +157,6 @@ public class ResourceServiceImpl implements ResourceService {
|
|
log.error("json解析失败: ", jsonException);
|
|
log.error("json解析失败: ", jsonException);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
-
|
|
|
|
-// BufferedWriter out = new BufferedWriter(new FileWriter("E://Java//movie10w.txt"));
|
|
|
|
-// for (int i = 0; i < 100000; i++) {
|
|
|
|
-// out.write("{\"data\":{\"movsource\" :\"http://media.meetsocial.cn/video%2Fvideo_46da6b590867b54be06cc087301ef7dc.mp4\",\"hint\":\"这是一段注释或参考文本内容\"}}}");
|
|
|
|
-// out.newLine();
|
|
|
|
-// }
|
|
|
|
} catch (Exception e) {
|
|
} catch (Exception e) {
|
|
log.error(e.getMessage());
|
|
log.error(e.getMessage());
|
|
} finally {
|
|
} finally {
|
|
@@ -185,86 +194,23 @@ public class ResourceServiceImpl implements ResourceService {
|
|
}
|
|
}
|
|
resourceManager.deleteDataByResourceId(resourceId);
|
|
resourceManager.deleteDataByResourceId(resourceId);
|
|
Integer type = resourceDO.getType();
|
|
Integer type = resourceDO.getType();
|
|
- log.info("开始解析资源...");
|
|
|
|
- String suffix = "." + getUploadSuffix(Objects.requireNonNull(uploadFile.getOriginalFilename()));
|
|
|
|
- if (!isJson(suffix)) {
|
|
|
|
- throw new BizException("只能上传json或txt后缀的文本文件");
|
|
|
|
- }
|
|
|
|
- List<String> effectiveJsonList = new ArrayList<>();
|
|
|
|
- BufferedReader br = null;
|
|
|
|
- List<ResourceDataDO> resourceDataDOS = new ArrayList<>();
|
|
|
|
- LocalDateTime now = LocalDateTime.now();
|
|
|
|
- int totalCount = 0;
|
|
|
|
- try {
|
|
|
|
- InputStreamReader read = new InputStreamReader(new ByteArrayInputStream(uploadFile.getBytes()));
|
|
|
|
- br = new BufferedReader(read);//BufferedReader读取文件
|
|
|
|
- String s;
|
|
|
|
- while ((s = br.readLine()) != null) {
|
|
|
|
- ResourceDataDO dataDO = new ResourceDataDO();
|
|
|
|
- dataDO.setResourceId(resourceId);
|
|
|
|
- dataDO.setDataId(snowflakeIdFactory.nextId());
|
|
|
|
- dataDO.setCreateTime(now);
|
|
|
|
- dataDO.setIsDraft(IsOrNotEnum.IS.getValue());
|
|
|
|
- totalCount++;
|
|
|
|
- try {
|
|
|
|
- Map map = (Map) JSONUtil.parse(s.trim());
|
|
|
|
- Map dataMap = (Map) map.get("data");
|
|
|
|
- if (dataMap != null) {
|
|
|
|
- Object hint = dataMap.get("hint");
|
|
|
|
- if (hint != null) {
|
|
|
|
- dataDO.setHint(hint.toString().trim());
|
|
|
|
- }
|
|
|
|
- if (ResourceTypeEnum.DOC.getValue().equals(type)) {
|
|
|
|
- Object txtSource = dataMap.get("txtsource");
|
|
|
|
- if (txtSource != null) {
|
|
|
|
- effectiveJsonList.add(s.trim());
|
|
|
|
- dataDO.setContent(txtSource.toString().trim());
|
|
|
|
- }
|
|
|
|
- } else if (ResourceTypeEnum.PICTURE.getValue().equals(type)) {
|
|
|
|
- Object picSource = dataMap.get("picsource");
|
|
|
|
- if (picSource != null) {
|
|
|
|
- effectiveJsonList.add(s.trim());
|
|
|
|
- dataDO.setPicUrl(picSource.toString().trim());
|
|
|
|
- }
|
|
|
|
- } else {
|
|
|
|
- Object movSource = dataMap.get("movsource");
|
|
|
|
- if (movSource != null) {
|
|
|
|
- effectiveJsonList.add(s.trim());
|
|
|
|
- dataDO.setMovUrl(movSource.toString().trim());
|
|
|
|
- }
|
|
|
|
- }
|
|
|
|
- }
|
|
|
|
- resourceDataDOS.add(dataDO);
|
|
|
|
- } catch (JSONException jsonException) {
|
|
|
|
- //解析失败不用理会
|
|
|
|
- log.error("json解析失败: ", jsonException);
|
|
|
|
- }
|
|
|
|
- }
|
|
|
|
|
|
|
|
-// BufferedWriter out = new BufferedWriter(new FileWriter("E://Java//movie10w.txt"));
|
|
|
|
-// for (int i = 0; i < 100000; i++) {
|
|
|
|
-// out.write("{\"data\":{\"movsource\" :\"http://media.meetsocial.cn/video%2Fvideo_46da6b590867b54be06cc087301ef7dc.mp4\",\"hint\":\"这是一段注释或参考文本内容\"}}}");
|
|
|
|
-// out.newLine();
|
|
|
|
-// }
|
|
|
|
- } catch (Exception e) {
|
|
|
|
- log.error(e.getMessage());
|
|
|
|
- } finally {
|
|
|
|
- if (br != null) {
|
|
|
|
- try {
|
|
|
|
- br.close();
|
|
|
|
- } catch (IOException e) {
|
|
|
|
- e.printStackTrace();
|
|
|
|
- }
|
|
|
|
- }
|
|
|
|
|
|
+ List<ResourceDataBO> resourceDataList;
|
|
|
|
+ if (type.equals(ResourceTypeEnum.LINK.getValue())) {
|
|
|
|
+ resourceDataList = parseExcelContent(uploadFile, resourceId, type);
|
|
|
|
+ } else {
|
|
|
|
+ resourceDataList = parseJsonContent(uploadFile, resourceId, type);
|
|
}
|
|
}
|
|
- if (isNotEmpty(resourceDataDOS)) {
|
|
|
|
- ListsUtil.splitRun(resourceDataDOS, 5000, resourceManager::insertBatch);
|
|
|
|
- logEventPublisher.publish(Log.builder().append("上传资源数据,ID:" + resourceId + ",数据量:" + resourceDataDOS.size()).isInsert());
|
|
|
|
|
|
+ List<ResourceDataDO> resourceDataDOList = resourceDataList.stream()
|
|
|
|
+ .filter(ResourceDataBO::getIsValid).map(ResourceDataBO::getResourceData).collect(Collectors.toList());
|
|
|
|
+ if (isNotEmpty(resourceDataDOList)) {
|
|
|
|
+ ListsUtil.splitRun(resourceDataDOList, 5000, resourceManager::insertBatch);
|
|
|
|
+ logEventPublisher.publish(Log.builder().append("上传资源数据,ID:" + resourceId + ",数据量:" + resourceDataDOList.size()).isInsert());
|
|
}
|
|
}
|
|
|
|
|
|
ImportResourceVO importResourceVO = new ImportResourceVO();
|
|
ImportResourceVO importResourceVO = new ImportResourceVO();
|
|
- importResourceVO.setResourceCount(resourceDataDOS.size());
|
|
|
|
- importResourceVO.setFailedCount(totalCount - resourceDataDOS.size());
|
|
|
|
|
|
+ importResourceVO.setResourceCount(resourceDataDOList.size());
|
|
|
|
+ importResourceVO.setFailedCount(resourceDataList.size() - resourceDataDOList.size());
|
|
log.info("资源解析结束,耗时: {}ms", (System.currentTimeMillis() - start));
|
|
log.info("资源解析结束,耗时: {}ms", (System.currentTimeMillis() - start));
|
|
return importResourceVO;
|
|
return importResourceVO;
|
|
}
|
|
}
|
|
@@ -373,6 +319,7 @@ public class ResourceServiceImpl implements ResourceService {
|
|
*/
|
|
*/
|
|
@Override
|
|
@Override
|
|
public PageResult<ResourcePageItemVO> getResourcePage(QueryResourceDTO queryResourceDTO) {
|
|
public PageResult<ResourcePageItemVO> getResourcePage(QueryResourceDTO queryResourceDTO) {
|
|
|
|
+ // 查询当前用户信息
|
|
startPage(queryResourceDTO.getPageNum(), queryResourceDTO.getPageSize());
|
|
startPage(queryResourceDTO.getPageNum(), queryResourceDTO.getPageSize());
|
|
Page<ResourceBO> resourceBOS = resourceManager.queryResourcePage(queryResourceDTO);
|
|
Page<ResourceBO> resourceBOS = resourceManager.queryResourcePage(queryResourceDTO);
|
|
Page<ResourcePageItemVO> page = new Page<>();
|
|
Page<ResourcePageItemVO> page = new Page<>();
|
|
@@ -479,10 +426,129 @@ public class ResourceServiceImpl implements ResourceService {
|
|
* @param suffix 后缀
|
|
* @param suffix 后缀
|
|
* @return boolean
|
|
* @return boolean
|
|
*/
|
|
*/
|
|
- public static boolean isJson(String suffix) {
|
|
|
|
|
|
+ private boolean isJson(String suffix) {
|
|
Pattern p = Pattern.compile("\\.(json|txt)");
|
|
Pattern p = Pattern.compile("\\.(json|txt)");
|
|
Matcher m = p.matcher(suffix);
|
|
Matcher m = p.matcher(suffix);
|
|
return m.find();
|
|
return m.find();
|
|
}
|
|
}
|
|
|
|
|
|
|
|
+ /**
|
|
|
|
+ * 判断是否文本
|
|
|
|
+ *
|
|
|
|
+ * @param suffix 后缀
|
|
|
|
+ * @return boolean
|
|
|
|
+ */
|
|
|
|
+ private boolean isExcel(String suffix) {
|
|
|
|
+ Pattern p = Pattern.compile("\\.(xls|xlsx)");
|
|
|
|
+ Matcher m = p.matcher(suffix);
|
|
|
|
+ return m.find();
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ /**
|
|
|
|
+ * 解析json内容(文本,图片,视频)
|
|
|
|
+ *
|
|
|
|
+ * @param uploadFile 上传文件
|
|
|
|
+ * @param resourceId 资源id
|
|
|
|
+ * @param type 类型
|
|
|
|
+ * @return {@link List}<{@link ResourceDataDO}>
|
|
|
|
+ */
|
|
|
|
+ private List<ResourceDataBO> parseJsonContent(MultipartFile uploadFile, Long resourceId, Integer type) {
|
|
|
|
+ String suffix = "." + getUploadSuffix(Objects.requireNonNull(uploadFile.getOriginalFilename()));
|
|
|
|
+ if (!isJson(suffix)) {
|
|
|
|
+ throw new BizException("只能上传json或txt后缀的文本文件");
|
|
|
|
+ }
|
|
|
|
+ List<ResourceDataBO> resourceDataList = new ArrayList<>();
|
|
|
|
+ LocalDateTime now = LocalDateTime.now();
|
|
|
|
+ try (InputStreamReader read = new InputStreamReader(new ByteArrayInputStream(uploadFile.getBytes()));
|
|
|
|
+ BufferedReader br = new BufferedReader(read)) {
|
|
|
|
+ String s;
|
|
|
|
+ while ((s = br.readLine()) != null) {
|
|
|
|
+ ResourceDataDO dataDO = new ResourceDataDO();
|
|
|
|
+ dataDO.setResourceId(resourceId);
|
|
|
|
+ dataDO.setDataId(snowflakeIdFactory.nextId());
|
|
|
|
+ dataDO.setCreateTime(now);
|
|
|
|
+ dataDO.setIsDraft(IsOrNotEnum.IS.getValue());
|
|
|
|
+ ResourceDataBO resourceDataBO = ResourceDataBO.builder().resourceData(dataDO).isValid(false).build();
|
|
|
|
+ try {
|
|
|
|
+ Map map = (Map) JSONUtil.parse(s.trim());
|
|
|
|
+ Map dataMap = (Map) map.get("data");
|
|
|
|
+ if (dataMap != null) {
|
|
|
|
+ Object hint = dataMap.get("hint");
|
|
|
|
+ if (hint != null) {
|
|
|
|
+ dataDO.setHint(hint.toString().trim());
|
|
|
|
+ }
|
|
|
|
+ if (ResourceTypeEnum.DOC.getValue().equals(type)) {
|
|
|
|
+ Object txtSource = dataMap.get("txtsource");
|
|
|
|
+ if (txtSource != null) {
|
|
|
|
+ dataDO.setContent(txtSource.toString().trim());
|
|
|
|
+ }
|
|
|
|
+ } else if (ResourceTypeEnum.PICTURE.getValue().equals(type)) {
|
|
|
|
+ Object picSource = dataMap.get("picsource");
|
|
|
|
+ if (picSource != null) {
|
|
|
|
+ dataDO.setPicUrl(picSource.toString().trim());
|
|
|
|
+ }
|
|
|
|
+ } else {
|
|
|
|
+ Object movSource = dataMap.get("movsource");
|
|
|
|
+ if (movSource != null) {
|
|
|
|
+ dataDO.setMovUrl(movSource.toString().trim());
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+ resourceDataBO.setIsValid(true);
|
|
|
|
+ resourceDataList.add(resourceDataBO);
|
|
|
|
+ } catch (JSONException jsonException) {
|
|
|
|
+ //解析失败不用理会
|
|
|
|
+ log.error("资源文件解析失败: ", jsonException);
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+ } catch (Exception e) {
|
|
|
|
+ log.error(e.getMessage());
|
|
|
|
+ }
|
|
|
|
+ return resourceDataList;
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ /**
|
|
|
|
+ * 解析excel内容(链接)
|
|
|
|
+ *
|
|
|
|
+ * @param uploadFile 上传文件
|
|
|
|
+ * @param resourceId 资源id
|
|
|
|
+ * @param type 类型
|
|
|
|
+ * @return {@link List}<{@link ResourceDataDO}>
|
|
|
|
+ */
|
|
|
|
+ private List<ResourceDataBO> parseExcelContent(MultipartFile uploadFile, Long resourceId, Integer type) {
|
|
|
|
+ String suffix = "." + getUploadSuffix(Objects.requireNonNull(uploadFile.getOriginalFilename()));
|
|
|
|
+ if (!isExcel(suffix)) {
|
|
|
|
+ throw new BizException("只能上传excel文件");
|
|
|
|
+ }
|
|
|
|
+ List<ResourceDataBO> resourceDataList = new ArrayList<>();
|
|
|
|
+ LocalDateTime now = LocalDateTime.now();
|
|
|
|
+
|
|
|
|
+ try (XSSFWorkbook excel = new XSSFWorkbook(uploadFile.getInputStream())) {
|
|
|
|
+ // 读取且仅仅读取第一个sheet
|
|
|
|
+ XSSFSheet sheet = excel.getSheetAt(0);
|
|
|
|
+ for (int i = 1; i <= sheet.getLastRowNum(); i++) {
|
|
|
|
+ ResourceDataDO dataDO = new ResourceDataDO();
|
|
|
|
+ dataDO.setResourceId(resourceId);
|
|
|
|
+ dataDO.setDataId(snowflakeIdFactory.nextId());
|
|
|
|
+ dataDO.setCreateTime(now);
|
|
|
|
+ dataDO.setIsDraft(IsOrNotEnum.IS.getValue());
|
|
|
|
+ ResourceDataBO resourceDataBO = ResourceDataBO.builder().resourceData(dataDO).isValid(false).build();
|
|
|
|
+ // 从第二行开始读取,第一行是头部
|
|
|
|
+ XSSFRow row = sheet.getRow(i);
|
|
|
|
+ // 读取第1列
|
|
|
|
+ XSSFCell cell = row.getCell(0);
|
|
|
|
+ String rawValue = cell.getStringCellValue();
|
|
|
|
+ if (StringUtils.isNotBlank(rawValue)) {
|
|
|
|
+ dataDO.setContent(rawValue);
|
|
|
|
+ resourceDataBO.setIsValid(true);
|
|
|
|
+ }
|
|
|
|
+ resourceDataList.add(resourceDataBO);
|
|
|
|
+ }
|
|
|
|
+ } catch (IOException e) {
|
|
|
|
+ // 解析失败不用理会
|
|
|
|
+ log.error("资源文件解析失败: ", e);
|
|
|
|
+ }
|
|
|
|
+ return resourceDataList;
|
|
|
|
+ }
|
|
|
|
+
|
|
}
|
|
}
|