Skip to content

L
lmp_server
Commit 9735901a authored by pengxin's avatar pengxin
Browse files
Options

数据集添加继承版本功能。

parent 5a8c60af
Hide whitespace changes
Inline Side-by-side
Showing with 201 additions and 46 deletions
application-webadmin/src/main/java/com/yice/webadmin/app/controller/DatasetVersionController.java
View file @ 9735901a
......@@ -2,7 +2,6 @@ package com.yice.webadmin.app.controller;
import cn.hutool.core.io.FileUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
......@@ -19,14 +18,11 @@ import com.yice.common.core.util.MyPageUtil;
import com.yice.common.log.annotation.OperationLog;
import com.yice.common.log.model.constant.SysOperationLogType;
import com.yice.webadmin.app.config.PythonConfig;
import com.yice.webadmin.app.data.DatasetData;
import com.yice.webadmin.app.dto.DatasetDetailDto;
import com.yice.webadmin.app.dto.DatasetVersionDto;
import com.yice.webadmin.app.model.DatasetDetail;
import com.yice.webadmin.app.model.DatasetVersion;
import com.yice.webadmin.app.service.DatasetDataService;
import com.yice.webadmin.app.service.DatasetVersionService;
import com.yice.webadmin.app.util.Sha1Util;
import com.yice.webadmin.app.vo.DatasetVersionVo;
import io.swagger.annotations.Api;
import lombok.extern.slf4j.Slf4j;
......@@ -45,7 +41,10 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
/**
* 数据集版本操作控制器类。
......@@ -63,8 +62,6 @@ public class DatasetVersionController {
private DatasetVersionService datasetVersionService;
@Autowired
private PythonConfig pythonConfig;
@Autowired
private DatasetDataService datasetDataService;
/**
* 新增数据集版本数据。
*
......@@ -270,7 +267,7 @@ public class DatasetVersionController {
//先存储文件
String fullName = this.saveDatasetFile(importFile, versionName, versionId);
//再存储数据集配置文件
this.saveDatasetInfo(versionName);
datasetVersionService.saveDatasetInfo(versionName);
datasetVersion.setFileUrl(fullName);
datasetVersion.setInputStatus(1);
datasetVersion.setDataVolume(Long.valueOf(JSON.parseArray(new String(importFile.getBytes(), StandardCharsets.UTF_8)).size()));
......@@ -278,36 +275,16 @@ public class DatasetVersionController {
return ResponseResult.success();
}
private void saveDatasetInfo(String versionName) throws IOException {
File file = new File(pythonConfig.getDatasetFileBaseDir(), pythonConfig.getDatasetInfo());
if (!file.exists()) {
file.createNewFile();
}
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.enable(SerializationFeature.INDENT_OUTPUT);
JsonNode rootNode = objectMapper.readTree(file);
if (rootNode == null || rootNode.isNull() || rootNode.size() < 1) {
rootNode = objectMapper.createObjectNode();
}
//此处应该要做键值映射关系，目前没做，后面需要修改
JSONObject jsonObject = new JSONObject();
jsonObject.put("file_name", versionName + ".json");
jsonObject.put("file_sha1", Sha1Util.getSha1(versionName));
((ObjectNode) rootNode).set(versionName, objectMapper.readTree(jsonObject.toJSONString()));
objectMapper.writeValue(new File(pythonConfig.getDatasetFileBaseDir(), pythonConfig.getDatasetInfo()), rootNode);
}
/**
* 保存导入文件。
*
* @param importFile 导入的文件。
* @return 保存的本地文件名。
*/
private String saveDatasetFile(MultipartFile importFile, String versionName, Long versionId) throws IOException {
private String saveDatasetFile(MultipartFile importFile, String versionName, Long versionId) throws IOException {
String fullName = pythonConfig.getDatasetFileBaseDir() + versionName + ".json";
byte[] bytes = null;
try {
bytes = importFile.getBytes();
byte[] bytes = importFile.getBytes();
Path path = Paths.get(fullName);
// 如果没有files文件夹，则创建
if (!Files.isWritable(path)) {
......@@ -315,25 +292,12 @@ public class DatasetVersionController {
}
// 文件写入指定路径
Files.write(path, bytes);
// 写入到mongodb中
datasetVersionService.writeDatasetFileForMongo(bytes,importFile.getOriginalFilename(),versionId);
} catch (IOException e) {
log.error("Failed to write imported file [" + importFile.getOriginalFilename() + " ].", e);
throw e;
}
try {
// 或者指定字符集进行转换,替换"UTF-8"为你想要使用的字符集
String result = new String(bytes, "UTF-8");
//先删除数据集
datasetDataService.delete(versionId);
//保存到mongodb中
datasetDataService.save(new DatasetData(null, versionId, result, new Date()));
}catch (Exception ex) {
log.error("Failed to write mongodb database [" + importFile.getOriginalFilename() + " ].", ex);
throw ex;
}
return fullName;
}
......
application-webadmin/src/main/java/com/yice/webadmin/app/service/DatasetVersionService.java
View file @ 9735901a
......@@ -4,6 +4,7 @@ import com.yice.common.core.base.service.IBaseService;
import com.yice.webadmin.app.model.DatasetDetail;
import com.yice.webadmin.app.model.DatasetVersion;
import java.io.IOException;
import java.util.List;
/**
......@@ -29,6 +30,21 @@ public interface DatasetVersionService extends IBaseService<DatasetVersion, Long
*/
void saveNewBatch(List<DatasetVersion> datasetVersionList);
/**
* 存储到python数据集中并进行更新操作
* @param versionName
* @throws IOException 异常操作
*/
void saveDatasetInfo(String versionName) throws IOException;
/**
* 写入到mongodb中
* @param bytes 字节
* @param originalFilename 导入文件名称
* @param versionId 版本标识
*/
void writeDatasetFileForMongo(byte[] bytes, String originalFilename, Long versionId);
/**
* 更新数据对象。
*
......
application-webadmin/src/main/java/com/yice/webadmin/app/service/impl/DatasetVersionServiceImpl.java
View file @ 9735901a
package com.yice.webadmin.app.service.impl;
import cn.hutool.core.collection.CollUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.github.pagehelper.Page;
import com.yice.common.core.base.dao.BaseDaoMapper;
import com.yice.common.core.base.service.BaseService;
......@@ -10,18 +16,30 @@ import com.yice.common.core.object.CallResult;
import com.yice.common.core.object.MyRelationParam;
import com.yice.common.core.util.MyModelUtil;
import com.yice.common.sequence.wrapper.IdGeneratorWrapper;
import com.yice.webadmin.app.config.PythonConfig;
import com.yice.webadmin.app.dao.DatasetVersionMapper;
import com.yice.webadmin.app.data.DatasetData;
import com.yice.webadmin.app.model.DatasetDetail;
import com.yice.webadmin.app.model.DatasetManage;
import com.yice.webadmin.app.model.DatasetVersion;
import com.yice.webadmin.app.service.DatasetDataService;
import com.yice.webadmin.app.service.DatasetDetailService;
import com.yice.webadmin.app.service.DatasetManageService;
import com.yice.webadmin.app.service.DatasetVersionService;
import com.yice.webadmin.app.util.Sha1Util;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Date;
import java.util.List;
/**
......@@ -42,6 +60,10 @@ public class DatasetVersionServiceImpl extends BaseService<DatasetVersion, Long>
private DatasetDetailService datasetDetailService;
@Autowired
private IdGeneratorWrapper idGenerator;
@Autowired
private PythonConfig pythonConfig;
@Autowired
private DatasetDataService datasetDataService;
/**
* 返回当前Service的主表Mapper对象。
......@@ -82,10 +104,163 @@ public class DatasetVersionServiceImpl extends BaseService<DatasetVersion, Long>
datasetVersion.setReleaseStatus(0);
datasetVersion.setTemplate(reDatasetManage.getTemplate());
datasetVersion.setDimensionType(reDatasetManage.getDimensionType());
datasetVersionMapper.insert(this.buildDefaultValue(datasetVersion));
insertDatasetVersion(datasetVersion);
return datasetVersion;
}
/**
* 添加数据对象
* @param datasetVersion 保存对象
*/
private void insertDatasetVersion(DatasetVersion datasetVersion) {
try {
//先进行数据填充
this.buildDefaultValue(datasetVersion);
//进行存储、写入以及更新配置等操作
DatasetVersion dataset = dealWithWriteAndSave(datasetVersion.getDatasetId(),
datasetVersion.getHisVersion(),datasetVersion.getVersionId(),
datasetVersion.getVersionName());
if(null != dataset) {
datasetVersion.setDataVolume(dataset.getDataVolume());
datasetVersion.setFileUrl(dataset.getFileUrl());
datasetVersion.setInputStatus(1);
}
//写入数据库中
datasetVersionMapper.insert(datasetVersion);
} catch (IOException e) {
log.error("Failed to save new data database [" + datasetVersion.getVersionName() + " ].", e);
}
}
/**
* 进行存储、写入以及更新配置等操作
* @param datasetId 数据集标识
* @param hisVersion 历史标识
* @param versionId 版本标识
* @param datasetName 数据集名称
*/
private DatasetVersion dealWithWriteAndSave(Long datasetId, Integer hisVersion,
Long versionId, String datasetName) throws IOException {
DatasetVersion filter = new DatasetVersion();
filter.setDatasetVersion(hisVersion);
filter.setDatasetId(datasetId);
DatasetVersion datasetVersion = this.getOne(filter);
if(null == datasetVersion) return null;
String fileUrl = datasetVersion.getFileUrl();
if(StringUtils.isNotBlank(fileUrl)) {
//获取新的json存放地址
String newFileName = this.extractFileName(fileUrl) + datasetName + ".json";
//写入到新的json文件格式中
byte [] bytes = writeAndSaveFileDetail(newFileName, fileUrl, versionId);
//再存储数据集配置文件
this.saveDatasetInfo(datasetName);
//重新赋值参数
datasetVersion.setFileUrl(newFileName);
datasetVersion.setInputStatus(1);
datasetVersion.setDataVolume(Long.valueOf(JSON.parseArray(new String(bytes, StandardCharsets.UTF_8)).size()));
}
return datasetVersion;
}
/**
* 存储到python数据集中并进行更新操作
* @param versionName 版本名称
* @throws IOException 异常操作
*/
public void saveDatasetInfo(String versionName) throws IOException {
File file = new File(pythonConfig.getDatasetFileBaseDir(), pythonConfig.getDatasetInfo());
if (!file.exists()) {
file.createNewFile();
}
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.enable(SerializationFeature.INDENT_OUTPUT);
JsonNode rootNode = objectMapper.readTree(file);
if (rootNode == null || rootNode.isNull() || rootNode.size() < 1) {
rootNode = objectMapper.createObjectNode();
}
//此处应该要做键值映射关系，目前没做，后面需要修改
JSONObject jsonObject = new JSONObject();
jsonObject.put("file_name", versionName + ".json");
jsonObject.put("file_sha1", Sha1Util.getSha1(versionName));
((ObjectNode) rootNode).set(versionName, objectMapper.readTree(jsonObject.toJSONString()));
objectMapper.writeValue(new File(pythonConfig.getDatasetFileBaseDir(), pythonConfig.getDatasetInfo()), rootNode);
}
/**
* 截取最后一个斜杠后的部分
* @param fileName 文件名称
* @return 返回截取后的字符串
*/
private static String extractFileName(String fileName) {
int lastSlashIndex = fileName.lastIndexOf('/');
if (lastSlashIndex != -1) {
fileName = fileName.substring(0, lastSlashIndex + 1);
}
return fileName;
}
/**
* 文件目录进行复制并写入到Mongodb数据库中
* @param fullName 新文件名称
* @param versionId 版本标识
* @param originFileName 原始文件名
* @return 字节编码
*/
private byte[] writeAndSaveFileDetail(String fullName,String originFileName,Long versionId) throws IOException{
byte[] bytes = null;
try{
bytes = readFileToBytes(originFileName);
Path path = Paths.get(fullName);
// 如果没有files文件夹，则创建
if (!Files.isWritable(path)) {
Files.createDirectories(Paths.get(pythonConfig.getDatasetFileBaseDir()));
}
// 文件写入指定路径
Files.write(path, bytes);
// 写入到mongodb中
writeDatasetFileForMongo(bytes,fullName,versionId);
} catch (IOException e) {
log.error("Failed to write imported file [" + fullName + " ].", e);
throw e;
}
return bytes;
}
/**
* 读取地址内容并转成bytes数据
* @param filePath 文件地址
* @return byte数组
* @throws IOException 异常
*/
private static byte[] readFileToBytes(String filePath) throws IOException {
// 使用Paths获取Path对象
Path path = Paths.get(filePath);
// 使用Files类的readAllBytes方法读取文件内容为byte[]
byte[] fileBytes = Files.readAllBytes(path);
return fileBytes;
}
/**
* 写入到mongodb中
* @param bytes 字节
* @param originalFilename 导入文件名称
* @param versionId 版本标识
*/
public void writeDatasetFileForMongo(byte[] bytes, String originalFilename, Long versionId) {
try {
// 或者指定字符集进行转换,替换"UTF-8"为你想要使用的字符集
String result = new String(bytes, StandardCharsets.UTF_8);
//先删除数据集
datasetDataService.delete(versionId);
//保存到mongodb中
datasetDataService.save(new DatasetData(null, versionId, result, new Date()));
}catch (Exception ex) {
log.error("Failed to write mongodb database [" + originalFilename + " ].", ex);
}
}
/**
* 利用数据库的insertList语法，批量插入对象列表。
*
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment