Skip to content

L
lmp_server
Commit 21d6d730 authored by pengxin's avatar pengxin
Browse files
Options

发布数据集生成的JSON格式数据进行美化。

parent 9ff61528
Hide whitespace changes
Inline Side-by-side
Showing with 92 additions and 49 deletions
application-webadmin/src/main/java/com/yice/webadmin/app/service/DatasetCleanService.java
View file @ 21d6d730
......@@ -28,13 +28,7 @@ public interface DatasetCleanService extends IBaseService<DatasetClean, Long> {
* @param filePath 文件地址
* @param dataList 数据集列表
*/
void appendDataListToFile(String filePath, List<DatasetData> dataList,Integer pageNum);
/**
* 解析文件去掉多余的数据，比如文件里面最后一个，多加了一个",",缺少符号[]
* @param filePath 文件地址
*/
void readJsonAppendSymbol(String filePath);
void appendDataListToFile(String filePath, List<DatasetData> dataList);
/**
* 重新清洗任务
......
application-webadmin/src/main/java/com/yice/webadmin/app/service/impl/DatasetCleanServiceImpl.java
View file @ 21d6d730
......@@ -5,6 +5,9 @@ import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.github.pagehelper.Page;
import com.yice.common.core.base.dao.BaseDaoMapper;
......@@ -34,6 +37,10 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.io.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.*;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
......@@ -271,8 +278,7 @@ public class DatasetCleanServiceImpl extends BaseService<DatasetClean, Long> imp
* 解析文件去掉多余的数据，比如文件里面最后一个，多加了一个",",缺少符号[]
* @param filePath 文件地址
*/
@Override
public void readJsonAppendSymbol(String filePath) {
private void readJsonAppendSymbol(String filePath) {
try (FileReader fileReader = new FileReader(filePath);
BufferedReader bufferedReader = new BufferedReader(fileReader)) {
......@@ -315,40 +321,62 @@ public class DatasetCleanServiceImpl extends BaseService<DatasetClean, Long> imp
* @param dataList 数据集列表
*/
@Override
public void appendDataListToFile(String filePath, List<DatasetData> dataList,Integer pageNum) {
FileWriter fileWriter = null;
public void appendDataListToFile(String filePath, List<DatasetData> dataList) {
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.enable(SerializationFeature.INDENT_OUTPUT);
Path path = Paths.get(filePath);
ArrayNode arrayNode;
try {
//为第一页的情况下
if(pageNum == 1) {
// 清空文件内容
File file = new File(filePath);
if (file.exists()) {
// 删除文件
file.delete();
// 读取现有文件内容（如果存在）
String existingJson = "";
if (Files.exists(path)) {
existingJson = new String(Files.readAllBytes(path));
// 去除可能存在的尾随逗号（如果有的话）
existingJson = existingJson.trim();
if (existingJson.endsWith(",")) {
existingJson = existingJson.substring(0, existingJson.length() - 1);
}
// 创建一个新的空文件
file.createNewFile();
// 将JSON字符串转换为JsonNode
JsonNode rootNode = objectMapper.readTree(existingJson);
// 如果rootNode是一个ArrayNode，则使用它
if (rootNode.isArray()) {
arrayNode = (ArrayNode) rootNode;
} else {
// 如果不是ArrayNode，则视为文件内容有误或空，创建新的ArrayNode
log.warn("Existing JSON file is not an array, creating a new one.");
arrayNode = JsonNodeFactory.instance.arrayNode();
}
} else {
// 如果文件不存在，创建新的ArrayNode
arrayNode = JsonNodeFactory.instance.arrayNode();
}
fileWriter = new FileWriter(filePath, true);
// 遍历你的数据列表，并将每一条数据写入到文件中
// 将新数据转换为JSON并添加到现有数组中
for (DatasetData data : dataList) {
if(StringUtils.isNotBlank(data.getData())) {
fileWriter.write(data.getData() + ",");
fileWriter.write("\n");
}
String jsonString = data.getData();
arrayNode.add(objectMapper.readTree(jsonString));
}
} catch (IOException e) {
log.error("file write close is error", e);
} finally {
// 如果fileWriter不为空，关闭它
if (fileWriter != null) {
try {
fileWriter.close();
} catch (IOException e) {
log.error("file write close is error", e);
}
// 将整个JSON数组转换为格式化的字符串
String formattedJsonString = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(arrayNode);
// 如果文件原本为空或者不是有效的JSON数组，我们需要写入完整的数组
if (existingJson.isEmpty() || ! arrayNode.isArray()) {
Files.write(path, formattedJsonString.getBytes(), StandardOpenOption.CREATE);
} else {
// 否则，只追加新的元素（不带开头的"["和结尾的"]"）
String newElements = formattedJsonString.substring(1, formattedJsonString.length() - 1); // 去除开闭括号
Files.write(path, (existingJson + "," + newElements + "\n").getBytes(), StandardOpenOption.CREATE, StandardOpenOption.APPEND);
}
log.info("Data appended to JSON file successfully.");
} catch (IOException e) {
log.error("Error appending to JSON file", e);
}
}
......
application-webadmin/src/main/java/com/yice/webadmin/app/service/impl/DatasetOutputServiceImpl.java
View file @ 21d6d730
......@@ -2,6 +2,7 @@ package com.yice.webadmin.app.service.impl;
import cn.hutool.core.collection.CollUtil;
import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
......@@ -260,8 +261,8 @@ public class DatasetOutputServiceImpl extends BaseService<DatasetOutput, Long> i
// 创建一个新的HashMap，只包含"output"和"instruction"字段
Map<String, String> map = new HashMap<>();
map.put(DatasetConstant.OUTPUT, output);
map.put(DatasetConstant.INSTRUCTION, instruction);
map.put(DatasetConstant.OUTPUT, output);
// 使用pretty printer来格式化JSON字符串
String prettyJsonStr = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(map);
......@@ -339,6 +340,10 @@ public class DatasetOutputServiceImpl extends BaseService<DatasetOutput, Long> i
cell.setCellValue(rootNode.get(DatasetConstant.INSTRUCTION).textValue());
cell = row.createCell(1);
cell.setCellValue(rootNode.get(DatasetConstant.OUTPUT).textValue());
//根据内容自动填充列大小
sheet.autoSizeColumn(0);
sheet.autoSizeColumn(1);
}
String fileName = null;
InputStream inputStream = null;
......@@ -377,6 +382,8 @@ public class DatasetOutputServiceImpl extends BaseService<DatasetOutput, Long> i
InputStream inputStream = null;
try (ByteArrayOutputStream baos = new ByteArrayOutputStream()){
PrintWriter pw = new PrintWriter(baos);
// 计算最大的宽度
int maxWidth = getMaxWidth(dataList);
// 构建表头数据
pw.append(DatasetConstant.INSTRUCTION).append(",").append(DatasetConstant.OUTPUT).append("\n");
......@@ -390,7 +397,7 @@ public class DatasetOutputServiceImpl extends BaseService<DatasetOutput, Long> i
instruction = instruction.replace("\n", "");
output = output.replace("\n", "");
pw.append(output).append(",").append(instruction).append("\n");
pw.printf(maxWidth > 0 ? "%-" + maxWidth + "s,%-" + maxWidth + "s\n" : "%s,%s\n", output, instruction);
}
pw.flush();
// 转换为InputStream
......@@ -411,6 +418,30 @@ public class DatasetOutputServiceImpl extends BaseService<DatasetOutput, Long> i
return fileName;
}
/**
* 获取最大数据
* @param dataList 数据列表
* @return 最大长度
* @throws JsonProcessingException 异常
*/
private int getMaxWidth(List<DatasetData> dataList) throws JsonProcessingException {
int maxWidth = 0;
ObjectMapper objectMapper = new ObjectMapper();
for (DatasetData datasetData : dataList) {
JsonNode rootNode = objectMapper.readTree(datasetData.getData());
String instruction = rootNode.get(DatasetConstant.INSTRUCTION).textValue();
String output = rootNode.get(DatasetConstant.OUTPUT).textValue();
// 对包含换行符的字符串进行处理
instruction = instruction.replace("\n", "");
output = output.replace("\n", "");
maxWidth = Math.max(maxWidth, instruction.length());
maxWidth = Math.max(maxWidth, output.length());
}
return maxWidth;
}
/**
* 构建文件下载链接
* @param fileUrl 文件地址
......
application-webadmin/src/main/java/com/yice/webadmin/app/service/impl/DatasetVersionServiceImpl.java
View file @ 21d6d730
......@@ -583,7 +583,6 @@ public class DatasetVersionServiceImpl extends BaseService<DatasetVersion, Long>
@Override
public void doDealTaskHandler(Long datasetId, String fileUrl) {
try {
Integer index = 0;
Long count = datasetDataService.count(datasetId);
if (count > 0) {
int pageSize = DatasetConstant.MAX_SIZE;
......@@ -597,18 +596,9 @@ public class DatasetVersionServiceImpl extends BaseService<DatasetVersion, Long>
//写入到数据集中
if(CollUtil.isNotEmpty(dataList)) {
datasetCleanService.appendDataListToFile(fileUrl, dataList, i);
index ++;
datasetCleanService.appendDataListToFile(fileUrl, dataList);
}
}
//解析文件去掉多余的数据，比如文件里面最后一个，多加了一个",",缺少符号[]
if(index > 0){
datasetCleanService.readJsonAppendSymbol(fileUrl);
}
//删除为空的数据集数据
this.datasetDataService.deleteByData(datasetId);
}
} catch (Exception ex) {
log.error("do deal with task handler is error:" , ex);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment