发布数据集生成的JSON格式数据进行美化。

application-webadmin/src/main/java/com/yice/webadmin/app/service/DatasetCleanService.java

@@ -28,13 +28,7 @@ public interface DatasetCleanService extends IBaseService<DatasetClean, Long> {
      * @param filePath 文件地址
      * @param dataList 数据集列表
      */
-    void appendDataListToFile(String filePath, List<DatasetData> dataList,Integer pageNum);
-
-    /**
-     * 解析文件去掉多余的数据，比如文件里面最后一个，多加了一个",",缺少符号[]
-     * @param filePath 文件地址
-     */
-    void readJsonAppendSymbol(String filePath);
+    void appendDataListToFile(String filePath, List<DatasetData> dataList);
 
     /**
      * 重新清洗任务

application-webadmin/src/main/java/com/yice/webadmin/app/service/impl/DatasetCleanServiceImpl.java

@@ -5,6 +5,9 @@ import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.github.pagehelper.Page;
 import com.yice.common.core.base.dao.BaseDaoMapper;
@@ -34,6 +37,10 @@ import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 
 import java.io.*;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
 import java.util.*;
 import java.util.concurrent.Future;
 import java.util.stream.Collectors;
@@ -271,8 +278,7 @@ public class DatasetCleanServiceImpl extends BaseService<DatasetClean, Long> imp
      * 解析文件去掉多余的数据，比如文件里面最后一个，多加了一个",",缺少符号[]
      * @param filePath 文件地址
      */
-    @Override
-    public void readJsonAppendSymbol(String filePath) {
+    private void readJsonAppendSymbol(String filePath) {
         try (FileReader fileReader = new FileReader(filePath);
              BufferedReader bufferedReader = new BufferedReader(fileReader)) {
 
@@ -315,40 +321,62 @@ public class DatasetCleanServiceImpl extends BaseService<DatasetClean, Long> imp
      * @param dataList 数据集列表
      */
     @Override
-    public void appendDataListToFile(String filePath, List<DatasetData> dataList,Integer pageNum) {
-        FileWriter fileWriter = null;
+    public void appendDataListToFile(String filePath, List<DatasetData> dataList) {
+        ObjectMapper objectMapper = new ObjectMapper();
+
+        objectMapper.enable(SerializationFeature.INDENT_OUTPUT);
+
+        Path path = Paths.get(filePath);
+        ArrayNode arrayNode;
+
         try {
-            //为第一页的情况下
-            if(pageNum == 1) {
-                // 清空文件内容
-                File file = new File(filePath);
-                if (file.exists()) {
-                    // 删除文件
-                    file.delete();
+            // 读取现有文件内容（如果存在）
+            String existingJson = "";
+            if (Files.exists(path)) {
+                existingJson = new String(Files.readAllBytes(path));
+                // 去除可能存在的尾随逗号（如果有的话）
+                existingJson = existingJson.trim();
+                if (existingJson.endsWith(",")) {
+                    existingJson = existingJson.substring(0, existingJson.length() - 1);
                 }
-                // 创建一个新的空文件
-                file.createNewFile();
+                // 将JSON字符串转换为JsonNode
+                JsonNode rootNode = objectMapper.readTree(existingJson);
+
+                // 如果rootNode是一个ArrayNode，则使用它
+                if (rootNode.isArray()) {
+                    arrayNode = (ArrayNode) rootNode;
+                } else {
+                    // 如果不是ArrayNode，则视为文件内容有误或空，创建新的ArrayNode
+                    log.warn("Existing JSON file is not an array, creating a new one.");
+                    arrayNode = JsonNodeFactory.instance.arrayNode();
+                }
+            } else {
+                // 如果文件不存在，创建新的ArrayNode
+                arrayNode = JsonNodeFactory.instance.arrayNode();
             }
 
-            fileWriter = new FileWriter(filePath, true);
-            // 遍历你的数据列表，并将每一条数据写入到文件中
+            // 将新数据转换为JSON并添加到现有数组中
             for (DatasetData data : dataList) {
-                if(StringUtils.isNotBlank(data.getData())) {
-                    fileWriter.write(data.getData() + ",");
-                    fileWriter.write("\n");
-                }
+                String jsonString = data.getData();
+                arrayNode.add(objectMapper.readTree(jsonString));
             }
-        } catch (IOException e) {
-            log.error("file write close is error", e);
-        } finally {
-            // 如果fileWriter不为空，关闭它
-            if (fileWriter != null) {
-                try {
-                    fileWriter.close();
-                } catch (IOException e) {
-                    log.error("file write close is error", e);
-                }
+
+            // 将整个JSON数组转换为格式化的字符串
+            String formattedJsonString = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(arrayNode);
+
+            // 如果文件原本为空或者不是有效的JSON数组，我们需要写入完整的数组
+            if (existingJson.isEmpty() || ! arrayNode.isArray()) {
+                Files.write(path, formattedJsonString.getBytes(), StandardOpenOption.CREATE);
+            } else {
+                // 否则，只追加新的元素（不带开头的"["和结尾的"]"）
+                String newElements = formattedJsonString.substring(1, formattedJsonString.length() - 1); // 去除开闭括号
+                Files.write(path, (existingJson + "," + newElements + "\n").getBytes(), StandardOpenOption.CREATE, StandardOpenOption.APPEND);
             }
+
+            log.info("Data appended to JSON file successfully.");
+
+        } catch (IOException e) {
+            log.error("Error appending to JSON file", e);
         }
     }
 

application-webadmin/src/main/java/com/yice/webadmin/app/service/impl/DatasetOutputServiceImpl.java

@@ -2,6 +2,7 @@ package com.yice.webadmin.app.service.impl;
 
 import cn.hutool.core.collection.CollUtil;
 import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper;
+import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -260,8 +261,8 @@ public class DatasetOutputServiceImpl extends BaseService<DatasetOutput, Long> i
 
                 // 创建一个新的HashMap，只包含"output"和"instruction"字段
                 Map<String, String> map = new HashMap<>();
-                map.put(DatasetConstant.OUTPUT, output);
                 map.put(DatasetConstant.INSTRUCTION, instruction);
+                map.put(DatasetConstant.OUTPUT, output);
 
                 // 使用pretty printer来格式化JSON字符串
                 String prettyJsonStr = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(map);
@@ -339,6 +340,10 @@ public class DatasetOutputServiceImpl extends BaseService<DatasetOutput, Long> i
             cell.setCellValue(rootNode.get(DatasetConstant.INSTRUCTION).textValue());
             cell = row.createCell(1);
             cell.setCellValue(rootNode.get(DatasetConstant.OUTPUT).textValue());
+
+            //根据内容自动填充列大小
+            sheet.autoSizeColumn(0);
+            sheet.autoSizeColumn(1);
         }
         String fileName = null;
         InputStream inputStream = null;
@@ -377,6 +382,8 @@ public class DatasetOutputServiceImpl extends BaseService<DatasetOutput, Long> i
         InputStream inputStream = null;
         try (ByteArrayOutputStream baos = new ByteArrayOutputStream()){
             PrintWriter pw = new PrintWriter(baos);
+            // 计算最大的宽度
+            int maxWidth = getMaxWidth(dataList);
             // 构建表头数据
             pw.append(DatasetConstant.INSTRUCTION).append(",").append(DatasetConstant.OUTPUT).append("\n");
 
@@ -390,7 +397,7 @@ public class DatasetOutputServiceImpl extends BaseService<DatasetOutput, Long> i
                 instruction = instruction.replace("\n", "");
                 output = output.replace("\n", "");
 
-                pw.append(output).append(",").append(instruction).append("\n");
+                pw.printf(maxWidth > 0 ? "%-" + maxWidth + "s,%-" + maxWidth + "s\n" : "%s,%s\n", output, instruction);
             }
             pw.flush();
             // 转换为InputStream
@@ -411,6 +418,30 @@ public class DatasetOutputServiceImpl extends BaseService<DatasetOutput, Long> i
         return fileName;
     }
 
+    /**
+     * 获取最大数据
+     * @param dataList 数据列表
+     * @return 最大长度
+     * @throws JsonProcessingException 异常
+     */
+    private int getMaxWidth(List<DatasetData> dataList) throws JsonProcessingException {
+        int maxWidth = 0;
+        ObjectMapper objectMapper = new ObjectMapper();
+        for (DatasetData datasetData : dataList) {
+            JsonNode rootNode = objectMapper.readTree(datasetData.getData());
+            String instruction = rootNode.get(DatasetConstant.INSTRUCTION).textValue();
+            String output = rootNode.get(DatasetConstant.OUTPUT).textValue();
+
+            // 对包含换行符的字符串进行处理
+            instruction = instruction.replace("\n", "");
+            output = output.replace("\n", "");
+
+            maxWidth = Math.max(maxWidth, instruction.length());
+            maxWidth = Math.max(maxWidth, output.length());
+        }
+        return maxWidth;
+    }
+
     /**
      * 构建文件下载链接
      * @param fileUrl 文件地址

application-webadmin/src/main/java/com/yice/webadmin/app/service/impl/DatasetVersionServiceImpl.java

@@ -583,7 +583,6 @@ public class DatasetVersionServiceImpl extends BaseService<DatasetVersion, Long>
     @Override
     public void doDealTaskHandler(Long datasetId, String fileUrl) {
         try {
-            Integer index = 0;
             Long count = datasetDataService.count(datasetId);
             if (count > 0) {
                 int pageSize = DatasetConstant.MAX_SIZE;
@@ -597,18 +596,9 @@ public class DatasetVersionServiceImpl extends BaseService<DatasetVersion, Long>
 
                     //写入到数据集中
                     if(CollUtil.isNotEmpty(dataList)) {
-                        datasetCleanService.appendDataListToFile(fileUrl, dataList, i);
-                        index ++;
+                        datasetCleanService.appendDataListToFile(fileUrl, dataList);
                     }
                 }
-
-                //解析文件去掉多余的数据，比如文件里面最后一个，多加了一个",",缺少符号[]
-                if(index > 0){
-                    datasetCleanService.readJsonAppendSymbol(fileUrl);
-                }
-
-                //删除为空的数据集数据
-                this.datasetDataService.deleteByData(datasetId);
             }
         } catch (Exception ex) {
             log.error("do deal with task handler is error:" , ex);