清洗过程去掉写人json文件中以及更新配置。

application-webadmin/src/main/java/com/yice/webadmin/app/constant/DatasetConstant.java

@@ -92,6 +92,11 @@ public class DatasetConstant {
      */
     public static final Integer MARK = 1;
 
+    /**
+     * 未清洗
+     */
+    public static final Integer NOT_CLEAN_PROGRESS = 3;
+
     /**
      * 清洗中
      */

application-webadmin/src/main/java/com/yice/webadmin/app/controller/DatasetCleanController.java

@@ -13,6 +13,7 @@ import com.yice.common.log.model.constant.SysOperationLogType;
 import com.yice.webadmin.app.dto.DatasetCleanDto;
 import com.yice.webadmin.app.model.DatasetClean;
 import com.yice.webadmin.app.service.DatasetCleanService;
+import com.yice.webadmin.app.service.DatasetDataService;
 import com.yice.webadmin.app.vo.DatasetCleanVo;
 import io.swagger.annotations.Api;
 import lombok.extern.slf4j.Slf4j;
@@ -35,6 +36,8 @@ public class DatasetCleanController {
 
     @Autowired
     private DatasetCleanService datasetCleanService;
+    @Autowired
+    private DatasetDataService datasetDataService;
 
     /**
      * 新增数据集清洗数据。
@@ -71,6 +74,8 @@ public class DatasetCleanController {
         }
         DatasetClean datasetClean = MyModelUtil.copyTo(datasetCleanDto, DatasetClean.class);
         datasetClean = datasetCleanService.addNew(datasetClean);
+        datasetDataService.doDatasetCleanHandler(datasetClean.getDatasetId(), datasetClean.getCleanId(),
+                datasetClean.getCleanConfigId());
         return ResponseResult.success(datasetClean);
     }
 

application-webadmin/src/main/java/com/yice/webadmin/app/model/DatasetClean.java

@@ -85,6 +85,12 @@ public class DatasetClean extends BaseModel {
     @TableField(exist = false)
     private List<String> cleanMethod;
 
+    /**
+     * 清洗配置标识。
+     */
+    @TableField(exist = false)
+    private Long cleanConfigId;
+
     /**
      * 清洗配置对象。
      */

application-webadmin/src/main/java/com/yice/webadmin/app/service/DatasetCleanService.java

@@ -53,9 +53,10 @@ public interface DatasetCleanService extends IBaseService<DatasetClean, Long> {
      * @param dataList 数据集列表
      * @param cleanId 清洗标识
      * @param datasetId 清洗集标识
+     * @param cleanConfigId 清洗配置标识
      * @return 线程方法
      */
-    Future<Long> executeCleanTaskAsync(List<DatasetData> dataList, Long cleanId, Long datasetId);
+    Future<Long> executeCleanTaskAsync(List<DatasetData> dataList, Long cleanId, Long datasetId, Long cleanConfigId);
 
     /**
      * 保存清洗对象以及清洗配置对象。

application-webadmin/src/main/java/com/yice/webadmin/app/service/DatasetDataService.java

@@ -25,8 +25,9 @@ public interface DatasetDataService {
      * 开始清洗工作
      * @param datasetId 清洗数据集
      * @param cleanId 清洗标识
+     * @param cleanConfigId 清洗配置标识
      */
-    void doDatasetCleanHandler(Long datasetId, Long cleanId);
+    void doDatasetCleanHandler(Long datasetId, Long cleanId, Long cleanConfigId);
 
     /**
      * 删除整个集合中的文档数据。

application-webadmin/src/main/java/com/yice/webadmin/app/service/impl/DatasetDataServiceImpl.java

@@ -22,8 +22,6 @@ import org.springframework.stereotype.Service;
 
 import java.util.ArrayList;
 import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Future;
 
 /**
  * 数据集版本数据操作服务类。
@@ -89,16 +87,17 @@ public class DatasetDataServiceImpl implements DatasetDataService {
      * 开始清洗工作
      * @param datasetId 数据集对应的版本
      * @param cleanId 清洗任务标识
+     * @param cleanConfigId 清洗配置标识
      */
     @Async
     @Override
-    public void doDatasetCleanHandler(Long datasetId, Long cleanId) {
+    public void doDatasetCleanHandler(Long datasetId, Long cleanId, Long cleanConfigId) {
         MyPageParam param = new MyPageParam();
         param.setPageNum(DatasetConstant.PAGE_NUM);
         param.setPageSize(DatasetConstant.MAX_PAGE_SIZE);
         List<DatasetData> dataList = this.list(datasetId, param);
         if(CollUtil.isNotEmpty(dataList)) {
-            datasetCleanService.executeCleanTaskAsync(dataList, cleanId, datasetId);
+            datasetCleanService.executeCleanTaskAsync(dataList, cleanId, datasetId, cleanConfigId);
         }
     }
 

application-webadmin/src/main/java/com/yice/webadmin/app/service/impl/DatasetVersionServiceImpl.java

@@ -100,7 +100,7 @@ public class DatasetVersionServiceImpl extends BaseService<DatasetVersion, Long>
         datasetVersion.setDatasetVersion(version);
         datasetVersion.setVersionName(reDatasetManage.getDatasetName() + "_V" + version);
         datasetVersion.setDatasetId(reDatasetManage.getDatasetId());
-        datasetVersion.setCleanStatus(0);
+        datasetVersion.setCleanStatus(DatasetConstant.NOT_CLEAN_PROGRESS);
         datasetVersion.setDataVolume(0L);
         datasetVersion.setEnhanceStatus(0);
         datasetVersion.setInputStatus(0);
@@ -326,7 +326,7 @@ public class DatasetVersionServiceImpl extends BaseService<DatasetVersion, Long>
         String versionName = datasetVersion.getVersionName();
         //先存储文件
         String fullName = pythonConfig.getDatasetFileBaseDir() + versionName + ".json";
-        this.doDealTaskHandler(datasetVersion.getVersionId(), versionName, fullName);
+        this.doDealTaskHandler(datasetVersion.getVersionId(), fullName);
         //再存储数据集配置文件
         this.saveDatasetInfo(versionName);
         DatasetVersion filter = new DatasetVersion();
@@ -346,10 +346,9 @@ public class DatasetVersionServiceImpl extends BaseService<DatasetVersion, Long>
      * ==============================
      * 处理数据集
      * @param datasetId 清洗任务id
-     * @param versionName 数据集名称
      * @return 清洗列表
      */
-    private void doDealTaskHandler(Long datasetId, String versionName,String fileUrl) {
+    private void doDealTaskHandler(Long datasetId, String fileUrl) {
         try {
             Integer index = 0;
             Long count = datasetDataService.count(datasetId);
@@ -365,7 +364,8 @@ public class DatasetVersionServiceImpl extends BaseService<DatasetVersion, Long>
 
                     //写入到数据集中
                     if(CollUtil.isNotEmpty(dataList)) {
-                        datasetCleanService.appendDataListToFile(fileUrl ,dataList, i);
+                        datasetCleanService.appendDataListToFile(fileUrl, dataList, i);
+                        index ++;
                     }
                 }
 
@@ -378,7 +378,7 @@ public class DatasetVersionServiceImpl extends BaseService<DatasetVersion, Long>
                 this.datasetDataService.deleteByData(datasetId);
             }
         } catch (Exception ex) {
-            log.error("deal with task handler is error:" , ex);
+            log.error("do deal with task handler is error:" , ex);
         }
     }