Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
L
lmp_server
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
lmp
lmp_server
Commits
f55561a9
Commit
f55561a9
authored
Nov 30, 2023
by
linpeiqin
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
数据集写入文件问题,需要再python数据集配置文件中进行写入才行
parent
9bb1ac79
Changes
4
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
104 additions
and
6 deletions
+104
-6
PythonConfig.java
.../main/java/com/yice/webadmin/app/config/PythonConfig.java
+26
-0
DatasetVersionController.java
...ice/webadmin/app/controller/DatasetVersionController.java
+48
-5
Sha1Util.java
...in/src/main/java/com/yice/webadmin/app/util/Sha1Util.java
+23
-0
application-dev.yml
application-webadmin/src/main/resources/application-dev.yml
+7
-1
No files found.
application-webadmin/src/main/java/com/yice/webadmin/app/config/PythonConfig.java
0 → 100644
View file @
f55561a9
package
com
.
yice
.
webadmin
.
app
.
config
;
import
lombok.Data
;
import
org.springframework.boot.context.properties.ConfigurationProperties
;
import
org.springframework.context.annotation.Configuration
;
/**
* 应用程序自定义的程序属性配置文件。
*
* @author linking
* @date 2023-04-13
*/
@Data
@Configuration
@ConfigurationProperties
(
prefix
=
"python"
)
public
class
PythonConfig
{
/**
* 数据集基础目录
*/
private
String
datasetFileBaseDir
;
/**
* 数据集配置文件
*/
private
String
datasetInfo
;
}
application-webadmin/src/main/java/com/yice/webadmin/app/controller/DatasetVersionController.java
View file @
f55561a9
...
@@ -4,24 +4,27 @@ import com.alibaba.fastjson.JSON;
...
@@ -4,24 +4,27 @@ import com.alibaba.fastjson.JSON;
import
com.fasterxml.jackson.databind.JsonNode
;
import
com.fasterxml.jackson.databind.JsonNode
;
import
com.fasterxml.jackson.databind.ObjectMapper
;
import
com.fasterxml.jackson.databind.ObjectMapper
;
import
com.fasterxml.jackson.databind.node.ArrayNode
;
import
com.fasterxml.jackson.databind.node.ArrayNode
;
import
com.fasterxml.jackson.databind.node.ObjectNode
;
import
com.github.pagehelper.page.PageMethod
;
import
com.github.pagehelper.page.PageMethod
;
import
com.github.xiaoymin.knife4j.annotations.ApiOperationSupport
;
import
com.github.xiaoymin.knife4j.annotations.ApiOperationSupport
;
import
com.yice.common.core.annotation.MyRequestBody
;
import
com.yice.common.core.annotation.MyRequestBody
;
import
com.yice.common.core.constant.ErrorCodeEnum
;
import
com.yice.common.core.constant.ErrorCodeEnum
;
import
com.yice.common.core.object.*
;
import
com.yice.common.core.object.*
;
import
com.yice.common.core.util.ImportUtil
;
import
com.yice.common.core.util.MyCommonUtil
;
import
com.yice.common.core.util.MyCommonUtil
;
import
com.yice.common.core.util.MyModelUtil
;
import
com.yice.common.core.util.MyModelUtil
;
import
com.yice.common.core.util.MyPageUtil
;
import
com.yice.common.core.util.MyPageUtil
;
import
com.yice.common.log.annotation.OperationLog
;
import
com.yice.common.log.annotation.OperationLog
;
import
com.yice.common.log.model.constant.SysOperationLogType
;
import
com.yice.common.log.model.constant.SysOperationLogType
;
import
com.yice.webadmin.app.config.PythonConfig
;
import
com.yice.webadmin.app.dto.DatasetDetailDto
;
import
com.yice.webadmin.app.dto.DatasetDetailDto
;
import
com.yice.webadmin.app.dto.DatasetVersionDto
;
import
com.yice.webadmin.app.dto.DatasetVersionDto
;
import
com.yice.webadmin.app.model.DatasetDetail
;
import
com.yice.webadmin.app.model.DatasetDetail
;
import
com.yice.webadmin.app.model.DatasetManage
;
import
com.yice.webadmin.app.model.DatasetVersion
;
import
com.yice.webadmin.app.model.DatasetVersion
;
import
com.yice.webadmin.app.service.DatasetManageService
;
import
com.yice.webadmin.app.service.DatasetVersionService
;
import
com.yice.webadmin.app.service.DatasetVersionService
;
import
com.yice.webadmin.app.util.Sha1Util
;
import
com.yice.webadmin.app.vo.DatasetVersionVo
;
import
com.yice.webadmin.app.vo.DatasetVersionVo
;
import
com.yice.webadmin.config.ApplicationConfig
;
import
io.swagger.annotations.Api
;
import
io.swagger.annotations.Api
;
import
lombok.extern.slf4j.Slf4j
;
import
lombok.extern.slf4j.Slf4j
;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.beans.factory.annotation.Autowired
;
...
@@ -36,6 +39,7 @@ import org.springframework.web.multipart.MultipartFile;
...
@@ -36,6 +39,7 @@ import org.springframework.web.multipart.MultipartFile;
import
java.io.File
;
import
java.io.File
;
import
java.io.IOException
;
import
java.io.IOException
;
import
java.nio.charset.StandardCharsets
;
import
java.nio.charset.StandardCharsets
;
import
java.nio.file.Files
;
import
java.nio.file.Path
;
import
java.nio.file.Path
;
import
java.nio.file.Paths
;
import
java.nio.file.Paths
;
import
java.util.ArrayList
;
import
java.util.ArrayList
;
...
@@ -58,7 +62,9 @@ public class DatasetVersionController {
...
@@ -58,7 +62,9 @@ public class DatasetVersionController {
@Autowired
@Autowired
private
DatasetVersionService
datasetVersionService
;
private
DatasetVersionService
datasetVersionService
;
@Autowired
@Autowired
private
ApplicationConfig
appConfig
;
private
DatasetManageService
datasetManageService
;
@Autowired
private
PythonConfig
pythonConfig
;
/**
/**
* 新增数据集版本数据。
* 新增数据集版本数据。
...
@@ -239,8 +245,9 @@ public class DatasetVersionController {
...
@@ -239,8 +245,9 @@ public class DatasetVersionController {
return
ResponseResult
.
error
(
ErrorCodeEnum
.
ARGUMENT_NULL_EXIST
,
errorMessage
);
return
ResponseResult
.
error
(
ErrorCodeEnum
.
ARGUMENT_NULL_EXIST
,
errorMessage
);
}
}
DatasetVersion
datasetVersion
=
this
.
datasetVersionService
.
getById
(
versionId
);
DatasetVersion
datasetVersion
=
this
.
datasetVersionService
.
getById
(
versionId
);
String
subDir
=
datasetId
+
"-"
+
datasetVersion
.
getDatasetVersion
();
DatasetManage
datasetManage
=
this
.
datasetManageService
.
getById
(
datasetId
);
String
fullName
=
ImportUtil
.
saveImportFile
(
appConfig
.
getUploadFileBaseDir
(),
subDir
,
importFile
);
String
versionName
=
datasetManage
.
getDatasetName
()
+
"_V"
+
datasetVersion
.
getDatasetVersion
();
String
fullName
=
this
.
saveDatasetFile
(
importFile
,
versionName
);
datasetVersion
.
setFileUrl
(
fullName
);
datasetVersion
.
setFileUrl
(
fullName
);
datasetVersion
.
setInputStatus
(
1
);
datasetVersion
.
setInputStatus
(
1
);
datasetVersion
.
setDataVolume
(
Long
.
valueOf
(
JSON
.
parseArray
(
new
String
(
importFile
.
getBytes
(),
StandardCharsets
.
UTF_8
)).
size
()));
datasetVersion
.
setDataVolume
(
Long
.
valueOf
(
JSON
.
parseArray
(
new
String
(
importFile
.
getBytes
(),
StandardCharsets
.
UTF_8
)).
size
()));
...
@@ -248,6 +255,42 @@ public class DatasetVersionController {
...
@@ -248,6 +255,42 @@ public class DatasetVersionController {
return
ResponseResult
.
success
();
return
ResponseResult
.
success
();
}
}
/**
* 保存导入文件。
*
* @param importFile 导入的文件。
* @return 保存的本地文件名。
*/
private
String
saveDatasetFile
(
MultipartFile
importFile
,
String
versionName
)
throws
IOException
{
String
fullName
=
pythonConfig
.
getDatasetFileBaseDir
()
+
versionName
+
".json"
;
try
{
byte
[]
bytes
=
importFile
.
getBytes
();
Path
path
=
Paths
.
get
(
fullName
);
// 如果没有files文件夹,则创建
if
(!
Files
.
isWritable
(
path
))
{
Files
.
createDirectories
(
Paths
.
get
(
pythonConfig
.
getDatasetFileBaseDir
()));
}
// 文件写入指定路径
Files
.
write
(
path
,
bytes
);
}
catch
(
IOException
e
)
{
log
.
error
(
"Failed to write imported file ["
+
importFile
.
getOriginalFilename
()
+
" ]."
,
e
);
throw
e
;
}
File
file
=
new
File
(
pythonConfig
.
getDatasetFileBaseDir
(),
pythonConfig
.
getDatasetInfo
());
if
(!
file
.
exists
()){
file
.
createNewFile
();
}
ObjectMapper
objectMapper
=
new
ObjectMapper
();
JsonNode
rootNode
=
objectMapper
.
readTree
(
file
);
if
(
rootNode
==
null
||
rootNode
.
isNull
()
||
rootNode
.
size
()
<
1
)
{
rootNode
=
objectMapper
.
createObjectNode
();
}
String
fieldValue
=
"{\"file_name\":\""
+
versionName
+
"\",\"file_sha1\":\""
+
Sha1Util
.
getSha1
(
versionName
)
+
"\"}"
;
((
ObjectNode
)
rootNode
).
put
(
versionName
,
objectMapper
.
readTree
(
fieldValue
));
objectMapper
.
writeValue
(
new
File
(
pythonConfig
.
getDatasetFileBaseDir
(),
pythonConfig
.
getDatasetInfo
()),
rootNode
);
return
fullName
;
}
@GetMapping
(
"/export"
)
@GetMapping
(
"/export"
)
public
ResponseEntity
<
Resource
>
export
(
@RequestParam
Long
versionId
)
throws
IOException
{
public
ResponseEntity
<
Resource
>
export
(
@RequestParam
Long
versionId
)
throws
IOException
{
DatasetVersion
datasetVersion
=
this
.
datasetVersionService
.
getById
(
versionId
);
DatasetVersion
datasetVersion
=
this
.
datasetVersionService
.
getById
(
versionId
);
...
...
application-webadmin/src/main/java/com/yice/webadmin/app/util/Sha1Util.java
0 → 100644
View file @
f55561a9
package
com
.
yice
.
webadmin
.
app
.
util
;
import
java.security.MessageDigest
;
import
java.security.NoSuchAlgorithmException
;
public
class
Sha1Util
{
public
static
String
getSha1
(
String
fileName
)
{
try
{
MessageDigest
md
=
MessageDigest
.
getInstance
(
"SHA-1"
);
byte
[]
hash
=
md
.
digest
(
fileName
.
getBytes
());
StringBuilder
hexString
=
new
StringBuilder
();
for
(
byte
b
:
hash
)
{
String
hex
=
Integer
.
toHexString
(
0xff
&
b
);
if
(
hex
.
length
()
==
1
)
hexString
.
append
(
'0'
);
hexString
.
append
(
hex
);
}
return
hexString
.
substring
(
0
,
40
);
}
catch
(
NoSuchAlgorithmException
e
)
{
e
.
printStackTrace
();
}
return
null
;
}
}
application-webadmin/src/main/resources/application-dev.yml
View file @
f55561a9
...
@@ -51,7 +51,8 @@ application:
...
@@ -51,7 +51,8 @@ application:
# 初始化密码。
# 初始化密码。
defaultUserPassword
:
123456
defaultUserPassword
:
123456
# 缺省的文件上传根目录。
# 缺省的文件上传根目录。
uploadFileBaseDir
:
./zz-resource/upload-files/app
uploadFileBaseDir
:
/home/linking/llms/code/LLaMA-Factory-0.3.2/lmp_data/
# 跨域的IP(http://192.168.10.10:8086)白名单列表,多个IP之间逗号分隔(* 表示全部信任,空白表示禁用跨域信任)。
# 跨域的IP(http://192.168.10.10:8086)白名单列表,多个IP之间逗号分隔(* 表示全部信任,空白表示禁用跨域信任)。
credentialIpList
:
"
*"
credentialIpList
:
"
*"
# Session的用户和数据权限在Redis中的过期时间(秒)。
# Session的用户和数据权限在Redis中的过期时间(秒)。
...
@@ -59,6 +60,11 @@ application:
...
@@ -59,6 +60,11 @@ application:
# 是否排他登录。
# 是否排他登录。
excludeLogin
:
false
excludeLogin
:
false
python
:
#数据集文件基础路径
datasetFileBaseDir
:
/home/linking/llms/code/LLaMA-Factory-0.3.2/lmp_data/
#数据集配置信息
datasetInfo
:
dataset_info.json
# 这里仅仅是一个第三方配置的示例,如果没有接入斯三方系统,
# 这里仅仅是一个第三方配置的示例,如果没有接入斯三方系统,
# 这里的配置项也不会影响到系统的行为,如果觉得多余,也可以手动删除。
# 这里的配置项也不会影响到系统的行为,如果觉得多余,也可以手动删除。
third-party
:
third-party
:
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment