Commit 31912f95 authored by linpeiqin's avatar linpeiqin

增加知识库接口

parent c24eb727
package com.yice.webadmin.app.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
@Data
@Configuration
@ConfigurationProperties(prefix = "knowledge")
public class KnowledgeConfig {
/**
* 知识库通用接口地址
*/
private String knowledgeInterface;
/**
* 创建知识库
*/
private String create;
/**
* 获取知识库列表
*/
private String list;
/**
* 获取知识库列表
*/
private String delete;
/**
* 获取知识库内的文件列表
*/
private String listFiles;
/**
* 搜索知识库
*/
private String searchDocs;
/**
* 上传文件到知识库,并/或进行向量化
*/
private String uploadDocs;
/**
* 删除知识库内指定文件
*/
private String deleteDocs;
/**
* 更新现有文件到知识库
*/
private String updateDocs;
/**
* 下载对应的知识文件
*/
private String downloadDocs;
/**
* 根据content中文档重建向量库,流式输出处理进度
*/
private String recreate;
}
......@@ -22,7 +22,6 @@ public class PythonConfig {
* 数据集配置文件
*/
private String datasetInfo;
/**
* 数据集配置目录
*/
......@@ -31,4 +30,5 @@ public class PythonConfig {
* python平台通用接口地址
*/
private String factoryInterface;
}
......@@ -6,6 +6,7 @@ import com.yice.common.core.object.ResponseResult;
import com.yice.common.log.annotation.OperationLog;
import com.yice.common.log.model.constant.SysOperationLogType;
import com.yice.webadmin.app.config.PythonConfig;
import com.yice.webadmin.app.service.ProxyPythonService;
import io.swagger.annotations.Api;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpEntity;
......@@ -34,6 +35,8 @@ import java.io.IOException;
public class ProxyController {
@Autowired
private PythonConfig pythonConfig;
@Autowired
private ProxyPythonService proxyPythonService;
private final WebClient webClient;
public ProxyController(WebClient webClient) {
......@@ -54,28 +57,6 @@ public class ProxyController {
@OperationLog(type = SysOperationLogType.OTHER)
@PostMapping("/predict")
public ResponseResult<String> predict(@RequestBody String requestBody) throws IOException {
CloseableHttpClient httpClient = HttpClients.createDefault();
try {
HttpPost httpPost = new HttpPost(this.pythonConfig.getFactoryInterface());
httpPost.setHeader("Content-Type", "application/json");
httpPost.setEntity(new StringEntity(requestBody, "UTF-8"));
CloseableHttpResponse response = httpClient.execute(httpPost);
try {
HttpEntity entity = response.getEntity();
if (entity != null) {
String result = EntityUtils.toString(entity);
return ResponseResult.success(result);
}
} finally {
response.close();
}
} catch (ClientProtocolException e) {
throw new RuntimeException(e);
} catch (IOException e) {
throw new RuntimeException(e);
} finally {
httpClient.close();
}
return null;
return ResponseResult.success(this.proxyPythonService.predictPost(pythonConfig.getFactoryInterface(),requestBody));
}
}
package com.yice.webadmin.app.dao;
import com.yice.common.core.base.dao.BaseDaoMapper;
import com.yice.webadmin.app.model.KnowledgeManage;
import org.apache.ibatis.annotations.Param;
import java.util.*;
/**
* 知识库管理数据操作访问接口。
*
* @author linking
* @date 2023-04-13
*/
public interface KnowledgeManageMapper extends BaseDaoMapper<KnowledgeManage> {
/**
* 批量插入对象列表。
*
* @param knowledgeManageList 新增对象列表。
*/
void insertList(List<KnowledgeManage> knowledgeManageList);
/**
* 获取过滤后的对象列表。
*
* @param knowledgeManageFilter 主表过滤对象。
* @param orderBy 排序字符串,order by从句的参数。
* @return 对象列表。
*/
List<KnowledgeManage> getKnowledgeManageList(
@Param("knowledgeManageFilter") KnowledgeManage knowledgeManageFilter, @Param("orderBy") String orderBy);
}
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.yice.webadmin.app.dao.KnowledgeManageMapper">
<resultMap id="BaseResultMap" type="com.yice.webadmin.app.model.KnowledgeManage">
<id column="knowledge_id" jdbcType="BIGINT" property="knowledgeId"/>
<result column="create_user_id" jdbcType="BIGINT" property="createUserId"/>
<result column="create_time" jdbcType="TIMESTAMP" property="createTime"/>
<result column="update_user_id" jdbcType="BIGINT" property="updateUserId"/>
<result column="update_time" jdbcType="TIMESTAMP" property="updateTime"/>
<result column="knowledge_name" jdbcType="VARCHAR" property="knowledgeName"/>
<result column="knowledge_describe" jdbcType="VARCHAR" property="knowledgeDescribe"/>
</resultMap>
<insert id="insertList">
INSERT INTO lmp_knowledge_manage
(knowledge_id,
create_user_id,
create_time,
update_user_id,
update_time,
knowledge_name,
knowledge_describe)
VALUES
<foreach collection="list" index="index" item="item" separator="," >
(#{item.knowledgeId},
#{item.createUserId},
#{item.createTime},
#{item.updateUserId},
#{item.updateTime},
#{item.knowledgeName},
#{item.knowledgeDescribe})
</foreach>
</insert>
<!-- 如果有逻辑删除字段过滤,请写到这里 -->
<sql id="filterRef">
<!-- 这里必须加上全包名,否则当filterRef被其他Mapper.xml包含引用的时候,就会调用Mapper.xml中的该SQL片段 -->
<include refid="com.yice.webadmin.app.dao.KnowledgeManageMapper.inputFilterRef"/>
</sql>
<!-- 这里仅包含调用接口输入的主表过滤条件 -->
<sql id="inputFilterRef">
<if test="knowledgeManageFilter != null">
<if test="knowledgeManageFilter.knowledgeId != null">
AND lmp_knowledge_manage.knowledge_id = #{knowledgeManageFilter.knowledgeId}
</if>
<if test="knowledgeManageFilter.createUserId != null">
AND lmp_knowledge_manage.create_user_id = #{knowledgeManageFilter.createUserId}
</if>
<if test="knowledgeManageFilter.knowledgeName != null">
AND lmp_knowledge_manage.knowledge_name = #{knowledgeManageFilter.knowledgeName}
</if>
<if test="knowledgeManageFilter.searchString != null and knowledgeManageFilter.searchString != ''">
<bind name = "safeKnowledgeManageSearchString" value = "'%' + knowledgeManageFilter.searchString + '%'" />
AND IFNULL(lmp_knowledge_manage.knowledge_name,'') LIKE #{safeKnowledgeManageSearchString}
</if>
</if>
</sql>
<select id="getKnowledgeManageList" resultMap="BaseResultMap" parameterType="com.yice.webadmin.app.model.KnowledgeManage">
SELECT * FROM lmp_knowledge_manage
<where>
<include refid="filterRef"/>
</where>
<if test="orderBy != null and orderBy != ''">
ORDER BY ${orderBy}
</if>
</select>
</mapper>
package com.yice.webadmin.app.dto;
import com.yice.common.core.validator.UpdateGroup;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import javax.validation.constraints.*;
/**
* KnowledgeManageDto对象。
*
* @author linking
* @date 2023-04-13
*/
@ApiModel("KnowledgeManageDto对象")
@Data
public class KnowledgeManageDto {
/**
* 知识库ID。
*/
@ApiModelProperty(value = "知识库ID", required = true)
@NotNull(message = "数据验证失败,知识库ID不能为空!", groups = {UpdateGroup.class})
private Long knowledgeId;
/**
* 知识库名称。
*/
@ApiModelProperty(value = "知识库名称", required = true)
@NotBlank(message = "数据验证失败,知识库名称不能为空!")
private String knowledgeName;
/**
* 知识库描述。
*/
@ApiModelProperty(value = "知识库描述")
private String knowledgeDescribe;
/**
* knowledge_name LIKE搜索字符串。
*/
@ApiModelProperty(value = "LIKE模糊搜索字符串")
private String searchString;
}
package com.yice.webadmin.app.model;
import com.baomidou.mybatisplus.annotation.*;
import com.yice.common.core.util.MyCommonUtil;
import com.yice.common.core.base.model.BaseModel;
import com.yice.common.core.base.mapper.BaseModelMapper;
import com.yice.webadmin.app.vo.KnowledgeManageVo;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.mapstruct.*;
import org.mapstruct.factory.Mappers;
/**
* KnowledgeManage实体对象。
*
* @author linking
* @date 2023-04-13
*/
@Data
@EqualsAndHashCode(callSuper = true)
@TableName(value = "lmp_knowledge_manage")
public class KnowledgeManage extends BaseModel {
/**
* 知识库ID。
*/
@TableId(value = "knowledge_id")
private Long knowledgeId;
/**
* 知识库名称。
*/
private String knowledgeName;
/**
* 知识库描述。
*/
private String knowledgeDescribe;
/**
* knowledge_name LIKE搜索字符串。
*/
@TableField(exist = false)
private String searchString;
public void setSearchString(String searchString) {
this.searchString = MyCommonUtil.replaceSqlWildcard(searchString);
}
@Mapper
public interface KnowledgeManageModelMapper extends BaseModelMapper<KnowledgeManageVo, KnowledgeManage> {
}
public static final KnowledgeManageModelMapper INSTANCE = Mappers.getMapper(KnowledgeManageModelMapper.class);
}
package com.yice.webadmin.app.service;
import com.yice.webadmin.app.model.*;
import com.yice.common.core.base.service.IBaseService;
import java.util.*;
/**
* 知识库管理数据操作服务接口。
*
* @author linking
* @date 2023-04-13
*/
public interface KnowledgeManageService extends IBaseService<KnowledgeManage, Long> {
/**
* 保存新增对象。
*
* @param knowledgeManage 新增对象。
* @return 返回新增对象。
*/
KnowledgeManage saveNew(KnowledgeManage knowledgeManage);
/**
* 利用数据库的insertList语法,批量插入对象列表。
*
* @param knowledgeManageList 新增对象列表。
*/
void saveNewBatch(List<KnowledgeManage> knowledgeManageList);
/**
* 更新数据对象。
*
* @param knowledgeManage 更新的对象。
* @param originalKnowledgeManage 原有数据对象。
* @return 成功返回true,否则false。
*/
boolean update(KnowledgeManage knowledgeManage, KnowledgeManage originalKnowledgeManage);
/**
* 删除指定数据。
*
* @param knowledgeId 主键Id。
* @return 成功返回true,否则false。
*/
boolean remove(Long knowledgeId);
/**
* 获取单表查询结果。由于没有关联数据查询,因此在仅仅获取单表数据的场景下,效率更高。
* 如果需要同时获取关联数据,请移步(getKnowledgeManageListWithRelation)方法。
*
* @param filter 过滤对象。
* @param orderBy 排序参数。
* @return 查询结果集。
*/
List<KnowledgeManage> getKnowledgeManageList(KnowledgeManage filter, String orderBy);
/**
* 获取主表的查询结果,以及主表关联的字典数据和一对一从表数据,以及一对一从表的字典数据。
* 该查询会涉及到一对一从表的关联过滤,或一对多从表的嵌套关联过滤,因此性能不如单表过滤。
* 如果仅仅需要获取主表数据,请移步(getKnowledgeManageList),以便获取更好的查询性能。
*
* @param filter 主表过滤对象。
* @param orderBy 排序参数。
* @return 查询结果集。
*/
List<KnowledgeManage> getKnowledgeManageListWithRelation(KnowledgeManage filter, String orderBy);
}
package com.yice.webadmin.app.service;
import java.io.IOException;
/**
* python代理服务接口。
*
* @author linking
* @date 2023-04-13
*/
public interface ProxyPythonService{
public String predictPost(String url, String requestBody) throws IOException;
public String predictGet(String url, String requestBody) throws IOException;
public String predictPostForFile(String url, String requestBody) throws IOException;
}
package com.yice.webadmin.app.service.impl;
import cn.hutool.core.collection.CollUtil;
import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper;
import com.github.pagehelper.Page;
import com.yice.common.core.base.dao.BaseDaoMapper;
import com.yice.common.core.base.service.BaseService;
import com.yice.common.core.object.MyRelationParam;
import com.yice.common.core.util.MyModelUtil;
import com.yice.common.sequence.wrapper.IdGeneratorWrapper;
import com.yice.webadmin.app.config.KnowledgeConfig;
import com.yice.webadmin.app.dao.KnowledgeManageMapper;
import com.yice.webadmin.app.model.KnowledgeManage;
import com.yice.webadmin.app.service.KnowledgeManageService;
import com.yice.webadmin.app.service.ProxyPythonService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.io.IOException;
import java.util.List;
/**
* 知识库管理数据操作服务类。
*
* @author linking
* @date 2023-04-13
*/
@Slf4j
@Service("knowledgeManageService")
public class KnowledgeManageServiceImpl extends BaseService<KnowledgeManage, Long> implements KnowledgeManageService {
@Autowired
private KnowledgeManageMapper knowledgeManageMapper;
@Autowired
private IdGeneratorWrapper idGenerator;
/**
* 返回当前Service的主表Mapper对象。
*
* @return 主表Mapper对象。
*/
@Override
protected BaseDaoMapper<KnowledgeManage> mapper() {
return knowledgeManageMapper;
}
/**
* 保存新增对象。
*
* @param knowledgeManage 新增对象。
* @return 返回新增对象。
*/
@Transactional(rollbackFor = Exception.class)
@Override
public KnowledgeManage saveNew(KnowledgeManage knowledgeManage) {
knowledgeManageMapper.insert(this.buildDefaultValue(knowledgeManage));
return knowledgeManage;
}
/**
* 利用数据库的insertList语法,批量插入对象列表。
*
* @param knowledgeManageList 新增对象列表。
*/
@Transactional(rollbackFor = Exception.class)
@Override
public void saveNewBatch(List<KnowledgeManage> knowledgeManageList) {
if (CollUtil.isNotEmpty(knowledgeManageList)) {
knowledgeManageList.forEach(this::buildDefaultValue);
knowledgeManageMapper.insertList(knowledgeManageList);
}
}
/**
* 更新数据对象。
*
* @param knowledgeManage 更新的对象。
* @param originalKnowledgeManage 原有数据对象。
* @return 成功返回true,否则false。
*/
@Transactional(rollbackFor = Exception.class)
@Override
public boolean update(KnowledgeManage knowledgeManage, KnowledgeManage originalKnowledgeManage) {
MyModelUtil.fillCommonsForUpdate(knowledgeManage, originalKnowledgeManage);
// 这里重点提示,在执行主表数据更新之前,如果有哪些字段不支持修改操作,请用原有数据对象字段替换当前数据字段。
UpdateWrapper<KnowledgeManage> uw = this.createUpdateQueryForNullValue(knowledgeManage, knowledgeManage.getKnowledgeId());
return knowledgeManageMapper.update(knowledgeManage, uw) == 1;
}
/**
* 删除指定数据。
*
* @param knowledgeId 主键Id。
* @return 成功返回true,否则false。
*/
@Transactional(rollbackFor = Exception.class)
@Override
public boolean remove(Long knowledgeId) {
return knowledgeManageMapper.deleteById(knowledgeId) == 1;
}
/**
* 获取单表查询结果。由于没有关联数据查询,因此在仅仅获取单表数据的场景下,效率更高。
* 如果需要同时获取关联数据,请移步(getKnowledgeManageListWithRelation)方法。
*
* @param filter 过滤对象。
* @param orderBy 排序参数。
* @return 查询结果集。
*/
@Override
public List<KnowledgeManage> getKnowledgeManageList(KnowledgeManage filter, String orderBy) {
return knowledgeManageMapper.getKnowledgeManageList(filter, orderBy);
}
/**
* 获取主表的查询结果,以及主表关联的字典数据和一对一从表数据,以及一对一从表的字典数据。
* 该查询会涉及到一对一从表的关联过滤,或一对多从表的嵌套关联过滤,因此性能不如单表过滤。
* 如果仅仅需要获取主表数据,请移步(getKnowledgeManageList),以便获取更好的查询性能。
*
* @param filter 主表过滤对象。
* @param orderBy 排序参数。
* @return 查询结果集。
*/
@Override
public List<KnowledgeManage> getKnowledgeManageListWithRelation(KnowledgeManage filter, String orderBy) {
List<KnowledgeManage> resultList = knowledgeManageMapper.getKnowledgeManageList(filter, orderBy);
// 在缺省生成的代码中,如果查询结果resultList不是Page对象,说明没有分页,那么就很可能是数据导出接口调用了当前方法。
// 为了避免一次性的大量数据关联,规避因此而造成的系统运行性能冲击,这里手动进行了分批次读取,开发者可按需修改该值。
int batchSize = resultList instanceof Page ? 0 : 1000;
this.buildRelationForDataList(resultList, MyRelationParam.normal(), batchSize);
return resultList;
}
private KnowledgeManage buildDefaultValue(KnowledgeManage knowledgeManage) {
if (knowledgeManage.getKnowledgeId() == null) {
knowledgeManage.setKnowledgeId(idGenerator.nextLongId());
}
MyModelUtil.fillCommonsForInsert(knowledgeManage);
return knowledgeManage;
}
}
package com.yice.webadmin.app.service.impl;
import com.yice.webadmin.app.config.PythonConfig;
import com.yice.webadmin.app.service.ProxyPythonService;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpEntity;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.io.IOException;
/**
* python代理服务接口类。
*
* @author linking
* @date 2023-04-13
*/
@Slf4j
@Service("proxyPythonService")
public class ProxyPythonServiceImpl implements ProxyPythonService {
public String predictPost(String url, String requestBody) throws IOException {
CloseableHttpClient httpClient = HttpClients.createDefault();
try {
HttpPost httpPost = new HttpPost(url);
httpPost.setHeader("Content-Type", "application/json");
httpPost.setEntity(new StringEntity(requestBody, "UTF-8"));
CloseableHttpResponse response = httpClient.execute(httpPost);
try {
HttpEntity entity = response.getEntity();
if (entity != null) {
return EntityUtils.toString(entity);
}
} finally {
response.close();
}
} catch (ClientProtocolException e) {
throw new RuntimeException(e);
} catch (IOException e) {
throw new RuntimeException(e);
} finally {
httpClient.close();
}
return null;
}
public String predictPostForFile(String url, String requestBody) throws IOException {
CloseableHttpClient httpClient = HttpClients.createDefault();
try {
HttpPost httpPost = new HttpPost(url);
httpPost.setHeader("accept", "application/json");
httpPost.setHeader("Content-Type", "multipart/form-data");
httpPost.setEntity(new StringEntity(requestBody, "UTF-8"));
CloseableHttpResponse response = httpClient.execute(httpPost);
try {
HttpEntity entity = response.getEntity();
if (entity != null) {
return EntityUtils.toString(entity);
}
} finally {
response.close();
}
} catch (ClientProtocolException e) {
throw new RuntimeException(e);
} catch (IOException e) {
throw new RuntimeException(e);
} finally {
httpClient.close();
}
return null;
}
public String predictGet(String url, String requestBody) throws IOException {
CloseableHttpClient httpClient = HttpClients.createDefault();
try {
HttpGet httpGet = new HttpGet(url + requestBody);
CloseableHttpResponse response = httpClient.execute(httpGet);
try {
HttpEntity entity = response.getEntity();
if (entity != null) {
return EntityUtils.toString(entity);
}
} finally {
response.close();
}
} finally {
httpClient.close();
}
return null;
}
}
package com.yice.webadmin.app.vo;
import com.yice.common.core.base.vo.BaseVo;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import lombok.EqualsAndHashCode;
import java.util.Date;
/**
* KnowledgeManageVO视图对象。
*
* @author linking
* @date 2023-04-13
*/
@ApiModel("KnowledgeManageVO视图对象")
@Data
@EqualsAndHashCode(callSuper = true)
public class KnowledgeManageVo extends BaseVo {
/**
* 知识库ID。
*/
@ApiModelProperty(value = "知识库ID")
private Long knowledgeId;
/**
* 知识库名称。
*/
@ApiModelProperty(value = "知识库名称")
private String knowledgeName;
/**
* 知识库描述。
*/
@ApiModelProperty(value = "知识库描述")
private String knowledgeDescribe;
}
......@@ -69,6 +69,32 @@ python:
datasetFileMenu: lmp_data
#python平台通用接口地址
factoryInterface: http://192.168.0.36:7860/run/predict
knowledge:
#知识库通用接口地址
knowledgeInterface: http://192.168.0.36:7861/knowledge_base/
#创建知识库
create: create_knowledge_base
#获取知识库列表
list: list_knowledge_bases
#获取知识库列表
delete: delete_knowledge_base
#获取知识库内的文件列表
listFiles: list_files
#搜索知识库
searchDocs: search_docs
#上传文件到知识库,bing/或进行向量化
uploadDocs: upload_docs
#删除知识库内指定文件
deleteDocs: delete_docs
#更新现有文件到知识库
updateDocs: update_docs
#下载对应的知识文件
downloadDocs: download_docs
#根据content中文档重建向量库,流式输出处理进度
recreate: recreate_vector_store
#获取知识库内的文件列表
# 这里仅仅是一个第三方配置的示例,如果没有接入斯三方系统,
# 这里的配置项也不会影响到系统的行为,如果觉得多余,也可以手动删除。
third-party:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment