From 1ad2c5402b40ebb14951ca9b7597dbb23fc1bed7 Mon Sep 17 00:00:00 2001 From: LXW <1264174498@qq.com> Date: Mon, 13 Nov 2023 22:07:18 +0800 Subject: [PATCH] (feature) (common) add sys parameter setting (#377) Co-authored-by: jolunoluo --- auth/authentication/pom.xml | 13 +---- chat/core/pom.xml | 27 ---------- .../service/impl/SysParameterServiceImpl.java | 49 +++++++++++++++++++ chat/knowledge/pom.xml | 5 -- common/pom.xml | 21 +++++++- .../dataobject/SysParameterDO.java | 18 +++++++ .../mapper/SysParameterMapper.java | 11 +++++ .../supersonic/common/pojo/Parameter.java | 16 ++++++ .../supersonic/common/pojo/SysParameter.java | 49 +++++++++++++++++++ .../common/rest/SysParameterController.java | 31 ++++++++++++ .../common/service/SysParameterService.java | 13 +++++ launchers/common/pom.xml | 15 ------ .../com/tencent/supersonic/ConfigureDemo.java | 13 +++++ .../tencent/supersonic/db/MybatisConfig.java | 7 +-- .../src/main/resources/db/schema-h2.sql | 6 +++ .../src/main/resources/db/schema-mysql.sql | 6 +++ .../src/main/resources/db/sql-update.sql | 10 +++- .../src/test/resources/db/schema-h2.sql | 6 +++ pom.xml | 15 +----- semantic/materialization/pom.xml | 4 -- semantic/model/pom.xml | 30 +++++------- semantic/query/pom.xml | 4 -- 22 files changed, 266 insertions(+), 103 deletions(-) create mode 100644 chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/SysParameterServiceImpl.java create mode 100644 common/src/main/java/com/tencent/supersonic/common/persistence/dataobject/SysParameterDO.java create mode 100644 common/src/main/java/com/tencent/supersonic/common/persistence/mapper/SysParameterMapper.java create mode 100644 common/src/main/java/com/tencent/supersonic/common/pojo/Parameter.java create mode 100644 common/src/main/java/com/tencent/supersonic/common/pojo/SysParameter.java create mode 100644 common/src/main/java/com/tencent/supersonic/common/rest/SysParameterController.java create mode 100644 common/src/main/java/com/tencent/supersonic/common/service/SysParameterService.java diff --git a/auth/authentication/pom.xml b/auth/authentication/pom.xml index 091e1f1de..d50293bda 100644 --- a/auth/authentication/pom.xml +++ b/auth/authentication/pom.xml @@ -33,12 +33,6 @@ spring-boot-starter-jdbc - - org.mybatis - mybatis - - - com.alibaba druid @@ -52,12 +46,7 @@ org.springframework.boot spring-boot-starter-web - - org.mybatis - mybatis-spring - ${mybatis-spring.version} - compile - + com.github.pagehelper pagehelper diff --git a/chat/core/pom.xml b/chat/core/pom.xml index c232f88a3..42cc3a8ad 100644 --- a/chat/core/pom.xml +++ b/chat/core/pom.xml @@ -59,16 +59,7 @@ org.springframework.boot spring-boot-starter-web - - org.mybatis - mybatis - - - org.mybatis.spring.boot - mybatis-spring-boot-starter - ${mybatis-spring.version} - com.alibaba druid @@ -78,24 +69,6 @@ mysql mysql-connector-java - - org.mybatis - mybatis-spring - test - - - - org.mybatis.spring.boot - mybatis-spring-boot-starter-test - ${mybatis.test.version} - test - - - - com.github.pagehelper - pagehelper-spring-boot-starter - ${pagehelper.spring.version} - com.h2database diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/SysParameterServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/SysParameterServiceImpl.java new file mode 100644 index 000000000..84c18acd1 --- /dev/null +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/SysParameterServiceImpl.java @@ -0,0 +1,49 @@ +package com.tencent.supersonic.chat.service.impl; + +import com.alibaba.fastjson.JSONObject; +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.tencent.supersonic.common.pojo.SysParameter; +import com.tencent.supersonic.common.persistence.dataobject.SysParameterDO; +import com.tencent.supersonic.common.persistence.mapper.SysParameterMapper; +import com.tencent.supersonic.common.service.SysParameterService; +import com.tencent.supersonic.common.util.BeanMapper; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; +import java.util.List; + +@Service +public class SysParameterServiceImpl + extends ServiceImpl implements SysParameterService { + + @Override + public SysParameter getSysParameter() { + List list = list(); + if (CollectionUtils.isEmpty(list)) { + return new SysParameter(); + } + return convert(list.iterator().next()); + } + + @Override + public void save(SysParameter sysParameter) { + SysParameterDO chatParameterDO = convert(sysParameter); + saveOrUpdate(chatParameterDO); + } + + private SysParameter convert(SysParameterDO sysParameterDO) { + SysParameter chatParameter = new SysParameter(); + BeanMapper.mapper(sysParameterDO, chatParameter); + chatParameter.setParameters(JSONObject.parseObject(sysParameterDO.getParameters(), List.class)); + chatParameter.setAdmin(sysParameterDO.getAdmin()); + return chatParameter; + } + + private SysParameterDO convert(SysParameter sysParameter) { + SysParameterDO sysParameterDO = new SysParameterDO(); + BeanMapper.mapper(sysParameter, sysParameterDO); + sysParameterDO.setParameters(JSONObject.toJSONString(sysParameter.getParameters())); + sysParameterDO.setAdmin(sysParameter.getAdmin()); + return sysParameterDO; + } + +} diff --git a/chat/knowledge/pom.xml b/chat/knowledge/pom.xml index 102b7289f..8bbf7f17b 100644 --- a/chat/knowledge/pom.xml +++ b/chat/knowledge/pom.xml @@ -20,11 +20,6 @@ lombok - - org.mybatis - mybatis - - com.hankcs hanlp diff --git a/common/pom.xml b/common/pom.xml index 3d0eca123..f092fa247 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -115,8 +115,25 @@ com.github.pagehelper - pagehelper - ${pagehelper.version} + pagehelper-spring-boot-starter + ${pagehelper.spring.version} + + + org.mybatis + mybatis + + + org.mybatis + mybatis-spring + + + + + + + com.baomidou + mybatis-plus-boot-starter + ${mybatis.plus.version} diff --git a/common/src/main/java/com/tencent/supersonic/common/persistence/dataobject/SysParameterDO.java b/common/src/main/java/com/tencent/supersonic/common/persistence/dataobject/SysParameterDO.java new file mode 100644 index 000000000..7f7ed272c --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/persistence/dataobject/SysParameterDO.java @@ -0,0 +1,18 @@ +package com.tencent.supersonic.common.persistence.dataobject; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +@Data +@TableName("s2_sys_parameter") +public class SysParameterDO { + @TableId(type = IdType.AUTO) + private Integer id; + + private String parameters; + + private String admin; + +} diff --git a/common/src/main/java/com/tencent/supersonic/common/persistence/mapper/SysParameterMapper.java b/common/src/main/java/com/tencent/supersonic/common/persistence/mapper/SysParameterMapper.java new file mode 100644 index 000000000..bb4be6c58 --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/persistence/mapper/SysParameterMapper.java @@ -0,0 +1,11 @@ +package com.tencent.supersonic.common.persistence.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.tencent.supersonic.common.persistence.dataobject.SysParameterDO; +import org.apache.ibatis.annotations.Mapper; + +@Mapper +public interface SysParameterMapper extends BaseMapper { + + +} \ No newline at end of file diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/Parameter.java b/common/src/main/java/com/tencent/supersonic/common/pojo/Parameter.java new file mode 100644 index 000000000..2a62c6f89 --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/Parameter.java @@ -0,0 +1,16 @@ +package com.tencent.supersonic.common.pojo; + + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class Parameter { + private String name; + private String value; + private String comment; + private String dataType; +} diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/SysParameter.java b/common/src/main/java/com/tencent/supersonic/common/pojo/SysParameter.java new file mode 100644 index 000000000..28429de4b --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/SysParameter.java @@ -0,0 +1,49 @@ +package com.tencent.supersonic.common.pojo; + +import com.google.common.collect.Lists; +import lombok.Data; +import org.apache.commons.lang3.StringUtils; +import org.springframework.util.CollectionUtils; +import java.util.Arrays; +import java.util.List; + +@Data +public class SysParameter { + + private Integer id; + + private List admins; + + private List parameters; + + public String getAdmin() { + if (CollectionUtils.isEmpty(admins)) { + return ""; + } + return StringUtils.join(admins, ","); + } + + public void setAdmin(String admin) { + if (StringUtils.isNotBlank(admin)) { + admins = Arrays.asList(admin.split(",")); + } else { + admins = Lists.newArrayList(); + } + } + + public void init() { + parameters = Lists.newArrayList(); + admins = Lists.newArrayList("admin"); + parameters.add(new Parameter("one.detection.size", "8", "一次探测个数", "number")); + parameters.add(new Parameter("one.detection.max.size", "20", "阈值", "number")); + parameters.add(new Parameter("metric.dimension.min.threshold", "0.3", "指标名、维度名最小文本相似度", "number")); + parameters.add(new Parameter("metric.dimension.threshold", "0.3", "指标名、维度名文本相似度", "number")); + parameters.add(new Parameter("dimension.value.threshold", "0.5", "维度值最小文本相似度", "number")); + parameters.add(new Parameter("embedding.mapper.word.min", "0.3", "用于向量召回最小的文本长度", "number")); + parameters.add(new Parameter("embedding.mapper.word.max", "0.3", "用于向量召回最大的文本长度", "number")); + parameters.add(new Parameter("embedding.mapper.batch", "0.3", "批量向量召回文本请求个数", "number")); + parameters.add(new Parameter("embedding.mapper.number", "0.3", "批量向量召回文本返回结果个数", "number")); + parameters.add(new Parameter("embedding.mapper.distance.threshold", "0.3", "Mapper阶段向量召回相似度阈值", "number")); + } + +} diff --git a/common/src/main/java/com/tencent/supersonic/common/rest/SysParameterController.java b/common/src/main/java/com/tencent/supersonic/common/rest/SysParameterController.java new file mode 100644 index 000000000..f6b12fab0 --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/rest/SysParameterController.java @@ -0,0 +1,31 @@ +package com.tencent.supersonic.common.rest; + +import com.tencent.supersonic.common.pojo.SysParameter; +import com.tencent.supersonic.common.service.SysParameterService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping({"/api/semantic/parameter"}) +public class SysParameterController { + + @Autowired + private SysParameterService sysParameterService; + + @PostMapping + public Boolean save(@RequestBody SysParameter sysParameter) { + sysParameterService.save(sysParameter); + return true; + } + + @GetMapping + SysParameter get() { + return sysParameterService.getSysParameter(); + } + + +} diff --git a/common/src/main/java/com/tencent/supersonic/common/service/SysParameterService.java b/common/src/main/java/com/tencent/supersonic/common/service/SysParameterService.java new file mode 100644 index 000000000..1c074f3c2 --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/service/SysParameterService.java @@ -0,0 +1,13 @@ +package com.tencent.supersonic.common.service; + +import com.baomidou.mybatisplus.extension.service.IService; +import com.tencent.supersonic.common.persistence.dataobject.SysParameterDO; +import com.tencent.supersonic.common.pojo.SysParameter; + +public interface SysParameterService extends IService { + + SysParameter getSysParameter(); + + void save(SysParameter sysParameter); + +} diff --git a/launchers/common/pom.xml b/launchers/common/pom.xml index 0863ac9c5..f0f3644d1 100644 --- a/launchers/common/pom.xml +++ b/launchers/common/pom.xml @@ -39,21 +39,6 @@ org.aspectj aspectjweaver - - - org.mybatis - mybatis - - - - org.mybatis - mybatis-spring - - - - com.github.pagehelper - pagehelper - org.projectlombok lombok diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java b/launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java index 1f3ca40da..80a8f93f3 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java @@ -8,6 +8,7 @@ import com.tencent.supersonic.chat.agent.AgentConfig; import com.tencent.supersonic.chat.agent.tool.AgentToolType; import com.tencent.supersonic.chat.agent.tool.LLMParserTool; import com.tencent.supersonic.chat.agent.tool.RuleQueryTool; +import com.tencent.supersonic.common.pojo.SysParameter; import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq; import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq; import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq; @@ -23,6 +24,7 @@ import com.tencent.supersonic.chat.plugin.PluginParseConfig; import com.tencent.supersonic.chat.query.plugin.ParamOption; import com.tencent.supersonic.chat.query.plugin.WebBase; import com.tencent.supersonic.chat.service.AgentService; +import com.tencent.supersonic.common.service.SysParameterService; import com.tencent.supersonic.chat.service.ChatService; import com.tencent.supersonic.chat.service.ConfigService; import com.tencent.supersonic.chat.service.PluginService; @@ -55,6 +57,9 @@ public class ConfigureDemo implements ApplicationListener private PluginService pluginService; @Autowired private AgentService agentService; + @Autowired + private SysParameterService sysParameterService; + @Value("${spring.h2.demo.enabled:false}") private boolean demoEnable; @@ -94,6 +99,13 @@ public class ConfigureDemo implements ApplicationListener parseAndExecute(2, "访问次数最高的部门"); } + public void addChatParameter() { + SysParameter chatParameter = new SysParameter(); + chatParameter.setId(1); + chatParameter.init(); + sysParameterService.save(chatParameter); + } + public void addDemoChatConfig_1() { ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq(); chatConfigBaseReq.setModelId(1L); @@ -294,6 +306,7 @@ public class ConfigureDemo implements ApplicationListener return; } try { + addChatParameter(); addDemoChatConfig_1(); addDemoChatConfig_2(); addPlugin_1(); diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/db/MybatisConfig.java b/launchers/standalone/src/main/java/com/tencent/supersonic/db/MybatisConfig.java index f3428741a..7362fb763 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/db/MybatisConfig.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/db/MybatisConfig.java @@ -1,8 +1,9 @@ package com.tencent.supersonic.db; +import com.baomidou.mybatisplus.core.MybatisConfiguration; +import com.baomidou.mybatisplus.extension.spring.MybatisSqlSessionFactoryBean; import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.session.SqlSessionFactory; -import org.mybatis.spring.SqlSessionFactoryBean; import org.mybatis.spring.annotation.MapperScan; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -19,8 +20,8 @@ public class MybatisConfig { @Bean public SqlSessionFactory sqlSessionFactory(DataSource dataSource) throws Exception { - SqlSessionFactoryBean bean = new SqlSessionFactoryBean(); - org.apache.ibatis.session.Configuration configuration = new org.apache.ibatis.session.Configuration(); + MybatisSqlSessionFactoryBean bean = new MybatisSqlSessionFactoryBean(); + MybatisConfiguration configuration = new MybatisConfiguration(); configuration.setMapUnderscoreToCamelCase(true); bean.setConfiguration(configuration); bean.setDataSource(dataSource); diff --git a/launchers/standalone/src/main/resources/db/schema-h2.sql b/launchers/standalone/src/main/resources/db/schema-h2.sql index f1bfebd9e..75cb40629 100644 --- a/launchers/standalone/src/main/resources/db/schema-h2.sql +++ b/launchers/standalone/src/main/resources/db/schema-h2.sql @@ -527,3 +527,9 @@ CREATE TABLE s2_materialization_record PRIMARY KEY (`id`) ); +CREATE TABLE s2_sys_parameter +( + id INT PRIMARY KEY AUTO_INCREMENT, + admin varchar(500), + parameters text null +); \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/db/schema-mysql.sql b/launchers/standalone/src/main/resources/db/schema-mysql.sql index 8603144a3..c3c140691 100644 --- a/launchers/standalone/src/main/resources/db/schema-mysql.sql +++ b/launchers/standalone/src/main/resources/db/schema-mysql.sql @@ -445,3 +445,9 @@ CREATE TABLE `s2_materialization_record` UNIQUE KEY `uq_id` (`materialization_id`,`element_type`,`element_id`,`data_time`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; +CREATE TABLE s2_sys_parameter +( + id int primary key AUTO_INCREMENT COMMENT '主键id', + admin varchar(500) COMMENT '系统管理员', + parameters text null COMMENT '配置项' +); \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/db/sql-update.sql b/launchers/standalone/src/main/resources/db/sql-update.sql index 66e12ef6a..ec53ed8e1 100644 --- a/launchers/standalone/src/main/resources/db/sql-update.sql +++ b/launchers/standalone/src/main/resources/db/sql-update.sql @@ -79,4 +79,12 @@ update s2_metric set status = 1; update s2_dimension set status = 1; --20231110 -UPDATE `s2_agent` SET `config` = replace (`config`,'LLM_S2QL','LLM_S2SQL') WHERE `config` LIKE '%LLM_S2QL%'; \ No newline at end of file +UPDATE `s2_agent` SET `config` = replace (`config`,'LLM_S2QL','LLM_S2SQL') WHERE `config` LIKE '%LLM_S2QL%'; + +--20231113 +CREATE TABLE s2_sys_parameter +( + id int primary key AUTO_INCREMENT COMMENT '主键id', + admin varchar(500) COMMENT '系统管理员', + parameters text null COMMENT '配置项' +); \ No newline at end of file diff --git a/launchers/standalone/src/test/resources/db/schema-h2.sql b/launchers/standalone/src/test/resources/db/schema-h2.sql index 7cb42165a..0fac6e562 100644 --- a/launchers/standalone/src/test/resources/db/schema-h2.sql +++ b/launchers/standalone/src/test/resources/db/schema-h2.sql @@ -467,6 +467,12 @@ CREATE TABLE s2_materialization_record ); +CREATE TABLE s2_sys_parameter +( + id INT PRIMARY KEY AUTO_INCREMENT, + admin varchar(500), + parameters text null +); diff --git a/pom.xml b/pom.xml index 38f89909f..c1b1cb7ac 100644 --- a/pom.xml +++ b/pom.xml @@ -48,8 +48,7 @@ 0.9.1 1.2.3 5.1.46 - 2.3.1 - 1.3.2 + 3.1.2 0.4.6 1.4.200 2.0.24 @@ -82,17 +81,7 @@ pagehelper ${pagehelper.version} - - - org.mybatis - mybatis - ${mybatis.version} - - - org.mybatis - mybatis-spring - ${mybatis-spring.version} - + org.projectlombok diff --git a/semantic/materialization/pom.xml b/semantic/materialization/pom.xml index 17bd618d5..81f58fbed 100644 --- a/semantic/materialization/pom.xml +++ b/semantic/materialization/pom.xml @@ -17,10 +17,6 @@ ${project.version} compile - - org.mybatis - mybatis - commons-beanutils commons-beanutils diff --git a/semantic/model/pom.xml b/semantic/model/pom.xml index 254ed4aa4..a2b83eb14 100644 --- a/semantic/model/pom.xml +++ b/semantic/model/pom.xml @@ -28,6 +28,18 @@ spring-context + + com.google.guava + guava + ${guava.version} + + + + + org.apache.commons + commons-lang3 + ${commons.lang3.version} + com.tencent.supersonic semantic-api @@ -48,11 +60,6 @@ spring-boot-starter-jdbc - - org.mybatis - mybatis - - com.alibaba druid @@ -66,18 +73,7 @@ org.springframework.boot spring-boot-starter-web - - org.mybatis - mybatis-spring - ${mybatis-spring.version} - compile - - - com.github.pagehelper - pagehelper - ${pagehelper.version} - compile - + org.testng testng diff --git a/semantic/query/pom.xml b/semantic/query/pom.xml index e451828d4..95e933433 100644 --- a/semantic/query/pom.xml +++ b/semantic/query/pom.xml @@ -50,10 +50,6 @@ org.aspectj aspectjweaver - - org.mybatis - mybatis - org.springframework.boot spring-boot-starter-web