(feature) (common) add sys parameter setting (#377)

Co-authored-by: jolunoluo

auth/authentication/pom.xml

@@ -33,12 +33,6 @@
             <artifactId>spring-boot-starter-jdbc</artifactId>
         </dependency>
 
-        <dependency>
-            <groupId>org.mybatis</groupId>
-            <artifactId>mybatis</artifactId>
-        </dependency>
-
-
         <dependency>
             <groupId>com.alibaba</groupId>
             <artifactId>druid</artifactId>
@@ -52,12 +46,7 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-web</artifactId>
         </dependency>
-        <dependency>
+
-            <groupId>org.mybatis</groupId>
-            <artifactId>mybatis-spring</artifactId>
-            <version>${mybatis-spring.version}</version>
-            <scope>compile</scope>
-        </dependency>
         <dependency>
             <groupId>com.github.pagehelper</groupId>
             <artifactId>pagehelper</artifactId>

chat/core/pom.xml

@@ -59,16 +59,7 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-web</artifactId>
         </dependency>
-        <dependency>
-            <groupId>org.mybatis</groupId>
-            <artifactId>mybatis</artifactId>
-        </dependency>
 
-        <dependency>
-            <groupId>org.mybatis.spring.boot</groupId>
-            <artifactId>mybatis-spring-boot-starter</artifactId>
-            <version>${mybatis-spring.version}</version>
-        </dependency>
         <dependency>
             <groupId>com.alibaba</groupId>
             <artifactId>druid</artifactId>
@@ -78,24 +69,6 @@
             <groupId>mysql</groupId>
             <artifactId>mysql-connector-java</artifactId>
         </dependency>
-        <dependency>
-            <groupId>org.mybatis</groupId>
-            <artifactId>mybatis-spring</artifactId>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.mybatis.spring.boot</groupId>
-            <artifactId>mybatis-spring-boot-starter-test</artifactId>
-            <version>${mybatis.test.version}</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>com.github.pagehelper</groupId>
-            <artifactId>pagehelper-spring-boot-starter</artifactId>
-            <version>${pagehelper.spring.version}</version>
-        </dependency>
 
         <dependency>
             <groupId>com.h2database</groupId>

chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/SysParameterServiceImpl.java

@@ -0,0 +1,49 @@
+package com.tencent.supersonic.chat.service.impl;
+
+import com.alibaba.fastjson.JSONObject;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.tencent.supersonic.common.pojo.SysParameter;
+import com.tencent.supersonic.common.persistence.dataobject.SysParameterDO;
+import com.tencent.supersonic.common.persistence.mapper.SysParameterMapper;
+import com.tencent.supersonic.common.service.SysParameterService;
+import com.tencent.supersonic.common.util.BeanMapper;
+import org.springframework.stereotype.Service;
+import org.springframework.util.CollectionUtils;
+import java.util.List;
+
+@Service
+public class SysParameterServiceImpl
+        extends ServiceImpl<SysParameterMapper, SysParameterDO> implements SysParameterService {
+
+    @Override
+    public SysParameter getSysParameter() {
+        List<SysParameterDO> list = list();
+        if (CollectionUtils.isEmpty(list)) {
+            return new SysParameter();
+        }
+        return convert(list.iterator().next());
+    }
+
+    @Override
+    public void save(SysParameter sysParameter) {
+        SysParameterDO chatParameterDO = convert(sysParameter);
+        saveOrUpdate(chatParameterDO);
+    }
+
+    private SysParameter convert(SysParameterDO sysParameterDO) {
+        SysParameter chatParameter = new SysParameter();
+        BeanMapper.mapper(sysParameterDO, chatParameter);
+        chatParameter.setParameters(JSONObject.parseObject(sysParameterDO.getParameters(), List.class));
+        chatParameter.setAdmin(sysParameterDO.getAdmin());
+        return chatParameter;
+    }
+
+    private SysParameterDO convert(SysParameter sysParameter) {
+        SysParameterDO sysParameterDO = new SysParameterDO();
+        BeanMapper.mapper(sysParameter, sysParameterDO);
+        sysParameterDO.setParameters(JSONObject.toJSONString(sysParameter.getParameters()));
+        sysParameterDO.setAdmin(sysParameter.getAdmin());
+        return sysParameterDO;
+    }
+
+}

chat/knowledge/pom.xml

@@ -20,11 +20,6 @@
             <artifactId>lombok</artifactId>
         </dependency>
 
-        <dependency>
-            <groupId>org.mybatis</groupId>
-            <artifactId>mybatis</artifactId>
-        </dependency>
-
         <dependency>
             <groupId>com.hankcs</groupId>
             <artifactId>hanlp</artifactId>

common/pom.xml

@@ -115,8 +115,25 @@
 
         <dependency>
             <groupId>com.github.pagehelper</groupId>
-            <artifactId>pagehelper</artifactId>
+            <artifactId>pagehelper-spring-boot-starter</artifactId>
-            <version>${pagehelper.version}</version>
+            <version>${pagehelper.spring.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.mybatis</groupId>
+                    <artifactId>mybatis</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.mybatis</groupId>
+                    <artifactId>mybatis-spring</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
+
+        <dependency>
+            <groupId>com.baomidou</groupId>
+            <artifactId>mybatis-plus-boot-starter</artifactId>
+            <version>${mybatis.plus.version}</version>
         </dependency>
 
         <dependency>

common/src/main/java/com/tencent/supersonic/common/persistence/dataobject/SysParameterDO.java

@@ -0,0 +1,18 @@
+package com.tencent.supersonic.common.persistence.dataobject;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import lombok.Data;
+
+@Data
+@TableName("s2_sys_parameter")
+public class SysParameterDO {
+    @TableId(type = IdType.AUTO)
+    private Integer id;
+
+    private String parameters;
+
+    private String admin;
+
+}

common/src/main/java/com/tencent/supersonic/common/persistence/mapper/SysParameterMapper.java

@@ -0,0 +1,11 @@
+package com.tencent.supersonic.common.persistence.mapper;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import com.tencent.supersonic.common.persistence.dataobject.SysParameterDO;
+import org.apache.ibatis.annotations.Mapper;
+
+@Mapper
+public interface SysParameterMapper extends BaseMapper<SysParameterDO> {
+
+
+}

common/src/main/java/com/tencent/supersonic/common/pojo/Parameter.java

@@ -0,0 +1,16 @@
+package com.tencent.supersonic.common.pojo;
+
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class Parameter {
+    private String name;
+    private String value;
+    private String comment;
+    private String dataType;
+}

common/src/main/java/com/tencent/supersonic/common/pojo/SysParameter.java

@@ -0,0 +1,49 @@
+package com.tencent.supersonic.common.pojo;
+
+import com.google.common.collect.Lists;
+import lombok.Data;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.util.CollectionUtils;
+import java.util.Arrays;
+import java.util.List;
+
+@Data
+public class SysParameter {
+
+    private Integer id;
+
+    private List<String> admins;
+
+    private List<Parameter> parameters;
+
+    public String getAdmin() {
+        if (CollectionUtils.isEmpty(admins)) {
+            return "";
+        }
+        return StringUtils.join(admins, ",");
+    }
+
+    public void setAdmin(String admin) {
+        if (StringUtils.isNotBlank(admin)) {
+            admins = Arrays.asList(admin.split(","));
+        } else {
+            admins = Lists.newArrayList();
+        }
+    }
+
+    public void init() {
+        parameters = Lists.newArrayList();
+        admins = Lists.newArrayList("admin");
+        parameters.add(new Parameter("one.detection.size", "8", "一次探测个数", "number"));
+        parameters.add(new Parameter("one.detection.max.size", "20", "阈值", "number"));
+        parameters.add(new Parameter("metric.dimension.min.threshold", "0.3", "指标名、维度名最小文本相似度", "number"));
+        parameters.add(new Parameter("metric.dimension.threshold", "0.3", "指标名、维度名文本相似度", "number"));
+        parameters.add(new Parameter("dimension.value.threshold", "0.5", "维度值最小文本相似度", "number"));
+        parameters.add(new Parameter("embedding.mapper.word.min", "0.3", "用于向量召回最小的文本长度", "number"));
+        parameters.add(new Parameter("embedding.mapper.word.max", "0.3", "用于向量召回最大的文本长度", "number"));
+        parameters.add(new Parameter("embedding.mapper.batch", "0.3", "批量向量召回文本请求个数", "number"));
+        parameters.add(new Parameter("embedding.mapper.number", "0.3", "批量向量召回文本返回结果个数", "number"));
+        parameters.add(new Parameter("embedding.mapper.distance.threshold", "0.3", "Mapper阶段向量召回相似度阈值", "number"));
+    }
+
+}

common/src/main/java/com/tencent/supersonic/common/rest/SysParameterController.java

@@ -0,0 +1,31 @@
+package com.tencent.supersonic.common.rest;
+
+import com.tencent.supersonic.common.pojo.SysParameter;
+import com.tencent.supersonic.common.service.SysParameterService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+@RequestMapping({"/api/semantic/parameter"})
+public class SysParameterController {
+
+    @Autowired
+    private SysParameterService sysParameterService;
+
+    @PostMapping
+    public Boolean save(@RequestBody SysParameter sysParameter) {
+        sysParameterService.save(sysParameter);
+        return true;
+    }
+
+    @GetMapping
+    SysParameter get() {
+        return sysParameterService.getSysParameter();
+    }
+
+
+}

common/src/main/java/com/tencent/supersonic/common/service/SysParameterService.java

@@ -0,0 +1,13 @@
+package com.tencent.supersonic.common.service;
+
+import com.baomidou.mybatisplus.extension.service.IService;
+import com.tencent.supersonic.common.persistence.dataobject.SysParameterDO;
+import com.tencent.supersonic.common.pojo.SysParameter;
+
+public interface SysParameterService extends IService<SysParameterDO> {
+
+    SysParameter getSysParameter();
+
+    void save(SysParameter sysParameter);
+
+}

launchers/common/pom.xml

@@ -39,21 +39,6 @@
             <groupId>org.aspectj</groupId>
             <artifactId>aspectjweaver</artifactId>
         </dependency>
-
-        <dependency>
-            <groupId>org.mybatis</groupId>
-            <artifactId>mybatis</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>org.mybatis</groupId>
-            <artifactId>mybatis-spring</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>com.github.pagehelper</groupId>
-            <artifactId>pagehelper</artifactId>
-        </dependency>
         <dependency>
             <groupId>org.projectlombok</groupId>
             <artifactId>lombok</artifactId>

launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java

@@ -8,6 +8,7 @@ import com.tencent.supersonic.chat.agent.AgentConfig;
 import com.tencent.supersonic.chat.agent.tool.AgentToolType;
 import com.tencent.supersonic.chat.agent.tool.LLMParserTool;
 import com.tencent.supersonic.chat.agent.tool.RuleQueryTool;
+import com.tencent.supersonic.common.pojo.SysParameter;
 import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq;
 import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq;
 import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq;
@@ -23,6 +24,7 @@ import com.tencent.supersonic.chat.plugin.PluginParseConfig;
 import com.tencent.supersonic.chat.query.plugin.ParamOption;
 import com.tencent.supersonic.chat.query.plugin.WebBase;
 import com.tencent.supersonic.chat.service.AgentService;
+import com.tencent.supersonic.common.service.SysParameterService;
 import com.tencent.supersonic.chat.service.ChatService;
 import com.tencent.supersonic.chat.service.ConfigService;
 import com.tencent.supersonic.chat.service.PluginService;
@@ -55,6 +57,9 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
     private PluginService pluginService;
     @Autowired
     private AgentService agentService;
+    @Autowired
+    private SysParameterService sysParameterService;
+
     @Value("${spring.h2.demo.enabled:false}")
     private boolean demoEnable;
 
@@ -94,6 +99,13 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
         parseAndExecute(2, "访问次数最高的部门");
     }
 
+    public void addChatParameter() {
+        SysParameter chatParameter = new SysParameter();
+        chatParameter.setId(1);
+        chatParameter.init();
+        sysParameterService.save(chatParameter);
+    }
+
     public void addDemoChatConfig_1() {
         ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq();
         chatConfigBaseReq.setModelId(1L);
@@ -294,6 +306,7 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
             return;
         }
         try {
+            addChatParameter();
             addDemoChatConfig_1();
             addDemoChatConfig_2();
             addPlugin_1();

launchers/standalone/src/main/java/com/tencent/supersonic/db/MybatisConfig.java

@@ -1,8 +1,9 @@
 package com.tencent.supersonic.db;
 
+import com.baomidou.mybatisplus.core.MybatisConfiguration;
+import com.baomidou.mybatisplus.extension.spring.MybatisSqlSessionFactoryBean;
 import org.apache.ibatis.annotations.Mapper;
 import org.apache.ibatis.session.SqlSessionFactory;
-import org.mybatis.spring.SqlSessionFactoryBean;
 import org.mybatis.spring.annotation.MapperScan;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
@@ -19,8 +20,8 @@ public class MybatisConfig {
 
     @Bean
     public SqlSessionFactory sqlSessionFactory(DataSource dataSource) throws Exception {
-        SqlSessionFactoryBean bean = new SqlSessionFactoryBean();
+        MybatisSqlSessionFactoryBean bean = new MybatisSqlSessionFactoryBean();
-        org.apache.ibatis.session.Configuration configuration = new org.apache.ibatis.session.Configuration();
+        MybatisConfiguration configuration = new MybatisConfiguration();
         configuration.setMapUnderscoreToCamelCase(true);
         bean.setConfiguration(configuration);
         bean.setDataSource(dataSource);

launchers/standalone/src/main/resources/db/schema-h2.sql

@@ -527,3 +527,9 @@ CREATE TABLE s2_materialization_record
     PRIMARY KEY (`id`)
 );
 
+CREATE TABLE s2_sys_parameter
+(
+    id  INT PRIMARY KEY AUTO_INCREMENT,
+    admin varchar(500),
+    parameters text null
+);

launchers/standalone/src/main/resources/db/schema-mysql.sql

@@ -445,3 +445,9 @@ CREATE TABLE `s2_materialization_record`
     UNIQUE KEY `uq_id` (`materialization_id`,`element_type`,`element_id`,`data_time`)
 ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
 
+CREATE TABLE s2_sys_parameter
+(
+    id  int primary key AUTO_INCREMENT COMMENT '主键id',
+    admin varchar(500) COMMENT '系统管理员',
+    parameters text null COMMENT '配置项'
+);

launchers/standalone/src/main/resources/db/sql-update.sql

@@ -79,4 +79,12 @@ update s2_metric set status = 1;
 update s2_dimension set status = 1;
 
 --20231110
-UPDATE `s2_agent` SET `config` = replace (`config`,'LLM_S2QL','LLM_S2SQL') WHERE `config` LIKE '%LLM_S2QL%';
+UPDATE `s2_agent` SET `config` = replace (`config`,'LLM_S2QL','LLM_S2SQL') WHERE `config` LIKE '%LLM_S2QL%';
+
+--20231113
+CREATE TABLE s2_sys_parameter
+(
+    id  int primary key AUTO_INCREMENT COMMENT '主键id',
+    admin varchar(500) COMMENT '系统管理员',
+    parameters text null COMMENT '配置项'
+);

launchers/standalone/src/test/resources/db/schema-h2.sql

@@ -467,6 +467,12 @@ CREATE TABLE s2_materialization_record
 );
 
 
+CREATE TABLE s2_sys_parameter
+(
+    id  INT PRIMARY KEY AUTO_INCREMENT,
+    admin varchar(500),
+    parameters text null
+);
 
 
 

pom.xml

@@ -48,8 +48,7 @@
         <jjwt.version>0.9.1</jjwt.version>
         <alibaba.druid.version>1.2.3</alibaba.druid.version>
         <mysql.connector.java.version>5.1.46</mysql.connector.java.version>
-        <mybatis.test.version>2.3.1</mybatis.test.version>
+        <mybatis.plus.version>3.1.2</mybatis.plus.version>
-        <mybatis-spring.version>1.3.2</mybatis-spring.version>
         <clickhouse.jdbc.version>0.4.6</clickhouse.jdbc.version>
         <h2.version>1.4.200</h2.version>
         <fastjson.version>2.0.24</fastjson.version>
@@ -82,17 +81,7 @@
                 <artifactId>pagehelper</artifactId>
                 <version>${pagehelper.version}</version>
             </dependency>
-            <!--mybatis-->
+
-            <dependency>
-                <groupId>org.mybatis</groupId>
-                <artifactId>mybatis</artifactId>
-                <version>${mybatis.version}</version>
-            </dependency>
-            <dependency>
-                <groupId>org.mybatis</groupId>
-                <artifactId>mybatis-spring</artifactId>
-                <version>${mybatis-spring.version}</version>
-            </dependency>
             <!--lombok-->
             <dependency>
                 <groupId>org.projectlombok</groupId>

semantic/materialization/pom.xml

@@ -17,10 +17,6 @@
             <version>${project.version}</version>
             <scope>compile</scope>
         </dependency>
-        <dependency>
-            <groupId>org.mybatis</groupId>
-            <artifactId>mybatis</artifactId>
-        </dependency>
         <dependency>
             <groupId>commons-beanutils</groupId>
             <artifactId>commons-beanutils</artifactId>

semantic/model/pom.xml

@@ -28,6 +28,18 @@
             <artifactId>spring-context</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>${guava.version}</version>
+        </dependency>
+
+        <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+            <version>${commons.lang3.version}</version>
+        </dependency>
         <dependency>
             <groupId>com.tencent.supersonic</groupId>
             <artifactId>semantic-api</artifactId>
@@ -48,11 +60,6 @@
             <artifactId>spring-boot-starter-jdbc</artifactId>
         </dependency>
 
-        <dependency>
-            <groupId>org.mybatis</groupId>
-            <artifactId>mybatis</artifactId>
-        </dependency>
-
         <dependency>
             <groupId>com.alibaba</groupId>
             <artifactId>druid</artifactId>
@@ -66,18 +73,7 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-web</artifactId>
         </dependency>
-        <dependency>
+
-            <groupId>org.mybatis</groupId>
-            <artifactId>mybatis-spring</artifactId>
-            <version>${mybatis-spring.version}</version>
-            <scope>compile</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.github.pagehelper</groupId>
-            <artifactId>pagehelper</artifactId>
-            <version>${pagehelper.version}</version>
-            <scope>compile</scope>
-        </dependency>
         <dependency>
             <groupId>org.testng</groupId>
             <artifactId>testng</artifactId>

semantic/query/pom.xml

@@ -50,10 +50,6 @@
             <groupId>org.aspectj</groupId>
             <artifactId>aspectjweaver</artifactId>
         </dependency>
-        <dependency>
-            <groupId>org.mybatis</groupId>
-            <artifactId>mybatis</artifactId>
-        </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-web</artifactId>