topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 12:07:42 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(feature) (common) add sys parameter setting (#377)

Browse Source 
Co-authored-by: jolunoluo
This commit is contained in:
LXW
2023-11-13 22:07:18 +08:00
committed by GitHub
parent 805abeb261
commit 1ad2c5402b
22 changed files with 266 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
auth/authentication/pom.xml
View File

@@ -33,12 +33,6 @@
<artifactId>spring-boot-starter-jdbc</artifactId>
</dependency>
<dependency>
<groupId>org.mybatis</groupId>
<artifactId>mybatis</artifactId>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>druid</artifactId>
@@ -52,12 +46,7 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.mybatis</groupId>
<artifactId>mybatis-spring</artifactId>
<version>${mybatis-spring.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.github.pagehelper</groupId>
<artifactId>pagehelper</artifactId>

27
chat/core/pom.xml
View File

@@ -59,16 +59,7 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.mybatis</groupId>
<artifactId>mybatis</artifactId>
</dependency>
<dependency>
<groupId>org.mybatis.spring.boot</groupId>
<artifactId>mybatis-spring-boot-starter</artifactId>
<version>${mybatis-spring.version}</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>druid</artifactId>
@@ -78,24 +69,6 @@
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
</dependency>
<dependency>
<groupId>org.mybatis</groupId>
<artifactId>mybatis-spring</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mybatis.spring.boot</groupId>
<artifactId>mybatis-spring-boot-starter-test</artifactId>
<version>${mybatis.test.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.github.pagehelper</groupId>
<artifactId>pagehelper-spring-boot-starter</artifactId>
<version>${pagehelper.spring.version}</version>
</dependency>
<dependency>
<groupId>com.h2database</groupId>

49
chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/SysParameterServiceImpl.java Normal file
View File

@@ -0,0 +1,49 @@
package com.tencent.supersonic.chat.service.impl;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.tencent.supersonic.common.pojo.SysParameter;
import com.tencent.supersonic.common.persistence.dataobject.SysParameterDO;
import com.tencent.supersonic.common.persistence.mapper.SysParameterMapper;
import com.tencent.supersonic.common.service.SysParameterService;
import com.tencent.supersonic.common.util.BeanMapper;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.List;
@Service
public class SysParameterServiceImpl
extends ServiceImpl<SysParameterMapper, SysParameterDO> implements SysParameterService {
@Override
public SysParameter getSysParameter() {
List<SysParameterDO> list = list();
if (CollectionUtils.isEmpty(list)) {
return new SysParameter();
}
return convert(list.iterator().next());
}
@Override
public void save(SysParameter sysParameter) {
SysParameterDO chatParameterDO = convert(sysParameter);
saveOrUpdate(chatParameterDO);
}
private SysParameter convert(SysParameterDO sysParameterDO) {
SysParameter chatParameter = new SysParameter();
BeanMapper.mapper(sysParameterDO, chatParameter);
chatParameter.setParameters(JSONObject.parseObject(sysParameterDO.getParameters(), List.class));
chatParameter.setAdmin(sysParameterDO.getAdmin());
return chatParameter;
}
private SysParameterDO convert(SysParameter sysParameter) {
SysParameterDO sysParameterDO = new SysParameterDO();
BeanMapper.mapper(sysParameter, sysParameterDO);
sysParameterDO.setParameters(JSONObject.toJSONString(sysParameter.getParameters()));
sysParameterDO.setAdmin(sysParameter.getAdmin());
return sysParameterDO;
}
}

5
chat/knowledge/pom.xml
View File

@@ -20,11 +20,6 @@
<artifactId>lombok</artifactId>
</dependency>
<dependency>
<groupId>org.mybatis</groupId>
<artifactId>mybatis</artifactId>
</dependency>
<dependency>
<groupId>com.hankcs</groupId>
<artifactId>hanlp</artifactId>

21
common/pom.xml
View File

@@ -115,8 +115,25 @@
<dependency>
<groupId>com.github.pagehelper</groupId>
<artifactId>pagehelper</artifactId>
<version>${pagehelper.version}</version>
<artifactId>pagehelper-spring-boot-starter</artifactId>
<version>${pagehelper.spring.version}</version>
<exclusions>
<exclusion>
<groupId>org.mybatis</groupId>
<artifactId>mybatis</artifactId>
</exclusion>
<exclusion>
<groupId>org.mybatis</groupId>
<artifactId>mybatis-spring</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-boot-starter</artifactId>
<version>${mybatis.plus.version}</version>
</dependency>
<dependency>

18
common/src/main/java/com/tencent/supersonic/common/persistence/dataobject/SysParameterDO.java Normal file
View File

@@ -0,0 +1,18 @@
package com.tencent.supersonic.common.persistence.dataobject;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
@Data
@TableName("s2_sys_parameter")
public class SysParameterDO {
@TableId(type = IdType.AUTO)
private Integer id;
private String parameters;
private String admin;
}

11
common/src/main/java/com/tencent/supersonic/common/persistence/mapper/SysParameterMapper.java Normal file
View File

@@ -0,0 +1,11 @@
package com.tencent.supersonic.common.persistence.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.tencent.supersonic.common.persistence.dataobject.SysParameterDO;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface SysParameterMapper extends BaseMapper<SysParameterDO> {
}

16
common/src/main/java/com/tencent/supersonic/common/pojo/Parameter.java Normal file
View File

@@ -0,0 +1,16 @@
package com.tencent.supersonic.common.pojo;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@AllArgsConstructor
@NoArgsConstructor
public class Parameter {
private String name;
private String value;
private String comment;
private String dataType;
}

49
common/src/main/java/com/tencent/supersonic/common/pojo/SysParameter.java Normal file
View File

@@ -0,0 +1,49 @@
package com.tencent.supersonic.common.pojo;
import com.google.common.collect.Lists;
import lombok.Data;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import java.util.Arrays;
import java.util.List;
@Data
public class SysParameter {
private Integer id;
private List<String> admins;
private List<Parameter> parameters;
public String getAdmin() {
if (CollectionUtils.isEmpty(admins)) {
return "";
}
return StringUtils.join(admins, ",");
}
public void setAdmin(String admin) {
if (StringUtils.isNotBlank(admin)) {
admins = Arrays.asList(admin.split(","));
} else {
admins = Lists.newArrayList();
}
}
public void init() {
parameters = Lists.newArrayList();
admins = Lists.newArrayList("admin");
parameters.add(new Parameter("one.detection.size", "8", "一次探测个数", "number"));
parameters.add(new Parameter("one.detection.max.size", "20", "阈值", "number"));
parameters.add(new Parameter("metric.dimension.min.threshold", "0.3", "指标名、维度名最小文本相似度", "number"));
parameters.add(new Parameter("metric.dimension.threshold", "0.3", "指标名、维度名文本相似度", "number"));
parameters.add(new Parameter("dimension.value.threshold", "0.5", "维度值最小文本相似度", "number"));
parameters.add(new Parameter("embedding.mapper.word.min", "0.3", "用于向量召回最小的文本长度", "number"));
parameters.add(new Parameter("embedding.mapper.word.max", "0.3", "用于向量召回最大的文本长度", "number"));
parameters.add(new Parameter("embedding.mapper.batch", "0.3", "批量向量召回文本请求个数", "number"));
parameters.add(new Parameter("embedding.mapper.number", "0.3", "批量向量召回文本返回结果个数", "number"));
parameters.add(new Parameter("embedding.mapper.distance.threshold", "0.3", "Mapper阶段向量召回相似度阈值", "number"));
}
}

31
common/src/main/java/com/tencent/supersonic/common/rest/SysParameterController.java Normal file
View File

@@ -0,0 +1,31 @@
package com.tencent.supersonic.common.rest;
import com.tencent.supersonic.common.pojo.SysParameter;
import com.tencent.supersonic.common.service.SysParameterService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@RestController
@RequestMapping({"/api/semantic/parameter"})
public class SysParameterController {
@Autowired
private SysParameterService sysParameterService;
@PostMapping
public Boolean save(@RequestBody SysParameter sysParameter) {
sysParameterService.save(sysParameter);
return true;
}
@GetMapping
SysParameter get() {
return sysParameterService.getSysParameter();
}
}

13
common/src/main/java/com/tencent/supersonic/common/service/SysParameterService.java Normal file
View File

@@ -0,0 +1,13 @@
package com.tencent.supersonic.common.service;
import com.baomidou.mybatisplus.extension.service.IService;
import com.tencent.supersonic.common.persistence.dataobject.SysParameterDO;
import com.tencent.supersonic.common.pojo.SysParameter;
public interface SysParameterService extends IService<SysParameterDO> {
SysParameter getSysParameter();
void save(SysParameter sysParameter);
}

15
launchers/common/pom.xml
View File

@@ -39,21 +39,6 @@
<groupId>org.aspectj</groupId>
<artifactId>aspectjweaver</artifactId>
</dependency>
<dependency>
<groupId>org.mybatis</groupId>
<artifactId>mybatis</artifactId>
</dependency>
<dependency>
<groupId>org.mybatis</groupId>
<artifactId>mybatis-spring</artifactId>
</dependency>
<dependency>
<groupId>com.github.pagehelper</groupId>
<artifactId>pagehelper</artifactId>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>

13
launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java
View File

@@ -8,6 +8,7 @@ import com.tencent.supersonic.chat.agent.AgentConfig;
import com.tencent.supersonic.chat.agent.tool.AgentToolType;
import com.tencent.supersonic.chat.agent.tool.LLMParserTool;
import com.tencent.supersonic.chat.agent.tool.RuleQueryTool;
import com.tencent.supersonic.common.pojo.SysParameter;
import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq;
import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq;
@@ -23,6 +24,7 @@ import com.tencent.supersonic.chat.plugin.PluginParseConfig;
import com.tencent.supersonic.chat.query.plugin.ParamOption;
import com.tencent.supersonic.chat.query.plugin.WebBase;
import com.tencent.supersonic.chat.service.AgentService;
import com.tencent.supersonic.common.service.SysParameterService;
import com.tencent.supersonic.chat.service.ChatService;
import com.tencent.supersonic.chat.service.ConfigService;
import com.tencent.supersonic.chat.service.PluginService;
@@ -55,6 +57,9 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
private PluginService pluginService;
@Autowired
private AgentService agentService;
@Autowired
private SysParameterService sysParameterService;
@Value("${spring.h2.demo.enabled:false}")
private boolean demoEnable;
@@ -94,6 +99,13 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
parseAndExecute(2, "访问次数最高的部门");
}
public void addChatParameter() {
SysParameter chatParameter = new SysParameter();
chatParameter.setId(1);
chatParameter.init();
sysParameterService.save(chatParameter);
}
public void addDemoChatConfig_1() {
ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq();
chatConfigBaseReq.setModelId(1L);
@@ -294,6 +306,7 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
return;
}
try {
addChatParameter();
addDemoChatConfig_1();
addDemoChatConfig_2();
addPlugin_1();

7
launchers/standalone/src/main/java/com/tencent/supersonic/db/MybatisConfig.java
View File

@@ -1,8 +1,9 @@
package com.tencent.supersonic.db;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.extension.spring.MybatisSqlSessionFactoryBean;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.session.SqlSessionFactory;
import org.mybatis.spring.SqlSessionFactoryBean;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@@ -19,8 +20,8 @@ public class MybatisConfig {
@Bean
public SqlSessionFactory sqlSessionFactory(DataSource dataSource) throws Exception {
SqlSessionFactoryBean bean = new SqlSessionFactoryBean();
org.apache.ibatis.session.Configuration configuration = new org.apache.ibatis.session.Configuration();
MybatisSqlSessionFactoryBean bean = new MybatisSqlSessionFactoryBean();
MybatisConfiguration configuration = new MybatisConfiguration();
configuration.setMapUnderscoreToCamelCase(true);
bean.setConfiguration(configuration);
bean.setDataSource(dataSource);

6
launchers/standalone/src/main/resources/db/schema-h2.sql
View File

@@ -527,3 +527,9 @@ CREATE TABLE s2_materialization_record
PRIMARY KEY (`id`)
);
CREATE TABLE s2_sys_parameter
(
id INT PRIMARY KEY AUTO_INCREMENT,
admin varchar(500),
parameters text null
);

6
launchers/standalone/src/main/resources/db/schema-mysql.sql
View File

@@ -445,3 +445,9 @@ CREATE TABLE `s2_materialization_record`
UNIQUE KEY `uq_id` (`materialization_id`,`element_type`,`element_id`,`data_time`)
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
CREATE TABLE s2_sys_parameter
(
id int primary key AUTO_INCREMENT COMMENT '主键id',
admin varchar(500) COMMENT '系统管理员',
parameters text null COMMENT '配置项'
);

8
launchers/standalone/src/main/resources/db/sql-update.sql
View File

@@ -80,3 +80,11 @@ update s2_dimension set status = 1;
--20231110
UPDATE `s2_agent` SET `config` = replace (`config`,'LLM_S2QL','LLM_S2SQL') WHERE `config` LIKE '%LLM_S2QL%';
--20231113
CREATE TABLE s2_sys_parameter
(
id int primary key AUTO_INCREMENT COMMENT '主键id',
admin varchar(500) COMMENT '系统管理员',
parameters text null COMMENT '配置项'
);

6
launchers/standalone/src/test/resources/db/schema-h2.sql
View File

@@ -467,6 +467,12 @@ CREATE TABLE s2_materialization_record
);
CREATE TABLE s2_sys_parameter
(
id INT PRIMARY KEY AUTO_INCREMENT,
admin varchar(500),
parameters text null
);

15
pom.xml
View File

@@ -48,8 +48,7 @@
<jjwt.version>0.9.1</jjwt.version>
<alibaba.druid.version>1.2.3</alibaba.druid.version>
<mysql.connector.java.version>5.1.46</mysql.connector.java.version>
<mybatis.test.version>2.3.1</mybatis.test.version>
<mybatis-spring.version>1.3.2</mybatis-spring.version>
<mybatis.plus.version>3.1.2</mybatis.plus.version>
<clickhouse.jdbc.version>0.4.6</clickhouse.jdbc.version>
<h2.version>1.4.200</h2.version>
<fastjson.version>2.0.24</fastjson.version>
@@ -82,17 +81,7 @@
<artifactId>pagehelper</artifactId>
<version>${pagehelper.version}</version>
</dependency>
<!--mybatis-->
<dependency>
<groupId>org.mybatis</groupId>
<artifactId>mybatis</artifactId>
<version>${mybatis.version}</version>
</dependency>
<dependency>
<groupId>org.mybatis</groupId>
<artifactId>mybatis-spring</artifactId>
<version>${mybatis-spring.version}</version>
</dependency>
<!--lombok-->
<dependency>
<groupId>org.projectlombok</groupId>

4
semantic/materialization/pom.xml
View File

@@ -17,10 +17,6 @@
<version>${project.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.mybatis</groupId>
<artifactId>mybatis</artifactId>
</dependency>
<dependency>
<groupId>commons-beanutils</groupId>
<artifactId>commons-beanutils</artifactId>

30
semantic/model/pom.xml
View File

@@ -28,6 +28,18 @@
<artifactId>spring-context</artifactId>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>${commons.lang3.version}</version>
</dependency>
<dependency>
<groupId>com.tencent.supersonic</groupId>
<artifactId>semantic-api</artifactId>
@@ -48,11 +60,6 @@
<artifactId>spring-boot-starter-jdbc</artifactId>
</dependency>
<dependency>
<groupId>org.mybatis</groupId>
<artifactId>mybatis</artifactId>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>druid</artifactId>
@@ -66,18 +73,7 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.mybatis</groupId>
<artifactId>mybatis-spring</artifactId>
<version>${mybatis-spring.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.github.pagehelper</groupId>
<artifactId>pagehelper</artifactId>
<version>${pagehelper.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>

4
semantic/query/pom.xml
View File

@@ -50,10 +50,6 @@
<groupId>org.aspectj</groupId>
<artifactId>aspectjweaver</artifactId>
</dependency>
<dependency>
<groupId>org.mybatis</groupId>
<artifactId>mybatis</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>