topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 03:58:14 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

headless integrates knowledge (#722)

Browse Source
This commit is contained in:
daikon
2024-02-05 20:30:57 +08:00
committed by GitHub
parent 74d0ec2b23
commit 9600456bae
174 changed files with 1908 additions and 1817 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
launchers/chat/src/main/resources/META-INF/spring.factories
View File

@@ -19,8 +19,8 @@ com.tencent.supersonic.chat.core.corrector.SemanticCorrector=\
com.tencent.supersonic.chat.core.corrector.GroupByCorrector, \
com.tencent.supersonic.chat.core.corrector.HavingCorrector
com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter=\
com.tencent.supersonic.chat.core.knowledge.semantic.RemoteSemanticInterpreter
com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter=\
com.tencent.supersonic.chat.core.query.semantic.RemoteSemanticInterpreter
com.tencent.supersonic.chat.core.parser.sql.llm.ViewResolver=\
com.tencent.supersonic.chat.core.parser.sql.llm.HeuristicModelResolver

34
launchers/chat/src/main/resources/db/chat-schema-h2.sql
View File

@@ -156,23 +156,23 @@ CREATE TABLE IF NOT EXISTS `s2_user_department` (
);
COMMENT ON TABLE s2_user_department IS 'user_department_info';
CREATE TABLE IF NOT EXISTS `s2_dictionary_task` (
`id` INT NOT NULL AUTO_INCREMENT,
`name` varchar(255) NOT NULL , -- task name
`description` varchar(255) ,
`command`LONGVARCHAR NOT NULL , -- task Request Parameters
`command_md5` varchar(255) NOT NULL , -- task Request Parameters md5
`status` INT NOT NULL , -- the final status of the task
`dimension_ids` varchar(500) NULL ,
`created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ,
`created_by` varchar(100) NOT NULL ,
`progress` DOUBLE default 0.00 , -- task real-time progress
`elapsed_ms` bigINT DEFAULT NULL , -- the task takes time in milliseconds
`message` LONGVARCHAR , -- remark related information
PRIMARY KEY (`id`)
);
COMMENT ON TABLE s2_dictionary_task IS 'dictionary task information table';
--
-- CREATE TABLE IF NOT EXISTS `s2_dictionary_task` (
-- `id` INT NOT NULL AUTO_INCREMENT,
-- `name` varchar(255) NOT NULL , -- task name
-- `description` varchar(255) ,
-- `command`LONGVARCHAR NOT NULL , -- task Request Parameters
-- `command_md5` varchar(255) NOT NULL , -- task Request Parameters md5
-- `status` INT NOT NULL , -- the final status of the task
-- `dimension_ids` varchar(500) NULL ,
-- `created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ,
-- `created_by` varchar(100) NOT NULL ,
-- `progress` DOUBLE default 0.00 , -- task real-time progress
-- `elapsed_ms` bigINT DEFAULT NULL , -- the task takes time in milliseconds
-- `message` LONGVARCHAR , -- remark related information
-- PRIMARY KEY (`id`)
-- );
-- COMMENT ON TABLE s2_dictionary_task IS 'dictionary task information table';

9
launchers/standalone/src/main/java/com/tencent/supersonic/ChatDemoLoader.java
View File

@@ -23,6 +23,7 @@ import com.tencent.supersonic.common.pojo.SysParameter;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.service.SysParameterService;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.headless.server.service.KnowledgeService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
@@ -30,6 +31,7 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.CommandLineRunner;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.Arrays;
import java.util.List;
@@ -51,6 +53,8 @@ public class ChatDemoLoader implements CommandLineRunner {
private AgentService agentService;
@Autowired
private SysParameterService sysParameterService;
@Autowired
private KnowledgeService knowledgeService;
@Value("${demo.enabled:false}")
private boolean demoEnabled;
@@ -91,7 +95,10 @@ public class ChatDemoLoader implements CommandLineRunner {
queryRequest.setAgentId(1);
queryRequest.setUser(User.getFakeUser());
ParseResp parseResp = queryService.performParsing(queryRequest);
if (CollectionUtils.isEmpty(parseResp.getSelectedParses())) {
log.info("parseResp.getSelectedParses() is empty");
return;
}
ExecuteQueryReq executeReq = ExecuteQueryReq.builder().build();
executeReq.setQueryId(parseResp.getQueryId());
executeReq.setParseId(parseResp.getSelectedParses().get(0).getId());

4
launchers/standalone/src/main/resources/META-INF/spring.factories
View File

@@ -28,8 +28,8 @@ com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor=\
com.tencent.supersonic.chat.server.processor.parse.RespBuildProcessor, \
com.tencent.supersonic.chat.server.processor.parse.QueryRecommendProcessor
com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter=\
com.tencent.supersonic.chat.core.knowledge.semantic.LocalSemanticInterpreter
com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter=\
com.tencent.supersonic.chat.core.query.semantic.LocalSemanticInterpreter
com.tencent.supersonic.chat.core.parser.sql.llm.ViewResolver=\
com.tencent.supersonic.chat.core.parser.sql.llm.HeuristicViewResolver

23
launchers/standalone/src/main/resources/db/schema-h2.sql
View File

@@ -393,19 +393,30 @@ CREATE TABLE IF NOT EXISTS `singer` (
);
COMMENT ON TABLE singer IS 'singer_info';
CREATE TABLE IF NOT EXISTS `s2_dictionary_conf` (
`id` INT NOT NULL AUTO_INCREMENT,
`description` varchar(255) ,
`type` varchar(255) NOT NULL ,
`item_id` INT NOT NULL , -- task Request Parameters md5
`config` LONGVARCHAR , -- remark related information
`status` varchar(255) NOT NULL , -- the final status of the task
`created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ,
`created_by` varchar(100) NOT NULL ,
PRIMARY KEY (`id`)
);
COMMENT ON TABLE s2_dictionary_conf IS 'dictionary conf information table';
CREATE TABLE IF NOT EXISTS `s2_dictionary_task` (
`id` INT NOT NULL AUTO_INCREMENT,
`name` varchar(255) NOT NULL , -- task name
`description` varchar(255) ,
`command`LONGVARCHAR NOT NULL , -- task Request Parameters
`command_md5` varchar(255) NOT NULL , -- task Request Parameters md5
`status` INT NOT NULL , -- the final status of the task
`dimension_ids` varchar(500) NULL ,
`type` varchar(255) NOT NULL ,
`item_id` INT NOT NULL , -- task Request Parameters md5
`config` LONGVARCHAR , -- remark related information
`status` varchar(255) NOT NULL , -- the final status of the task
`created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ,
`created_by` varchar(100) NOT NULL ,
`progress` DOUBLE default 0.00 , -- task real-time progress
`elapsed_ms` bigINT DEFAULT NULL , -- the task takes time in milliseconds
`message` LONGVARCHAR , -- remark related information
PRIMARY KEY (`id`)
);
COMMENT ON TABLE s2_dictionary_task IS 'dictionary task information table';

56
launchers/standalone/src/main/resources/db/schema-mysql.sql
View File

@@ -201,37 +201,33 @@ CREATE TABLE `s2_database` (
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='数据库实例表';
CREATE TABLE `s2_dictionary` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`item_id` bigint(20) DEFAULT NULL COMMENT '对应维度id、指标id等',
`type` varchar(50) DEFAULT NULL COMMENT '对应维度、指标等',
`black_list` mediumtext COMMENT '字典黑名单',
`white_list` mediumtext COMMENT '字典白名单',
`rule_list` mediumtext COMMENT '字典规则',
`is_dict_Info` tinyint(1) NOT NULL DEFAULT '0' COMMENT '1-开启写入字典，0-不开启',
`created_at` datetime NOT NULL COMMENT '创建时间',
`updated_at` datetime NOT NULL COMMENT '更新时间',
`created_by` varchar(100) NOT NULL COMMENT '创建人',
`updated_by` varchar(100) DEFAULT NULL COMMENT '更新人',
`is_deleted` tinyint(1) NOT NULL DEFAULT '0' COMMENT '1-删除,0-可用',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='字典配置信息表';
CREATE TABLE IF NOT EXISTS `s2_dictionary_conf` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`description` varchar(255) ,
`type` varchar(255) NOT NULL ,
`item_id` INT NOT NULL , -- task Request Parameters md5
`config` mediumtext , -- remark related information
`status` varchar(255) NOT NULL , -- the final status of the task
`created_at` datetime NOT NULL COMMENT '创建时间' ,
`created_by` varchar(100) NOT NULL ,
PRIMARY KEY (`id`)
);
COMMENT ON TABLE s2_dictionary_conf IS '字典配置信息表';
CREATE TABLE `s2_dictionary_task` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`name` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '任务名称',
`description` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '任务描述',
`command` mediumtext COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '任务请求参数',
`command_md5` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '任务请求参数',
`dimension_ids` mediumtext NULL COMMENT '本次执行维度列表',
`status` int(10) NOT NULL COMMENT '任务最终运行状态',
`created_at` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
`created_by` varchar(100) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '创建人',
`progress` double(3,2) DEFAULT '0.00' COMMENT '任务进度',
`elapsed_ms` bigint(10) DEFAULT NULL COMMENT '任务耗时',
`message` mediumtext COLLATE utf8mb4_unicode_ci COMMENT '备注相关信息',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='字典任务信息表';
CREATE TABLE IF NOT EXISTS `s2_dictionary_task` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`name` varchar(255) NOT NULL , -- task name
`description` varchar(255) ,
`type` varchar(255) NOT NULL ,
`item_id` INT NOT NULL , -- task Request Parameters md5
`config` mediumtext , -- remark related information
`status` varchar(255) NOT NULL , -- the final status of the task
`created_at` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
`created_by` varchar(100) NOT NULL ,
`elapsed_ms` int(10) DEFAULT NULL , -- the task takes time in milliseconds
PRIMARY KEY (`id`)
);
COMMENT ON TABLE s2_dictionary_task IS 'dictionary task information table';
CREATE TABLE `s2_dimension` (

2
launchers/standalone/src/test/java/com/tencent/supersonic/chat/BaseTest.java
View File

@@ -1,7 +1,7 @@
package com.tencent.supersonic.chat;
import com.tencent.supersonic.BaseApplication;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;

2
launchers/standalone/src/test/java/com/tencent/supersonic/chat/TagTest.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.chat;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;

2
launchers/standalone/src/test/java/com/tencent/supersonic/chat/mapper/MapperTest.java
View File

@@ -1,6 +1,6 @@
package com.tencent.supersonic.chat.mapper;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;

94
launchers/standalone/src/test/java/com/tencent/supersonic/headless/DictTest.java Normal file
View File

@@ -0,0 +1,94 @@
package com.tencent.supersonic.headless;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.common.util.JsonUtil;
import com.tencent.supersonic.headless.api.pojo.ItemValueConfig;
import com.tencent.supersonic.headless.api.pojo.request.DictSingleTaskReq;
import com.tencent.supersonic.headless.server.persistence.dataobject.DictConfDO;
import com.tencent.supersonic.headless.server.persistence.mapper.DictConfMapper;
import com.tencent.supersonic.headless.server.service.DictTaskService;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
public class DictTest extends BaseTest {
@Autowired
private DictConfMapper confMapper;
@Autowired
private DictTaskService taskService;
@Test
public void insertConf() {
DictConfDO confDO = new DictConfDO();
Date createAt = new Date();
confDO.setType(TypeEnums.DIMENSION.name());
confDO.setItemId(1L);
confDO.setConfig(JsonUtil.toString(new ItemValueConfig()));
confDO.setStatus(StatusEnum.ONLINE.getStatus());
confDO.setCreatedAt(createAt);
confDO.setCreatedBy("admin");
confMapper.insert(confDO);
DictConfDO confDODb = confMapper.selectById(1L);
System.out.println(confDODb.getId());
}
@Test
public void editConf() {
DictConfDO confDO = new DictConfDO();
Date createAt = new Date();
confDO.setType(TypeEnums.DIMENSION.name());
confDO.setItemId(3L);
ItemValueConfig config = new ItemValueConfig();
config.setMetricId(4L);
config.setBlackList(new ArrayList<>(Arrays.asList("1", "3")));
config.setWhiteList(new ArrayList<>(Arrays.asList("2", "4")));
confDO.setConfig(JsonUtil.toString(config));
confDO.setStatus(TaskStatusEnum.PENDING.getStatus());
confDO.setCreatedAt(createAt);
confDO.setCreatedBy("admin");
confMapper.insert(confDO);
DictConfDO confDODb = confMapper.selectById(1L);
confDO.setStatus(StatusEnum.OFFLINE.getStatus());
// config.setMetricId(3L);
config.setBlackList(new ArrayList<>(Arrays.asList("p2")));
config.setWhiteList(new ArrayList<>(Arrays.asList("p10", "p12")));
confDODb.setConfig(JsonUtil.toString(config));
confMapper.updateById(confDODb);
DictConfDO confDODb1 = confMapper.selectById(1L);
System.out.println(confDODb1.getId());
}
@Test
void testBatchInsert() {
for (int i = 0; i < 5; i++) {
insertConf();
}
QueryWrapper<DictConfDO> wrapper = new QueryWrapper<>();
wrapper.lambda().eq(DictConfDO::getType, "DIMENSION");
List<DictConfDO> dictConfDOList = confMapper.selectList(wrapper);
System.out.println(dictConfDOList);
}
@Test
void testAddTask() {
editConf();
DictConfDO confDODb = confMapper.selectById(1L);
DictSingleTaskReq dictTask = DictSingleTaskReq.builder().itemId(confDODb.getItemId())
.type(TypeEnums.DIMENSION).build();
taskService.addDictTask(dictTask, null);
DictSingleTaskReq taskReq = DictSingleTaskReq.builder().itemId(3L).type(TypeEnums.DIMENSION).build();
taskService.deleteDictTask(taskReq, null);
System.out.println();
}
}

2
launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java
View File

@@ -3,7 +3,7 @@ package com.tencent.supersonic.util;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.core.agent.Agent;

4
launchers/standalone/src/test/resources/META-INF/spring.factories
View File

@@ -27,8 +27,8 @@ com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor=\
com.tencent.supersonic.chat.server.processor.parse.RespBuildProcessor, \
com.tencent.supersonic.chat.server.processor.parse.QueryRecommendProcessor
com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter=\
com.tencent.supersonic.chat.core.knowledge.semantic.LocalSemanticInterpreter
com.tencent.supersonic.chat.core.query.semantic.SemanticInterpreter=\
com.tencent.supersonic.chat.core.query.semantic.LocalSemanticInterpreter
com.tencent.supersonic.chat.core.parser.sql.llm.ViewResolver=\
com.tencent.supersonic.chat.core.parser.sql.llm.HeuristicViewResolver

23
launchers/standalone/src/test/resources/db/schema-h2.sql
View File

@@ -394,19 +394,30 @@ CREATE TABLE IF NOT EXISTS `singer` (
);
COMMENT ON TABLE singer IS 'singer_info';
CREATE TABLE IF NOT EXISTS `s2_dictionary_conf` (
`id` INT NOT NULL AUTO_INCREMENT,
`description` varchar(255) ,
`type` varchar(255) NOT NULL ,
`item_id` INT NOT NULL , -- task Request Parameters md5
`config` LONGVARCHAR , -- remark related information
`status` varchar(255) NOT NULL , -- the final status of the task
`created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ,
`created_by` varchar(100) NOT NULL ,
PRIMARY KEY (`id`)
);
COMMENT ON TABLE s2_dictionary_conf IS 'dictionary conf information table';
CREATE TABLE IF NOT EXISTS `s2_dictionary_task` (
`id` INT NOT NULL AUTO_INCREMENT,
`name` varchar(255) NOT NULL , -- task name
`description` varchar(255) ,
`command`LONGVARCHAR NOT NULL , -- task Request Parameters
`command_md5` varchar(255) NOT NULL , -- task Request Parameters md5
`status` INT NOT NULL , -- the final status of the task
`dimension_ids` varchar(500) NULL ,
`type` varchar(255) NOT NULL ,
`item_id` INT NOT NULL , -- task Request Parameters md5
`config` LONGVARCHAR , -- remark related information
`status` varchar(255) NOT NULL , -- the final status of the task
`created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ,
`created_by` varchar(100) NOT NULL ,
`progress` DOUBLE default 0.00 , -- task real-time progress
`elapsed_ms` bigINT DEFAULT NULL , -- the task takes time in milliseconds
`message` LONGVARCHAR , -- remark related information
PRIMARY KEY (`id`)
);
COMMENT ON TABLE s2_dictionary_task IS 'dictionary task information table';