topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 03:58:14 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(headless)(chat) Add views and adapt chat and headless (#700)

Browse Source 
* (improvement)(headless)(chat) Add views and adapt chat and headless

---------

Co-authored-by: jolunoluo
This commit is contained in:
LXW
2024-01-30 20:43:53 +08:00
committed by GitHub
parent 31f8c1df35
commit 24b442baef
237 changed files with 3205 additions and 4479 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
launchers/chat/src/main/resources/META-INF/spring.factories
View File

@@ -21,7 +21,7 @@ com.tencent.supersonic.chat.core.corrector.SemanticCorrector=\
com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter=\
com.tencent.supersonic.chat.core.knowledge.semantic.RemoteSemanticInterpreter
com.tencent.supersonic.chat.core.parser.sql.llm.ModelResolver=\
com.tencent.supersonic.chat.core.parser.sql.llm.ViewResolver=\
com.tencent.supersonic.chat.core.parser.sql.llm.HeuristicModelResolver
com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\

297
launchers/standalone/src/main/java/com/tencent/supersonic/ChatDemoLoader.java
View File

@@ -3,15 +3,8 @@ package com.tencent.supersonic;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigBaseReq;
import com.tencent.supersonic.chat.api.pojo.request.ChatDefaultConfigReq;
import com.tencent.supersonic.chat.api.pojo.request.ChatDetailConfigReq;
import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq;
import com.tencent.supersonic.chat.api.pojo.request.ItemVisibility;
import com.tencent.supersonic.chat.api.pojo.request.KnowledgeInfoReq;
import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.pojo.request.RecommendedQuestionReq;
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
import com.tencent.supersonic.chat.core.agent.Agent;
import com.tencent.supersonic.chat.core.agent.AgentConfig;
@@ -39,9 +32,7 @@ import org.springframework.boot.CommandLineRunner;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@Component
@@ -82,14 +73,6 @@ public class ChatDemoLoader implements CommandLineRunner {
public void doRun() {
try {
addSysParameter();
addDemoChatConfig_1();
addDemoChatConfig_2();
addDemoChatConfig_3();
addDemoChatConfig_4();
addDemoChatConfig_5();
addDemoChatConfig_6();
addDemoChatConfig_7();
addDemoChatConfig_8();
addPlugin_1();
addAgent1();
addAgent2();
@@ -147,275 +130,10 @@ public class ChatDemoLoader implements CommandLineRunner {
sysParameterService.save(sysParameter);
}
public void addDemoChatConfig_1() {
ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq();
chatConfigBaseReq.setModelId(1L);
ChatDetailConfigReq chatDetailConfig = new ChatDetailConfigReq();
ChatDefaultConfigReq chatDefaultConfigDetail = new ChatDefaultConfigReq();
List<Long> dimensionIds0 = Collections.singletonList(1L);
List<Long> metricIds0 = Lists.newArrayList();
chatDefaultConfigDetail.setDimensionIds(dimensionIds0);
chatDefaultConfigDetail.setMetricIds(metricIds0);
chatDefaultConfigDetail.setUnit(7);
chatDefaultConfigDetail.setPeriod("DAY");
chatDetailConfig.setChatDefaultConfig(chatDefaultConfigDetail);
ItemVisibility visibility0 = new ItemVisibility();
chatDetailConfig.setVisibility(visibility0);
chatConfigBaseReq.setChatDetailConfig(chatDetailConfig);
ChatAggConfigReq chatAggConfig = new ChatAggConfigReq();
ChatDefaultConfigReq chatDefaultConfigAgg = new ChatDefaultConfigReq();
List<Long> dimensionIds1 = Arrays.asList(1L);
List<Long> metricIds1 = Lists.newArrayList();
chatDefaultConfigAgg.setDimensionIds(dimensionIds1);
chatDefaultConfigAgg.setMetricIds(metricIds1);
chatDefaultConfigAgg.setUnit(7);
chatDefaultConfigAgg.setPeriod("DAY");
chatDefaultConfigAgg.setTimeMode(ChatDefaultConfigReq.TimeMode.RECENT);
chatAggConfig.setChatDefaultConfig(chatDefaultConfigAgg);
ItemVisibility visibility1 = new ItemVisibility();
chatAggConfig.setVisibility(visibility1);
List<KnowledgeInfoReq> knowledgeInfos = new ArrayList<>();
KnowledgeInfoReq knowledgeInfoReq = new KnowledgeInfoReq();
knowledgeInfoReq.setItemId(3L);
knowledgeInfoReq.setSearchEnable(true);
knowledgeInfos.add(knowledgeInfoReq);
chatAggConfig.setKnowledgeInfos(knowledgeInfos);
chatConfigBaseReq.setChatAggConfig(chatAggConfig);
configService.addConfig(chatConfigBaseReq, user);
}
public void addDemoChatConfig_2() {
ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq();
chatConfigBaseReq.setModelId(2L);
ChatDetailConfigReq chatDetailConfig = new ChatDetailConfigReq();
ChatDefaultConfigReq chatDefaultConfigDetail = new ChatDefaultConfigReq();
List<Long> dimensionIds0 = new ArrayList<>();
List<Long> metricIds0 = Arrays.asList(1L, 2L);
chatDefaultConfigDetail.setDimensionIds(dimensionIds0);
chatDefaultConfigDetail.setMetricIds(metricIds0);
chatDefaultConfigDetail.setUnit(7);
chatDefaultConfigDetail.setPeriod("DAY");
chatDetailConfig.setChatDefaultConfig(chatDefaultConfigDetail);
ItemVisibility visibility0 = new ItemVisibility();
chatDetailConfig.setVisibility(visibility0);
chatConfigBaseReq.setChatDetailConfig(chatDetailConfig);
ChatAggConfigReq chatAggConfig = new ChatAggConfigReq();
ChatDefaultConfigReq chatDefaultConfigAgg = new ChatDefaultConfigReq();
List<Long> dimensionIds1 = new ArrayList<>();
List<Long> metricIds1 = Arrays.asList(1L, 2L);
chatDefaultConfigAgg.setDimensionIds(dimensionIds1);
chatDefaultConfigAgg.setMetricIds(metricIds1);
chatDefaultConfigAgg.setUnit(7);
chatDefaultConfigAgg.setPeriod("DAY");
chatDefaultConfigAgg.setTimeMode(ChatDefaultConfigReq.TimeMode.RECENT);
chatAggConfig.setChatDefaultConfig(chatDefaultConfigAgg);
ItemVisibility visibility1 = new ItemVisibility();
chatAggConfig.setVisibility(visibility1);
List<KnowledgeInfoReq> knowledgeInfos = new ArrayList<>();
KnowledgeInfoReq knowledgeInfoReq = new KnowledgeInfoReq();
knowledgeInfoReq.setItemId(3L);
knowledgeInfoReq.setSearchEnable(true);
knowledgeInfos.add(knowledgeInfoReq);
chatAggConfig.setKnowledgeInfos(knowledgeInfos);
chatConfigBaseReq.setChatAggConfig(chatAggConfig);
configService.addConfig(chatConfigBaseReq, user);
}
public void addDemoChatConfig_3() {
ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq();
chatConfigBaseReq.setModelId(3L);
ChatDetailConfigReq chatDetailConfig = new ChatDetailConfigReq();
ChatDefaultConfigReq chatDefaultConfigDetail = new ChatDefaultConfigReq();
List<Long> dimensionIds0 = Collections.singletonList(3L);
List<Long> metricIds0 = Collections.singletonList(4L);
chatDefaultConfigDetail.setDimensionIds(dimensionIds0);
chatDefaultConfigDetail.setMetricIds(metricIds0);
chatDefaultConfigDetail.setUnit(7);
chatDefaultConfigDetail.setPeriod("DAY");
chatDetailConfig.setChatDefaultConfig(chatDefaultConfigDetail);
ItemVisibility visibility0 = new ItemVisibility();
chatDetailConfig.setVisibility(visibility0);
chatConfigBaseReq.setChatDetailConfig(chatDetailConfig);
ChatAggConfigReq chatAggConfig = new ChatAggConfigReq();
ChatDefaultConfigReq chatDefaultConfigAgg = new ChatDefaultConfigReq();
List<Long> dimensionIds1 = Collections.singletonList(3L);
List<Long> metricIds1 = Collections.singletonList(4L);
chatDefaultConfigAgg.setDimensionIds(dimensionIds1);
chatDefaultConfigAgg.setMetricIds(metricIds1);
chatDefaultConfigAgg.setUnit(7);
chatDefaultConfigAgg.setPeriod("DAY");
chatDefaultConfigAgg.setTimeMode(ChatDefaultConfigReq.TimeMode.RECENT);
chatAggConfig.setChatDefaultConfig(chatDefaultConfigAgg);
ItemVisibility visibility1 = new ItemVisibility();
chatAggConfig.setVisibility(visibility1);
List<KnowledgeInfoReq> knowledgeInfos = new ArrayList<>();
KnowledgeInfoReq knowledgeInfoReq = new KnowledgeInfoReq();
knowledgeInfoReq.setItemId(5L);
knowledgeInfoReq.setSearchEnable(true);
knowledgeInfos.add(knowledgeInfoReq);
chatAggConfig.setKnowledgeInfos(knowledgeInfos);
chatConfigBaseReq.setChatAggConfig(chatAggConfig);
configService.addConfig(chatConfigBaseReq, user);
}
public void addDemoChatConfig_4() {
ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq();
chatConfigBaseReq.setModelId(4L);
ChatDetailConfigReq chatDetailConfig = new ChatDetailConfigReq();
ChatDefaultConfigReq chatDefaultConfigDetail = new ChatDefaultConfigReq();
List<Long> dimensionIds0 = Arrays.asList(4L, 5L, 6L, 7L);
List<Long> metricIds0 = Arrays.asList(5L);
chatDefaultConfigDetail.setDimensionIds(dimensionIds0);
chatDefaultConfigDetail.setMetricIds(metricIds0);
chatDefaultConfigDetail.setUnit(7);
chatDefaultConfigDetail.setPeriod("DAY");
chatDetailConfig.setChatDefaultConfig(chatDefaultConfigDetail);
ItemVisibility visibility0 = new ItemVisibility();
chatDetailConfig.setVisibility(visibility0);
chatConfigBaseReq.setChatDetailConfig(chatDetailConfig);
ChatAggConfigReq chatAggConfig = new ChatAggConfigReq();
ChatDefaultConfigReq chatDefaultConfigAgg = new ChatDefaultConfigReq();
List<Long> dimensionIds1 = Arrays.asList(4L, 4L, 6L, 7L);
List<Long> metricIds1 = Arrays.asList(5L);
chatDefaultConfigAgg.setDimensionIds(dimensionIds1);
chatDefaultConfigAgg.setMetricIds(metricIds1);
chatDefaultConfigAgg.setUnit(7);
chatDefaultConfigAgg.setPeriod("DAY");
chatDefaultConfigAgg.setTimeMode(ChatDefaultConfigReq.TimeMode.RECENT);
chatAggConfig.setChatDefaultConfig(chatDefaultConfigAgg);
ItemVisibility visibility1 = new ItemVisibility();
chatAggConfig.setVisibility(visibility1);
chatConfigBaseReq.setChatAggConfig(chatAggConfig);
List<RecommendedQuestionReq> recommendedQuestions = new ArrayList<>();
chatConfigBaseReq.setRecommendedQuestions(recommendedQuestions);
configService.addConfig(chatConfigBaseReq, user);
}
public void addDemoChatConfig_5() {
ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq();
chatConfigBaseReq.setModelId(5L);
ChatDetailConfigReq chatDetailConfig = getChatDetailConfigReq();
List<KnowledgeInfoReq> knowledgeInfos = new ArrayList<>();
KnowledgeInfoReq knowledgeInfoReq = new KnowledgeInfoReq();
knowledgeInfoReq.setSearchEnable(true);
knowledgeInfoReq.setItemId(10L);
knowledgeInfoReq.setBizName("most_popular_in");
knowledgeInfos.add(knowledgeInfoReq);
KnowledgeInfoReq knowledgeInfoReq2 = new KnowledgeInfoReq();
knowledgeInfoReq2.setSearchEnable(true);
knowledgeInfoReq2.setItemId(11L);
knowledgeInfoReq2.setBizName("g_name");
knowledgeInfos.add(knowledgeInfoReq2);
chatDetailConfig.setKnowledgeInfos(knowledgeInfos);
chatConfigBaseReq.setChatDetailConfig(chatDetailConfig);
List<RecommendedQuestionReq> recommendedQuestions = new ArrayList<>();
chatConfigBaseReq.setRecommendedQuestions(recommendedQuestions);
configService.addConfig(chatConfigBaseReq, user);
}
private ChatDetailConfigReq getChatDetailConfigReq() {
ChatDetailConfigReq chatDetailConfig = new ChatDetailConfigReq();
ChatDefaultConfigReq chatDefaultConfigDetail = new ChatDefaultConfigReq();
chatDefaultConfigDetail.setUnit(-1);
chatDefaultConfigDetail.setPeriod("DAY");
chatDetailConfig.setChatDefaultConfig(chatDefaultConfigDetail);
ItemVisibility visibility0 = new ItemVisibility();
chatDetailConfig.setVisibility(visibility0);
return chatDetailConfig;
}
public void addDemoChatConfig_6() {
ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq();
chatConfigBaseReq.setModelId(6L);
ChatDetailConfigReq chatDetailConfig = getChatDetailConfigReq();
List<KnowledgeInfoReq> knowledgeInfos = new ArrayList<>();
KnowledgeInfoReq knowledgeInfoReq = new KnowledgeInfoReq();
knowledgeInfoReq.setSearchEnable(true);
knowledgeInfoReq.setItemId(12L);
knowledgeInfoReq.setBizName("country");
knowledgeInfos.add(knowledgeInfoReq);
KnowledgeInfoReq knowledgeInfoReq2 = new KnowledgeInfoReq();
knowledgeInfoReq2.setSearchEnable(true);
knowledgeInfoReq2.setItemId(13L);
knowledgeInfoReq2.setBizName("gender");
knowledgeInfos.add(knowledgeInfoReq2);
chatDetailConfig.setKnowledgeInfos(knowledgeInfos);
chatConfigBaseReq.setChatDetailConfig(chatDetailConfig);
List<RecommendedQuestionReq> recommendedQuestions = new ArrayList<>();
chatConfigBaseReq.setRecommendedQuestions(recommendedQuestions);
configService.addConfig(chatConfigBaseReq, user);
}
public void addDemoChatConfig_7() {
ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq();
chatConfigBaseReq.setModelId(7L);
ChatDetailConfigReq chatDetailConfig = getChatDetailConfigReq();
List<KnowledgeInfoReq> knowledgeInfos = new ArrayList<>();
KnowledgeInfoReq knowledgeInfoReq = new KnowledgeInfoReq();
knowledgeInfoReq.setSearchEnable(true);
knowledgeInfoReq.setItemId(16L);
knowledgeInfoReq.setBizName("formats");
knowledgeInfos.add(knowledgeInfoReq);
chatDetailConfig.setKnowledgeInfos(knowledgeInfos);
chatConfigBaseReq.setChatDetailConfig(chatDetailConfig);
List<RecommendedQuestionReq> recommendedQuestions = new ArrayList<>();
chatConfigBaseReq.setRecommendedQuestions(recommendedQuestions);
configService.addConfig(chatConfigBaseReq, user);
}
public void addDemoChatConfig_8() {
ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq();
chatConfigBaseReq.setModelId(8L);
ChatDetailConfigReq chatDetailConfig = getChatDetailConfigReq();
List<KnowledgeInfoReq> knowledgeInfos = new ArrayList<>();
KnowledgeInfoReq knowledgeInfoReq = new KnowledgeInfoReq();
knowledgeInfoReq.setSearchEnable(true);
knowledgeInfoReq.setItemId(18L);
knowledgeInfoReq.setBizName("country");
knowledgeInfos.add(knowledgeInfoReq);
KnowledgeInfoReq knowledgeInfoReq2 = new KnowledgeInfoReq();
knowledgeInfoReq2.setSearchEnable(true);
knowledgeInfoReq2.setItemId(19L);
knowledgeInfoReq2.setBizName("languages");
knowledgeInfos.add(knowledgeInfoReq2);
KnowledgeInfoReq knowledgeInfoReq3 = new KnowledgeInfoReq();
knowledgeInfoReq3.setSearchEnable(true);
knowledgeInfoReq3.setItemId(21L);
knowledgeInfoReq3.setBizName("song_name");
knowledgeInfos.add(knowledgeInfoReq3);
chatDetailConfig.setKnowledgeInfos(knowledgeInfos);
chatConfigBaseReq.setChatDetailConfig(chatDetailConfig);
List<RecommendedQuestionReq> recommendedQuestions = new ArrayList<>();
chatConfigBaseReq.setRecommendedQuestions(recommendedQuestions);
configService.addConfig(chatConfigBaseReq, user);
}
private void addPlugin_1() {
Plugin plugin1 = new Plugin();
plugin1.setType("WEB_PAGE");
plugin1.setModelList(Arrays.asList(1L));
plugin1.setViewList(Arrays.asList(1L));
plugin1.setPattern("用于分析超音数的流量概况，包含UV、PV等核心指标的追踪。P.S. 仅作为示例展示，无实际看板");
plugin1.setName("超音数流量分析看板");
PluginParseConfig pluginParseConfig = new PluginParseConfig();
@@ -450,15 +168,14 @@ public class ChatDemoLoader implements CommandLineRunner {
RuleParserTool ruleQueryTool = new RuleParserTool();
ruleQueryTool.setType(AgentToolType.NL2SQL_RULE);
ruleQueryTool.setId("0");
ruleQueryTool.setModelIds(Lists.newArrayList(-1L));
ruleQueryTool.setViewIds(Lists.newArrayList(-1L));
ruleQueryTool.setQueryTypes(Lists.newArrayList(QueryType.METRIC.name()));
agentConfig.getTools().add(ruleQueryTool);
if (demoEnabledNl2SqlLlm) {
LLMParserTool llmParserTool = new LLMParserTool();
llmParserTool.setId("1");
llmParserTool.setType(AgentToolType.NL2SQL_LLM);
llmParserTool.setModelIds(Lists.newArrayList(-1L));
llmParserTool.setViewIds(Lists.newArrayList(-1L));
agentConfig.getTools().add(llmParserTool);
}
agent.setAgentConfig(JSONObject.toJSONString(agentConfig));
@@ -477,7 +194,7 @@ public class ChatDemoLoader implements CommandLineRunner {
RuleParserTool ruleQueryTool = new RuleParserTool();
ruleQueryTool.setId("0");
ruleQueryTool.setType(AgentToolType.NL2SQL_RULE);
ruleQueryTool.setModelIds(Lists.newArrayList(-1L));
ruleQueryTool.setViewIds(Lists.newArrayList(-1L));
ruleQueryTool.setQueryTypes(Lists.newArrayList(QueryType.TAG.name()));
agentConfig.getTools().add(ruleQueryTool);
@@ -485,7 +202,7 @@ public class ChatDemoLoader implements CommandLineRunner {
LLMParserTool llmParserTool = new LLMParserTool();
llmParserTool.setId("1");
llmParserTool.setType(AgentToolType.NL2SQL_LLM);
llmParserTool.setModelIds(Lists.newArrayList(-1L));
llmParserTool.setViewIds(Lists.newArrayList(-1L));
agentConfig.getTools().add(llmParserTool);
}
agent.setAgentConfig(JSONObject.toJSONString(agentConfig));
@@ -507,7 +224,7 @@ public class ChatDemoLoader implements CommandLineRunner {
LLMParserTool llmParserTool = new LLMParserTool();
llmParserTool.setId("1");
llmParserTool.setType(AgentToolType.NL2SQL_LLM);
llmParserTool.setModelIds(Lists.newArrayList(5L, 6L, 7L, 8L));
llmParserTool.setViewIds(Lists.newArrayList(5L, 6L, 7L, 8L));
agentConfig.getTools().add(llmParserTool);
}
@@ -529,7 +246,7 @@ public class ChatDemoLoader implements CommandLineRunner {
LLMParserTool llmParserTool = new LLMParserTool();
llmParserTool.setId("1");
llmParserTool.setType(AgentToolType.NL2SQL_LLM);
llmParserTool.setModelIds(Lists.newArrayList(9L, 10L, 11L, 12L));
llmParserTool.setViewIds(Lists.newArrayList(9L, 10L, 11L, 12L));
agentConfig.getTools().add(llmParserTool);
}

73
launchers/standalone/src/main/java/com/tencent/supersonic/ModelDemoDataLoader.java
View File

@@ -12,6 +12,12 @@ import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.common.pojo.enums.TimeMode;
import com.tencent.supersonic.common.pojo.enums.TypeEnums;
import com.tencent.supersonic.headless.api.pojo.MetricTypeDefaultConfig;
import com.tencent.supersonic.headless.api.pojo.QueryConfig;
import com.tencent.supersonic.headless.api.pojo.TagTypeDefaultConfig;
import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig;
import com.tencent.supersonic.headless.api.pojo.enums.DataType;
import com.tencent.supersonic.headless.api.pojo.enums.DimensionType;
import com.tencent.supersonic.headless.api.pojo.enums.IdentifyType;
@@ -31,17 +37,21 @@ import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams;
import com.tencent.supersonic.headless.api.pojo.MetricParam;
import com.tencent.supersonic.headless.api.pojo.ModelDetail;
import com.tencent.supersonic.headless.api.pojo.RelateDimension;
import com.tencent.supersonic.headless.api.pojo.ViewDetail;
import com.tencent.supersonic.headless.api.pojo.ViewModelConfig;
import com.tencent.supersonic.headless.api.pojo.request.DatabaseReq;
import com.tencent.supersonic.headless.api.pojo.request.DimensionReq;
import com.tencent.supersonic.headless.api.pojo.request.DomainReq;
import com.tencent.supersonic.headless.api.pojo.request.MetricReq;
import com.tencent.supersonic.headless.api.pojo.request.ModelReq;
import com.tencent.supersonic.headless.api.pojo.request.ViewReq;
import com.tencent.supersonic.headless.server.service.DatabaseService;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.DomainService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.ModelRelaService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.ViewService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
@@ -73,6 +83,8 @@ public class ModelDemoDataLoader {
@Autowired
private AuthService authService;
@Autowired
private ViewService viewService;
@Autowired
private DataSourceProperties dataSourceProperties;
public void doRun() {
@@ -91,6 +103,8 @@ public class ModelDemoDataLoader {
updateDimension();
updateMetric();
updateMetric_pv();
addView_1();
addView_2();
addAuthGroup_1();
addAuthGroup_2();
} catch (Exception e) {
@@ -429,6 +443,65 @@ public class ModelDemoDataLoader {
metricService.createMetric(metricReq, user);
}
public void addView_1() {
ViewReq viewReq = new ViewReq();
viewReq.setName("超音数");
viewReq.setBizName("s2");
viewReq.setDomainId(1L);
viewReq.setDescription("包含超音数访问统计相关的指标和维度等");
viewReq.setAdmins(Lists.newArrayList("admin"));
List<ViewModelConfig> viewModelConfigs = Lists.newArrayList(
new ViewModelConfig(1L, true),
new ViewModelConfig(2L, true),
new ViewModelConfig(3L, true));
ViewDetail viewDetail = new ViewDetail();
viewDetail.setViewModelConfigs(viewModelConfigs);
viewReq.setViewDetail(viewDetail);
viewReq.setTypeEnum(TypeEnums.VIEW);
QueryConfig queryConfig = new QueryConfig();
MetricTypeDefaultConfig metricTypeDefaultConfig = new MetricTypeDefaultConfig();
TimeDefaultConfig timeDefaultConfig = new TimeDefaultConfig();
timeDefaultConfig.setTimeMode(TimeMode.RECENT);
timeDefaultConfig.setUnit(7);
metricTypeDefaultConfig.setTimeDefaultConfig(timeDefaultConfig);
queryConfig.setMetricTypeDefaultConfig(metricTypeDefaultConfig);
viewReq.setQueryConfig(queryConfig);
viewService.save(viewReq, User.getFakeUser());
}
public void addView_2() {
ViewReq viewReq = new ViewReq();
viewReq.setName("艺人库");
viewReq.setBizName("singer");
viewReq.setDomainId(2L);
viewReq.setDescription("包含艺人相关标签和指标信息");
viewReq.setAdmins(Lists.newArrayList("admin"));
List<ViewModelConfig> viewModelConfigs =
Lists.newArrayList(new ViewModelConfig(4L, true));
ViewDetail viewDetail = new ViewDetail();
viewDetail.setViewModelConfigs(viewModelConfigs);
viewReq.setViewDetail(viewDetail);
viewReq.setTypeEnum(TypeEnums.VIEW);
QueryConfig queryConfig = new QueryConfig();
TagTypeDefaultConfig tagTypeDefaultConfig = new TagTypeDefaultConfig();
TimeDefaultConfig tagTimeDefaultConfig = new TimeDefaultConfig();
tagTimeDefaultConfig.setTimeMode(TimeMode.LAST);
tagTimeDefaultConfig.setUnit(7);
tagTypeDefaultConfig.setTimeDefaultConfig(tagTimeDefaultConfig);
tagTypeDefaultConfig.setDimensionIds(Lists.newArrayList(4L, 5L, 6L, 7L));
tagTypeDefaultConfig.setMetricIds(Lists.newArrayList(5L));
MetricTypeDefaultConfig metricTypeDefaultConfig = new MetricTypeDefaultConfig();
TimeDefaultConfig timeDefaultConfig = new TimeDefaultConfig();
timeDefaultConfig.setTimeMode(TimeMode.RECENT);
timeDefaultConfig.setUnit(7);
metricTypeDefaultConfig.setTimeDefaultConfig(timeDefaultConfig);
queryConfig.setTagTypeDefaultConfig(tagTypeDefaultConfig);
queryConfig.setMetricTypeDefaultConfig(metricTypeDefaultConfig);
viewReq.setQueryConfig(queryConfig);
viewService.save(viewReq, User.getFakeUser());
}
public void addAuthGroup_1() {
AuthGroup authGroupReq = new AuthGroup();
authGroupReq.setModelId(3L);

10
launchers/standalone/src/main/resources/META-INF/spring.factories
View File

@@ -2,8 +2,7 @@ com.tencent.supersonic.chat.core.mapper.SchemaMapper=\
com.tencent.supersonic.chat.core.mapper.EmbeddingMapper, \
com.tencent.supersonic.chat.core.mapper.KeywordMapper, \
com.tencent.supersonic.chat.core.mapper.QueryFilterMapper, \
com.tencent.supersonic.chat.core.mapper.EntityMapper, \
com.tencent.supersonic.chat.core.mapper.ModelClusterMapper
com.tencent.supersonic.chat.core.mapper.EntityMapper
com.tencent.supersonic.chat.core.parser.SemanticParser=\
com.tencent.supersonic.chat.core.parser.sql.rule.RuleSqlParser, \
@@ -17,8 +16,7 @@ com.tencent.supersonic.chat.core.corrector.SemanticCorrector=\
com.tencent.supersonic.chat.core.corrector.SelectCorrector, \
com.tencent.supersonic.chat.core.corrector.WhereCorrector, \
com.tencent.supersonic.chat.core.corrector.GroupByCorrector, \
com.tencent.supersonic.chat.core.corrector.HavingCorrector, \
com.tencent.supersonic.chat.core.corrector.FromCorrector
com.tencent.supersonic.chat.core.corrector.HavingCorrector
com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor=\
com.tencent.supersonic.chat.server.processor.parse.MetricCheckProcessor, \
@@ -33,8 +31,8 @@ com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor=\
com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter=\
com.tencent.supersonic.chat.core.knowledge.semantic.LocalSemanticInterpreter
com.tencent.supersonic.chat.core.parser.sql.llm.ModelResolver=\
com.tencent.supersonic.chat.core.parser.sql.llm.HeuristicModelResolver
com.tencent.supersonic.chat.core.parser.sql.llm.ViewResolver=\
com.tencent.supersonic.chat.core.parser.sql.llm.HeuristicViewResolver
com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\
com.tencent.supersonic.auth.authentication.interceptor.DefaultAuthenticationInterceptor

23
launchers/standalone/src/main/resources/config.update/sql-update.sql
View File

@@ -168,4 +168,25 @@ CREATE TABLE `s2_app`
--20240115
alter table s2_metric add column `define_type` varchar(50) DEFAULT NULL; -- MEASURE, FIELD, METRIC
update s2_metric set define_type = 'MEASURE';
update s2_metric set define_type = 'MEASURE';
--20240129
CREATE TABLE s2_view(
id BIGINT AUTO_INCREMENT PRIMARY KEY,
domain_id BIGINT,
`name` VARCHAR(255),
biz_name VARCHAR(255),
`description` VARCHAR(255),
`status` INT,
alias VARCHAR(255),
view_detail text,
created_at datetime,
created_by VARCHAR(255),
updated_at datetime,
updated_by VARCHAR(255)
)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
alter table s2_plugin change column model `view` varchar(100);
alter table s2_view_info rename to s2_canvas;
alter table s2_query_stat_info add column `view_id` bigint(20) DEFAULT NULL after `model_id`;

22
launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_2.txt
View File

@@ -5,14 +5,14 @@ dean _1_2 36
john _1_2 50
jack _1_2 38
admin _1_2 70
周杰伦 _4_7 100
陈奕迅 _4_7 100
林俊杰 _4_7 100
张碧晨 _4_7 100
程响 _4_7 100
Taylor#Swift _4_7 100
内地 _4_4 100
欧美 _4_4 100
港台 _4_4 100
流行 _4_6 100
国风 _4_6 100
周杰伦 _2_7 100
陈奕迅 _2_7 100
林俊杰 _2_7 100
张碧晨 _2_7 100
程响 _2_7 100
Taylor#Swift _2_7 100
内地 _2_4 100
欧美 _2_4 100
港台 _2_4 100
流行 _2_6 100
国风 _2_6 100

12
launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_3.txt
View File

@@ -1,6 +1,6 @@
p1 _3_3 52
p2 _3_3 47
p3 _3_3 31
p4 _3_3 36
p5 _3_3 50
p6 _3_3 38
p1 _2_3 52
p2 _2_3 47
p3 _2_3 31
p4 _2_3 36
p5 _2_3 50
p6 _2_3 38

18
launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_4_9.txt
View File

@@ -1,9 +1,9 @@
周杰伦 _4_7 9000
周深 _4_7 8000
周传雄 _4_7 7000
周华建 _4_7 6000
陈奕迅 _4_7 8000
林俊杰 _4_7 7000
张碧晨 _4_7 7000
程响 _4_7 7000
Taylor#Swift _4_7 7000
周杰伦 _2_7 9000
周深 _2_7 8000
周传雄 _2_7 7000
周华建 _2_7 6000
陈奕迅 _2_7 8000
林俊杰 _2_7 7000
张碧晨 _2_7 7000
程响 _2_7 7000
Taylor#Swift _2_7 7000

2
launchers/standalone/src/main/resources/db/data-h2.sql
View File

@@ -12,7 +12,7 @@ values (2 , 2, 'dimension', 'yyyy-MM-dd', DATEADD('DAY', -28, CURRENT_DATE()), D
MERGE INTO s2_available_date_info(`id`,`item_id` ,`type` ,`date_format` ,`start_date` ,`end_date` ,`unavailable_date` ,`created_at` ,`created_by` ,`updated_at` ,`updated_by` )
values (3 , 3, 'dimension', 'yyyy-MM-dd', DATEADD('DAY', -28, CURRENT_DATE()), DATEADD('DAY', -1, CURRENT_DATE()), '[]', '2023-06-01', 'admin', '2023-06-01', 'admin');
MERGE INTO s2_view_info(`id`, `domain_id`, `type`, `config` ,`created_at` ,`created_by` ,`updated_at` ,`updated_by` )
MERGE INTO s2_canvas(`id`, `domain_id`, `type`, `config` ,`created_at` ,`created_by` ,`updated_at` ,`updated_by` )
values (1, 1, 'modelEdgeRelation', '[{"source":"datasource-1","target":"datasource-3","type":"polyline","id":"edge-0.305251275235679741702883718912","style":{"active":{"stroke":"rgb(95, 149, 255)","lineWidth":1},"selected":{"stroke":"rgb(95, 149, 255)","lineWidth":2,"shadowColor":"rgb(95, 149, 255)","shadowBlur":10,"text-shape":{"fontWeight":500}},"highlight":{"stroke":"rgb(95, 149, 255)","lineWidth":2,"text-shape":{"fontWeight":500}},"inactive":{"stroke":"rgb(234, 234, 234)","lineWidth":1},"disable":{"stroke":"rgb(245, 245, 245)","lineWidth":1},"stroke":"#296df3","endArrow":true},"startPoint":{"x":-94,"y":-137.5,"anchorIndex":0,"id":"-94|||-137.5"},"endPoint":{"x":-234,"y":-45,"anchorIndex":1,"id":"-234|||-45"},"sourceAnchor":2,"targetAnchor":1,"label":"模型关系编辑"},{"source":"datasource-1","target":"datasource-2","type":"polyline","id":"edge-0.466237264629309141702883756359","style":{"active":{"stroke":"rgb(95, 149, 255)","lineWidth":1},"selected":{"stroke":"rgb(95, 149, 255)","lineWidth":2,"shadowColor":"rgb(95, 149, 255)","shadowBlur":10,"text-shape":{"fontWeight":500}},"highlight":{"stroke":"rgb(95, 149, 255)","lineWidth":2,"text-shape":{"fontWeight":500}},"inactive":{"stroke":"rgb(234, 234, 234)","lineWidth":1},"disable":{"stroke":"rgb(245, 245, 245)","lineWidth":1},"stroke":"#296df3","endArrow":true},"startPoint":{"x":-12,"y":-137.5,"anchorIndex":1,"id":"-12|||-137.5"},"endPoint":{"x":85,"y":31.5,"anchorIndex":0,"id":"85|||31.5"},"sourceAnchor":1,"targetAnchor":2,"label":"模型关系编辑"}]', '2023-06-01', 'admin', '2023-06-01', 'admin');
-- sample data

2
launchers/standalone/src/main/resources/db/data-mysql.sql
View File

@@ -12,7 +12,7 @@ VALUES (2, 'dimension', 'yyyy-MM-dd', DATE_SUB(CURRENT_DATE(), INTERVAL 28 DAY),
INSERT INTO s2_available_date_info (`item_id`, `type`, `date_format`, `start_date`, `end_date`, `unavailable_date`, `created_at`, `created_by`, `updated_at`, `updated_by`)
VALUES (3, 'dimension', 'yyyy-MM-dd', DATE_SUB(CURRENT_DATE(), INTERVAL 28 DAY), DATE_SUB(CURRENT_DATE(), INTERVAL 1 DAY), '[]', '2023-06-01', 'admin', '2023-06-01', 'admin');
insert into s2_view_info(`id`, `domain_id`, `type`, `config` ,`created_at` ,`created_by` ,`updated_at` ,`updated_by` )
insert into s2_canvas(`id`, `domain_id`, `type`, `config` ,`created_at` ,`created_by` ,`updated_at` ,`updated_by` )
values (1, 1, 'modelEdgeRelation', '[{"source":"datasource-1","target":"datasource-3","type":"polyline","id":"edge-0.305251275235679741702883718912","style":{"active":{"stroke":"rgb(95, 149, 255)","lineWidth":1},"selected":{"stroke":"rgb(95, 149, 255)","lineWidth":2,"shadowColor":"rgb(95, 149, 255)","shadowBlur":10,"text-shape":{"fontWeight":500}},"highlight":{"stroke":"rgb(95, 149, 255)","lineWidth":2,"text-shape":{"fontWeight":500}},"inactive":{"stroke":"rgb(234, 234, 234)","lineWidth":1},"disable":{"stroke":"rgb(245, 245, 245)","lineWidth":1},"stroke":"#296df3","endArrow":true},"startPoint":{"x":-94,"y":-137.5,"anchorIndex":0,"id":"-94|||-137.5"},"endPoint":{"x":-234,"y":-45,"anchorIndex":1,"id":"-234|||-45"},"sourceAnchor":2,"targetAnchor":1,"label":"模型关系编辑"},{"source":"datasource-1","target":"datasource-2","type":"polyline","id":"edge-0.466237264629309141702883756359","style":{"active":{"stroke":"rgb(95, 149, 255)","lineWidth":1},"selected":{"stroke":"rgb(95, 149, 255)","lineWidth":2,"shadowColor":"rgb(95, 149, 255)","shadowBlur":10,"text-shape":{"fontWeight":500}},"highlight":{"stroke":"rgb(95, 149, 255)","lineWidth":2,"text-shape":{"fontWeight":500}},"inactive":{"stroke":"rgb(234, 234, 234)","lineWidth":1},"disable":{"stroke":"rgb(245, 245, 245)","lineWidth":1},"stroke":"#296df3","endArrow":true},"startPoint":{"x":-12,"y":-137.5,"anchorIndex":1,"id":"-12|||-137.5"},"endPoint":{"x":85,"y":31.5,"anchorIndex":0,"id":"85|||31.5"},"sourceAnchor":1,"targetAnchor":2,"label":"模型关系编辑"}]', '2023-06-01', 'admin', '2023-06-01', 'admin');
-- sample data

27
launchers/standalone/src/main/resources/db/schema-h2.sql
View File

@@ -230,8 +230,7 @@ CREATE TABLE IF NOT EXISTS s2_model_rela
PRIMARY KEY (`id`)
);
create table IF NOT EXISTS s2_view_info
(
create table IF NOT EXISTS `s2_canvas` (
id INT auto_increment,
domain_id INT null,
type varchar(20) null comment 'model、dimension、metric',
@@ -242,13 +241,14 @@ create table IF NOT EXISTS s2_view_info
updated_by varchar(100) not null,
PRIMARY KEY (`id`)
);
COMMENT ON TABLE s2_view_info IS 'view information table';
COMMENT ON TABLE s2_canvas IS 'canvas table';
CREATE TABLE IF NOT EXISTS `s2_query_stat_info` (
`id` INT NOT NULL AUTO_INCREMENT,
`trace_id` varchar(200) DEFAULT NULL, -- query unique identifier
`model_id` INT DEFAULT NULL,
`view_id` INT DEFAULT NULL,
`user` varchar(200) DEFAULT NULL,
`created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ,
`query_type` varchar(200) DEFAULT NULL, -- the corresponding scene
@@ -326,7 +326,7 @@ CREATE TABLE IF NOT EXISTS `s2_plugin`
(
`id` INT AUTO_INCREMENT,
`type` varchar(50) NULL,
`model` varchar(100) NULL,
`view` varchar(100) NULL,
`pattern` varchar(500) NULL,
`parse_mode` varchar(100) NULL,
`parse_mode_config` LONGVARCHAR NULL,
@@ -544,3 +544,22 @@ CREATE TABLE IF NOT EXISTS `s2_app` (
updated_at TIMESTAMP,
updated_by VARCHAR(255)
);
CREATE TABLE IF NOT EXISTS `s2_view` (
id BIGINT AUTO_INCREMENT PRIMARY KEY,
domain_id BIGINT,
`name` VARCHAR(255),
biz_name VARCHAR(255),
description VARCHAR(255),
status INT,
alias VARCHAR(255),
view_detail TEXT,
created_at TIMESTAMP,
created_by VARCHAR(255),
updated_at TIMESTAMP,
updated_by VARCHAR(255),
filter_sql VARCHAR(1000),
query_config VARCHAR(3000),
`admin` varchar(3000) DEFAULT NULL,
`admin_org` varchar(3000) DEFAULT NULL
);

47
launchers/standalone/src/main/resources/db/schema-mysql.sql
View File

@@ -1,4 +1,3 @@
-------demo for semantic and chat
CREATE TABLE `s2_user_department` (
`user_name` varchar(200) NOT NULL,
`department` varchar(200) NOT NULL
@@ -336,7 +335,7 @@ CREATE TABLE `s2_model` (
CREATE TABLE `s2_plugin` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`type` varchar(50) CHARACTER SET utf8 COLLATE utf8_unicode_ci DEFAULT NULL COMMENT 'DASHBOARD,WIDGET,URL',
`model` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`view` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`pattern` varchar(500) CHARACTER SET utf8 COLLATE utf8_unicode_ci DEFAULT NULL,
`parse_mode` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`parse_mode_config` text COLLATE utf8mb4_unicode_ci,
@@ -354,6 +353,7 @@ CREATE TABLE `s2_query_stat_info` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
`trace_id` varchar(200) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '查询标识',
`model_id` bigint(20) DEFAULT NULL,
`view_id` bigint(20) DEFAULT NULL,
`user` varchar(200) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '执行sql的用户',
`created_at` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
`query_type` varchar(200) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '查询对应的场景',
@@ -406,17 +406,18 @@ CREATE TABLE `s2_semantic_pasre_info` (
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='语义层sql解析信息表';
CREATE TABLE `s2_view_info` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`domain_id` bigint(20) DEFAULT NULL,
`type` varchar(20) DEFAULT NULL COMMENT 'datasource、dimension、metric',
`config` text COMMENT 'config detail',
`created_at` datetime DEFAULT NULL,
`created_by` varchar(100) DEFAULT NULL,
`updated_at` datetime DEFAULT NULL,
`updated_by` varchar(100) NOT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `s2_canvas`
(
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`domain_id` bigint(20) DEFAULT NULL,
`type` varchar(20) DEFAULT NULL COMMENT 'datasource、dimension、metric',
`config` text COMMENT 'config detail',
`created_at` datetime DEFAULT NULL,
`created_by` varchar(100) DEFAULT NULL,
`updated_at` datetime DEFAULT NULL,
`updated_by` varchar(100) NOT NULL,
PRIMARY KEY (`id`)
) ENGINE = InnoDB DEFAULT CHARSET = utf8;
create table s2_user
(
@@ -481,4 +482,24 @@ CREATE TABLE `s2_app`
`updated_at` datetime null,
`created_by` varchar(255) null,
`updated_by` varchar(255) null
)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
CREATE TABLE s2_view
(
id BIGINT AUTO_INCREMENT PRIMARY KEY,
domain_id BIGINT,
`name` VARCHAR(255),
biz_name VARCHAR(255),
`description` VARCHAR(255),
`status` INT,
alias VARCHAR(255),
view_detail text,
created_at datetime,
created_by VARCHAR(255),
updated_at datetime,
updated_by VARCHAR(255),
filter_sql VARCHAR(1000),
query_config VARCHAR(3000),
`admin` varchar(3000) DEFAULT NULL,
`admin_org` varchar(3000) DEFAULT NULL
)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

11
launchers/standalone/src/test/java/com/tencent/supersonic/chat/BaseTest.java
View File

@@ -1,7 +1,5 @@
package com.tencent.supersonic.chat;
import static org.junit.Assert.assertEquals;
import com.tencent.supersonic.BaseApplication;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
@@ -16,13 +14,16 @@ import com.tencent.supersonic.chat.server.service.AgentService;
import com.tencent.supersonic.chat.server.service.ChatService;
import com.tencent.supersonic.chat.server.service.ConfigService;
import com.tencent.supersonic.chat.server.service.QueryService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.test.mock.mockito.MockBean;
import java.time.LocalDate;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.test.mock.mockito.MockBean;
import static org.junit.Assert.assertEquals;
public class BaseTest extends BaseApplication {

27
launchers/standalone/src/test/java/com/tencent/supersonic/chat/MetricInterpretTest.java
View File

@@ -1,27 +0,0 @@
package com.tencent.supersonic.chat;
import com.tencent.supersonic.BaseApplication;
import com.tencent.supersonic.chat.core.query.llm.analytics.LLMAnswerResp;
import com.tencent.supersonic.chat.server.service.AgentService;
import com.tencent.supersonic.common.config.EmbeddingConfig;
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.mock.mockito.MockBean;
public class MetricInterpretTest extends BaseApplication {
@MockBean
private AgentService agentService;
@MockBean
private EmbeddingConfig embeddingConfig;
@Test
public void testMetricInterpret() throws Exception {
MockConfiguration.mockMetricAgent(agentService);
MockConfiguration.mockEmbeddingUrl(embeddingConfig);
LLMAnswerResp lLmAnswerResp = new LLMAnswerResp();
lLmAnswerResp.setAssistantMessage("alice最近在超音数的访问情况有增多");
}
}

12
launchers/standalone/src/test/java/com/tencent/supersonic/chat/MetricTest.java
View File

@@ -1,8 +1,5 @@
package com.tencent.supersonic.chat;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.SUM;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
@@ -15,13 +12,17 @@ import com.tencent.supersonic.util.DataUtils;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import org.junit.Assert;
import org.junit.jupiter.api.Test;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import org.junit.Assert;
import org.junit.jupiter.api.Test;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.SUM;
public class MetricTest extends BaseTest {
@@ -150,7 +151,6 @@ public class MetricTest extends BaseTest {
expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("访问次数"));
expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("人均访问次数"));
expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("访问用户数"));
expectedParseInfo.getDimensions().add(DataUtils.getSchemaElement("用户"));
expectedParseInfo.setDateInfo(DataUtils.getDateConf(3, DateConf.DateMode.RECENT, "DAY"));

6
launchers/standalone/src/test/java/com/tencent/supersonic/chat/MockConfiguration.java
View File

@@ -1,18 +1,18 @@
package com.tencent.supersonic.chat;
import static org.mockito.Mockito.when;
import com.google.common.collect.Lists;
import com.tencent.supersonic.util.DataUtils;
import com.tencent.supersonic.chat.core.plugin.PluginManager;
import com.tencent.supersonic.chat.server.service.AgentService;
import com.tencent.supersonic.common.config.EmbeddingConfig;
import com.tencent.supersonic.common.util.embedding.Retrieval;
import com.tencent.supersonic.common.util.embedding.RetrieveQueryResult;
import com.tencent.supersonic.util.DataUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Configuration;
import static org.mockito.Mockito.when;
@Configuration
@Slf4j
public class MockConfiguration {

12
launchers/standalone/src/test/java/com/tencent/supersonic/chat/TagTest.java
View File

@@ -1,7 +1,5 @@
package com.tencent.supersonic.chat;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
@@ -11,11 +9,13 @@ import com.tencent.supersonic.chat.core.query.rule.tag.TagFilterQuery;
import com.tencent.supersonic.util.DataUtils;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.DateConf.DateMode;
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.List;
import org.junit.jupiter.api.Test;
public class TagTest extends BaseTest {
@@ -29,7 +29,7 @@ public class TagTest extends BaseTest {
expectedResult.setChatContext(expectedParseInfo);
expectedResult.setQueryMode(MetricTagQuery.QUERY_MODE);
expectedParseInfo.setAggType(NONE);
expectedParseInfo.setAggType(AggregateTypeEnum.NONE);
QueryFilter dimensionFilter = DataUtils.getFilter("singer_name", FilterOperatorEnum.EQUALS, "周杰伦", "歌手名", 7L);
expectedParseInfo.getDimensionFilters().add(dimensionFilter);
@@ -53,7 +53,7 @@ public class TagTest extends BaseTest {
expectedResult.setChatContext(expectedParseInfo);
expectedResult.setQueryMode(TagFilterQuery.QUERY_MODE);
expectedParseInfo.setAggType(NONE);
expectedParseInfo.setAggType(AggregateTypeEnum.NONE);
List<String> list = new ArrayList<>();
list.add("流行");
@@ -73,7 +73,7 @@ public class TagTest extends BaseTest {
expectedParseInfo.getDimensions().add(dim3);
expectedParseInfo.getDimensions().add(dim4);
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.BETWEEN, startDay, startDay));
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.BETWEEN, startDay, startDay, 7));
expectedParseInfo.setQueryType(QueryType.TAG);
assertQueryResult(expectedResult, actualResult);

4
launchers/standalone/src/test/java/com/tencent/supersonic/chat/mapper/MapperTest.java
View File

@@ -1,7 +1,5 @@
package com.tencent.supersonic.chat.mapper;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
@@ -15,6 +13,8 @@ import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.QueryType;
import org.junit.jupiter.api.Test;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE;
public class MapperTest extends BaseTest {
@Test

32
launchers/standalone/src/test/java/com/tencent/supersonic/chat/model/MetricServiceImplTest.java
View File

@@ -1,32 +0,0 @@
package com.tencent.supersonic.chat.model;
import com.google.common.collect.Lists;
import com.tencent.supersonic.BaseApplication;
import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
import com.tencent.supersonic.headless.api.pojo.enums.MetricType;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.server.pojo.MetricFilter;
import com.tencent.supersonic.headless.server.service.MetricService;
import java.util.List;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
public class MetricServiceImplTest extends BaseApplication {
@Autowired
protected MetricService metricService;
@Test
void getMetrics() {
MetricFilter metricFilter = new MetricFilter();
metricFilter.setType(MetricType.ATOMIC.name());
metricFilter.setModelIds(Lists.newArrayList(1L));
metricFilter.setSensitiveLevel(SensitiveLevelEnum.LOW.ordinal());
List<MetricResp> metricResps = metricService.getMetrics(metricFilter);
Assertions.assertTrue(metricResps.stream().noneMatch(metricResp -> metricResp.getModelId().equals(2L)));
Assertions.assertTrue(metricResps.stream().noneMatch(metricResp ->
metricResp.getSensitiveLevel().equals(SensitiveLevelEnum.HIGH.ordinal())));
}
}

11
launchers/standalone/src/test/java/com/tencent/supersonic/headless/ExplainTest.java
View File

@@ -1,9 +1,6 @@
package com.tencent.supersonic.headless;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.core.utils.QueryReqBuilder;
import com.tencent.supersonic.headless.api.pojo.enums.QueryType;
@@ -12,9 +9,13 @@ import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
import com.tencent.supersonic.headless.api.pojo.response.ExplainResp;
import com.tencent.supersonic.util.DataUtils;
import java.util.Arrays;
import org.junit.jupiter.api.Test;
import java.util.Arrays;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
public class ExplainTest extends BaseTest {
@Test
@@ -22,7 +23,7 @@ public class ExplainTest extends BaseTest {
String sql = "SELECT 部门, SUM(访问次数) AS 访问次数 FROM 超音数PVUV统计 GROUP BY 部门 ";
ExplainSqlReq<QuerySqlReq> explainSqlReq = ExplainSqlReq.<QuerySqlReq>builder()
.queryTypeEnum(QueryType.SQL)
.queryReq(QueryReqBuilder.buildS2SQLReq(sql, DataUtils.getMetricAgentIModelIds()))
.queryReq(QueryReqBuilder.buildS2SQLReq(sql, DataUtils.getMetricAgentView()))
.build();
ExplainResp explain = queryService.explain(explainSqlReq, User.getFakeUser());
assertNotNull(explain);

19
launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java
View File

@@ -1,7 +1,5 @@
package com.tencent.supersonic.util;
import static java.time.LocalDate.now;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
@@ -18,6 +16,8 @@ import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import java.util.HashSet;
import java.util.Set;
import static java.time.LocalDate.now;
public class DataUtils {
public static final Integer metricAgentId = 1;
@@ -92,6 +92,15 @@ public class DataUtils {
return dateInfo;
}
public static DateConf getDateConf(DateConf.DateMode dateMode, String startDate, String endDate, int unit) {
DateConf dateInfo = new DateConf();
dateInfo.setDateMode(dateMode);
dateInfo.setStartDate(startDate);
dateInfo.setEndDate(endDate);
dateInfo.setUnit(unit);
return dateInfo;
}
public static Agent getMetricAgent() {
Agent agent = new Agent();
agent.setId(1);
@@ -117,7 +126,7 @@ public class DataUtils {
private static RuleParserTool getRuleQueryTool() {
RuleParserTool ruleQueryTool = new RuleParserTool();
ruleQueryTool.setType(AgentToolType.NL2SQL_RULE);
ruleQueryTool.setModelIds(Lists.newArrayList(-1L));
ruleQueryTool.setViewIds(Lists.newArrayList(-1L));
ruleQueryTool.setQueryModes(Lists.newArrayList("METRIC_TAG", "METRIC_FILTER", "METRIC_MODEL",
"TAG_DETAIL", "TAG_LIST_FILTER", "TAG_ID"));
return ruleQueryTool;
@@ -137,4 +146,8 @@ public class DataUtils {
result.add(3L);
return result;
}
public static Long getMetricAgentView() {
return 1L;
}
}

10
launchers/standalone/src/test/resources/META-INF/spring.factories
View File

@@ -2,8 +2,7 @@ com.tencent.supersonic.chat.core.mapper.SchemaMapper=\
com.tencent.supersonic.chat.core.mapper.EmbeddingMapper, \
com.tencent.supersonic.chat.core.mapper.KeywordMapper, \
com.tencent.supersonic.chat.core.mapper.QueryFilterMapper, \
com.tencent.supersonic.chat.core.mapper.EntityMapper, \
com.tencent.supersonic.chat.core.mapper.ModelClusterMapper
com.tencent.supersonic.chat.core.mapper.EntityMapper
com.tencent.supersonic.chat.core.parser.SemanticParser=\
com.tencent.supersonic.chat.core.parser.sql.rule.RuleSqlParser, \
@@ -17,8 +16,7 @@ com.tencent.supersonic.chat.core.corrector.SemanticCorrector=\
com.tencent.supersonic.chat.core.corrector.SelectCorrector, \
com.tencent.supersonic.chat.core.corrector.WhereCorrector, \
com.tencent.supersonic.chat.core.corrector.GroupByCorrector, \
com.tencent.supersonic.chat.core.corrector.HavingCorrector, \
com.tencent.supersonic.chat.core.corrector.FromCorrector
com.tencent.supersonic.chat.core.corrector.HavingCorrector
com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor=\
com.tencent.supersonic.chat.server.processor.parse.MetricCheckProcessor, \
@@ -33,8 +31,8 @@ com.tencent.supersonic.chat.server.processor.parse.ParseResultProcessor=\
com.tencent.supersonic.chat.core.knowledge.semantic.SemanticInterpreter=\
com.tencent.supersonic.chat.core.knowledge.semantic.LocalSemanticInterpreter
com.tencent.supersonic.chat.core.parser.sql.llm.ModelResolver=\
com.tencent.supersonic.chat.core.parser.sql.llm.HeuristicModelResolver
com.tencent.supersonic.chat.core.parser.sql.llm.ViewResolver=\
com.tencent.supersonic.chat.core.parser.sql.llm.HeuristicViewResolver
com.tencent.supersonic.auth.authentication.interceptor.AuthenticationInterceptor=\
com.tencent.supersonic.auth.authentication.interceptor.DefaultAuthenticationInterceptor

8
launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_10_20.txt Normal file
View File

@@ -0,0 +1,8 @@
阿里云 _10_20 5
天猫 _10_20 5
腾讯游戏 _10_20 5
度小满 _10_20 5
京东金融 _10_20 5

8
launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_10_22.txt Normal file
View File

@@ -0,0 +1,8 @@
张勇 _10_22 5
马化腾 _10_22 5
朱光 _10_22 5
刘强东 _10_22 5

22
launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_2.txt
View File

@@ -5,14 +5,14 @@ dean _1_2 36
john _1_2 50
jack _1_2 38
admin _1_2 70
周杰伦 _4_7 100
陈奕迅 _4_7 100
林俊杰 _4_7 100
张碧晨 _4_7 100
程响 _4_7 100
Taylor#Swift _4_7 100
内地 _4_4 100
欧美 _4_4 100
港台 _4_4 100
流行 _4_6 100
国风 _4_6 100
周杰伦 _2_7 100
陈奕迅 _2_7 100
林俊杰 _2_7 100
张碧晨 _2_7 100
程响 _2_7 100
Taylor#Swift _2_7 100
内地 _2_4 100
欧美 _2_4 100
港台 _2_4 100
流行 _2_6 100
国风 _2_6 100

12
launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_1_3.txt
View File

@@ -1,6 +1,6 @@
p1 _3_3 52
p2 _3_3 47
p3 _3_3 31
p4 _3_3 36
p5 _3_3 50
p6 _3_3 38
p1 _2_3 52
p2 _2_3 47
p3 _2_3 31
p4 _2_3 36
p5 _2_3 50
p6 _2_3 38

18
launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_4_9.txt
View File

@@ -1,9 +1,9 @@
周杰伦 _4_7 9000
周深 _4_7 8000
周传雄 _4_7 7000
周华建 _4_7 6000
陈奕迅 _4_7 8000
林俊杰 _4_7 7000
张碧晨 _4_7 7000
程响 _4_7 7000
Taylor#Swift _4_7 7000
周杰伦 _2_7 9000
周深 _2_7 8000
周传雄 _2_7 7000
周华建 _2_7 6000
陈奕迅 _2_7 8000
林俊杰 _2_7 7000
张碧晨 _2_7 7000
程响 _2_7 7000
Taylor#Swift _2_7 7000

5
launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_9_15.txt Normal file
View File

@@ -0,0 +1,5 @@
百度集团 _9_15 5
阿里巴巴集团 _9_15 5
深圳市腾讯计算机系统有限公司 _9_15 5
北京京东世纪贸易有限公司 _9_15 5
网易公司 _9_15 5

4
launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_9_16.txt Normal file
View File

@@ -0,0 +1,4 @@
北京 _9_16 5
杭州 _9_16 5
深圳 _9_16 5

7
launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_9_18.txt Normal file
View File

@@ -0,0 +1,7 @@
李彦宏 _9_18 5
马云 _9_18 5
马化腾 _9_18 5
刘强东 _9_18 5
丁磊 _9_18 5

7
launchers/standalone/src/test/resources/data/dictionary/custom/DimValue_9_19.txt Normal file
View File

@@ -0,0 +1,7 @@
李彦宏 _9_19 5
张勇 _9_19 5
刘炽平 _9_19 5
刘强东 _9_19 5
丁磊 _9_19 5

2
launchers/standalone/src/test/resources/db/data-h2.sql
View File

@@ -14,7 +14,7 @@ MERGE INTO s2_available_date_info(`id`,`item_id` ,`type` ,`date_format` ,`sta
values (3 , 3, 'dimension', 'yyyy-MM-dd', DATEADD('DAY', -28, CURRENT_DATE()), DATEADD('DAY', -1, CURRENT_DATE()), '[]', '2023-06-01', 'admin', '2023-06-01', 'admin');
MERGE INTO s2_view_info(`id`, `domain_id`, `type`, `config` ,`created_at` ,`created_by` ,`updated_at` ,`updated_by` )
MERGE INTO s2_canvas(`id`, `domain_id`, `type`, `config` ,`created_at` ,`created_by` ,`updated_at` ,`updated_by` )
values (1, 1, 'modelEdgeRelation', '[{"source":"datasource-1","target":"datasource-3","type":"polyline","id":"edge-0.305251275235679741702883718912","style":{"active":{"stroke":"rgb(95, 149, 255)","lineWidth":1},"selected":{"stroke":"rgb(95, 149, 255)","lineWidth":2,"shadowColor":"rgb(95, 149, 255)","shadowBlur":10,"text-shape":{"fontWeight":500}},"highlight":{"stroke":"rgb(95, 149, 255)","lineWidth":2,"text-shape":{"fontWeight":500}},"inactive":{"stroke":"rgb(234, 234, 234)","lineWidth":1},"disable":{"stroke":"rgb(245, 245, 245)","lineWidth":1},"stroke":"#296df3","endArrow":true},"startPoint":{"x":-94,"y":-137.5,"anchorIndex":0,"id":"-94|||-137.5"},"endPoint":{"x":-234,"y":-45,"anchorIndex":1,"id":"-234|||-45"},"sourceAnchor":2,"targetAnchor":1,"label":"模型关系编辑"},{"source":"datasource-1","target":"datasource-2","type":"polyline","id":"edge-0.466237264629309141702883756359","style":{"active":{"stroke":"rgb(95, 149, 255)","lineWidth":1},"selected":{"stroke":"rgb(95, 149, 255)","lineWidth":2,"shadowColor":"rgb(95, 149, 255)","shadowBlur":10,"text-shape":{"fontWeight":500}},"highlight":{"stroke":"rgb(95, 149, 255)","lineWidth":2,"text-shape":{"fontWeight":500}},"inactive":{"stroke":"rgb(234, 234, 234)","lineWidth":1},"disable":{"stroke":"rgb(245, 245, 245)","lineWidth":1},"stroke":"#296df3","endArrow":true},"startPoint":{"x":-12,"y":-137.5,"anchorIndex":1,"id":"-12|||-137.5"},"endPoint":{"x":85,"y":31.5,"anchorIndex":0,"id":"85|||31.5"},"sourceAnchor":1,"targetAnchor":2,"label":"模型关系编辑"}]', '2023-06-01', 'admin', '2023-06-01', 'admin');
-- sample data

26
launchers/standalone/src/test/resources/db/schema-h2.sql
View File

@@ -230,7 +230,7 @@ CREATE TABLE IF NOT EXISTS s2_model_rela
PRIMARY KEY (`id`)
);
create table IF NOT EXISTS s2_view_info
create table IF NOT EXISTS s2_canvas
(
id INT auto_increment,
domain_id INT null,
@@ -242,13 +242,14 @@ create table IF NOT EXISTS s2_view_info
updated_by varchar(100) not null,
PRIMARY KEY (`id`)
);
COMMENT ON TABLE s2_view_info IS 'view information table';
COMMENT ON TABLE s2_canvas IS 'view information table';
CREATE TABLE IF NOT EXISTS `s2_query_stat_info` (
`id` INT NOT NULL AUTO_INCREMENT,
`trace_id` varchar(200) DEFAULT NULL, -- query unique identifier
`model_id` INT DEFAULT NULL,
`view_id` INT DEFAULT NULL,
`user` varchar(200) DEFAULT NULL,
`created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ,
`query_type` varchar(200) DEFAULT NULL, -- the corresponding scene
@@ -326,7 +327,7 @@ CREATE TABLE IF NOT EXISTS `s2_plugin`
(
`id` INT AUTO_INCREMENT,
`type` varchar(50) NULL,
`model` varchar(100) NULL,
`view` varchar(100) NULL,
`pattern` varchar(500) NULL,
`parse_mode` varchar(100) NULL,
`parse_mode_config` LONGVARCHAR NULL,
@@ -498,4 +499,23 @@ CREATE TABLE IF NOT EXISTS `s2_app` (
created_by VARCHAR(255),
updated_at TIMESTAMP,
updated_by VARCHAR(255)
);
CREATE TABLE IF NOT EXISTS `s2_view` (
id BIGINT AUTO_INCREMENT PRIMARY KEY,
domain_id BIGINT,
`name` VARCHAR(255),
biz_name VARCHAR(255),
description VARCHAR(255),
status INT,
alias VARCHAR(255),
view_detail TEXT,
created_at TIMESTAMP,
created_by VARCHAR(255),
updated_at TIMESTAMP,
updated_by VARCHAR(255),
filter_sql VARCHAR(1000),
query_config VARCHAR(3000),
`admin` varchar(3000) DEFAULT NULL,
`admin_org` varchar(3000) DEFAULT NULL
);