topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2026-04-29 12:34:28 +08:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

(improvement) Move out the datasource and merge the datasource with the model, and adapt the chat module (#423)

Browse Source 
Co-authored-by: jolunoluo <jolunoluo@tencent.com>
This commit is contained in:
jipeli
2023-11-27 11:05:24 +08:00
committed by GitHub
parent 0534053ff9
commit 27bb1b322e
190 changed files with 3900 additions and 10561 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java
View File

@@ -40,6 +40,7 @@ import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@Component
@@ -109,12 +110,12 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
public void addDemoChatConfig_1() {
ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq();
chatConfigBaseReq.setModelId(1L);
chatConfigBaseReq.setModelId(2L);
ChatDetailConfigReq chatDetailConfig = new ChatDetailConfigReq();
ChatDefaultConfigReq chatDefaultConfigDetail = new ChatDefaultConfigReq();
List<Long> dimensionIds0 = Arrays.asList(1L, 2L);
List<Long> metricIds0 = Arrays.asList(1L);
List<Long> dimensionIds0 = Collections.singletonList(3L);
List<Long> metricIds0 = Arrays.asList(1L, 2L);
chatDefaultConfigDetail.setDimensionIds(dimensionIds0);
chatDefaultConfigDetail.setMetricIds(metricIds0);
chatDefaultConfigDetail.setUnit(7);
@@ -126,8 +127,8 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
ChatAggConfigReq chatAggConfig = new ChatAggConfigReq();
ChatDefaultConfigReq chatDefaultConfigAgg = new ChatDefaultConfigReq();
List<Long> dimensionIds1 = Arrays.asList(1L, 2L);
List<Long> metricIds1 = Arrays.asList(1L);
List<Long> dimensionIds1 = Arrays.asList(3L);
List<Long> metricIds1 = Arrays.asList(1L, 2L);
chatDefaultConfigAgg.setDimensionIds(dimensionIds1);
chatDefaultConfigAgg.setMetricIds(metricIds1);
chatDefaultConfigAgg.setUnit(7);
@@ -138,34 +139,60 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
chatAggConfig.setVisibility(visibility1);
List<KnowledgeInfoReq> knowledgeInfos = new ArrayList<>();
KnowledgeInfoReq knowledgeInfoReq = new KnowledgeInfoReq();
knowledgeInfoReq.setItemId(1L);
knowledgeInfoReq.setItemId(3L);
knowledgeInfoReq.setSearchEnable(true);
knowledgeInfos.add(knowledgeInfoReq);
KnowledgeInfoReq knowledgeInfoReq2 = new KnowledgeInfoReq();
knowledgeInfoReq2.setItemId(2L);
knowledgeInfoReq2.setSearchEnable(true);
knowledgeInfos.add(knowledgeInfoReq2);
chatAggConfig.setKnowledgeInfos(knowledgeInfos);
chatConfigBaseReq.setChatAggConfig(chatAggConfig);
List<RecommendedQuestionReq> recommendedQuestions = new ArrayList<>();
recommendedQuestions.add(new RecommendedQuestionReq("超音数访问次数"));
recommendedQuestions.add(new RecommendedQuestionReq("近15天超音数访问次数汇总"));
recommendedQuestions.add(new RecommendedQuestionReq("按部门统计超音数的访问人数"));
recommendedQuestions.add(new RecommendedQuestionReq("对比alice和lucy的停留时长"));
recommendedQuestions.add(new RecommendedQuestionReq("超音数访问次数最高的部门"));
chatConfigBaseReq.setRecommendedQuestions(recommendedQuestions);
configService.addConfig(chatConfigBaseReq, user);
}
public void addDemoChatConfig_2() {
ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq();
chatConfigBaseReq.setModelId(2L);
chatConfigBaseReq.setModelId(3L);
ChatDetailConfigReq chatDetailConfig = new ChatDetailConfigReq();
ChatDefaultConfigReq chatDefaultConfigDetail = new ChatDefaultConfigReq();
List<Long> dimensionIds0 = Arrays.asList(4L, 5L, 6L, 7L);
List<Long> dimensionIds0 = Arrays.asList(4L, 5L);
List<Long> metricIds0 = Arrays.asList(3L);
chatDefaultConfigDetail.setDimensionIds(dimensionIds0);
chatDefaultConfigDetail.setMetricIds(metricIds0);
chatDefaultConfigDetail.setUnit(7);
chatDefaultConfigDetail.setPeriod("DAY");
chatDetailConfig.setChatDefaultConfig(chatDefaultConfigDetail);
ItemVisibility visibility0 = new ItemVisibility();
chatDetailConfig.setVisibility(visibility0);
chatConfigBaseReq.setChatDetailConfig(chatDetailConfig);
ChatAggConfigReq chatAggConfig = new ChatAggConfigReq();
ChatDefaultConfigReq chatDefaultConfigAgg = new ChatDefaultConfigReq();
List<Long> dimensionIds1 = Arrays.asList(4L, 5L);
List<Long> metricIds1 = Arrays.asList(3L);
chatDefaultConfigAgg.setDimensionIds(dimensionIds1);
chatDefaultConfigAgg.setMetricIds(metricIds1);
chatDefaultConfigAgg.setUnit(7);
chatDefaultConfigAgg.setPeriod("DAY");
chatDefaultConfigAgg.setTimeMode(ChatDefaultConfigReq.TimeMode.RECENT);
chatAggConfig.setChatDefaultConfig(chatDefaultConfigAgg);
ItemVisibility visibility1 = new ItemVisibility();
chatAggConfig.setVisibility(visibility1);
List<KnowledgeInfoReq> knowledgeInfos = new ArrayList<>();
KnowledgeInfoReq knowledgeInfoReq = new KnowledgeInfoReq();
knowledgeInfoReq.setItemId(5L);
knowledgeInfoReq.setSearchEnable(true);
knowledgeInfos.add(knowledgeInfoReq);
chatAggConfig.setKnowledgeInfos(knowledgeInfos);
chatConfigBaseReq.setChatAggConfig(chatAggConfig);
configService.addConfig(chatConfigBaseReq, user);
}
public void addDemoChatConfig_3() {
ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq();
chatConfigBaseReq.setModelId(4L);
ChatDetailConfigReq chatDetailConfig = new ChatDetailConfigReq();
ChatDefaultConfigReq chatDefaultConfigDetail = new ChatDefaultConfigReq();
List<Long> dimensionIds0 = Arrays.asList(6L, 7L, 8L, 9L);
List<Long> metricIds0 = Arrays.asList(4L);
chatDefaultConfigDetail.setDimensionIds(dimensionIds0);
chatDefaultConfigDetail.setMetricIds(metricIds0);
@@ -178,7 +205,7 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
ChatAggConfigReq chatAggConfig = new ChatAggConfigReq();
ChatDefaultConfigReq chatDefaultConfigAgg = new ChatDefaultConfigReq();
List<Long> dimensionIds1 = Arrays.asList(4L, 5L, 6L, 7L);
List<Long> dimensionIds1 = Arrays.asList(6L, 7L, 8L, 9L);
List<Long> metricIds1 = Arrays.asList(4L);
chatDefaultConfigAgg.setDimensionIds(dimensionIds1);
chatDefaultConfigAgg.setMetricIds(metricIds1);
@@ -255,8 +282,8 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
private void addAgent2() {
Agent agent = new Agent();
agent.setId(2);
agent.setName("圈实体");
agent.setDescription("帮助您用自然语言圈选实体，支持多条件组合筛选");
agent.setName("标签圈选");
agent.setDescription("帮助您用自然语言进行圈选，支持多条件组合筛选");
agent.setStatus(1);
agent.setEnableSearch(1);
agent.setExamples(Lists.newArrayList("国风风格艺人", "港台地区的艺人", "风格为流行的艺人"));
@@ -266,7 +293,7 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
ruleQueryTool.setType(AgentToolType.RULE);
ruleQueryTool.setModelIds(Lists.newArrayList(-1L));
ruleQueryTool.setQueryModes(Lists.newArrayList(
"ENTITY_DETAIL", "ENTITY_LIST_FILTER", "ENTITY_ID"));
"TAG_DETAIL", "TAG_LIST_FILTER", "TAG_ID"));
agentConfig.getTools().add(ruleQueryTool);
LLMParserTool llmParserTool = new LLMParserTool();
@@ -310,6 +337,7 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
addSysParameter();
addDemoChatConfig_1();
addDemoChatConfig_2();
addDemoChatConfig_3();
addPlugin_1();
addAgent1();
addAgent2();

135
launchers/standalone/src/main/java/com/tencent/supersonic/LoadBenchMarkDemo.java
View File

@@ -8,10 +8,9 @@ import com.tencent.supersonic.semantic.api.model.pojo.Dim;
import com.tencent.supersonic.semantic.api.model.pojo.DimensionTimeTypeParams;
import com.tencent.supersonic.semantic.api.model.pojo.Identify;
import com.tencent.supersonic.semantic.api.model.pojo.Measure;
import com.tencent.supersonic.semantic.api.model.request.DatasourceReq;
import com.tencent.supersonic.semantic.api.model.pojo.ModelDetail;
import com.tencent.supersonic.semantic.api.model.request.DomainReq;
import com.tencent.supersonic.semantic.api.model.request.ModelReq;
import com.tencent.supersonic.semantic.model.domain.DatasourceService;
import com.tencent.supersonic.semantic.model.domain.DomainService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import lombok.extern.slf4j.Slf4j;
@@ -20,6 +19,7 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.CommandLineRunner;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -39,8 +39,6 @@ public class LoadBenchMarkDemo implements CommandLineRunner {
private DomainService domainService;
@Autowired
private ModelService modelService;
@Autowired
private DatasourceService datasourceService;
@Override
public void run(String... args) {
@@ -48,12 +46,11 @@ public class LoadBenchMarkDemo implements CommandLineRunner {
return;
}
try {
addDomain();
addModel_1();
addDatasource_1();
addDatasource_2();
addDatasource_3();
addDatasource_4();
//addDomain();
//addModel_1();
//addModel_2();
//addModel_3();
//addModel_4();
} catch (Exception e) {
log.error("Failed to add bench mark demo data", e);
}
@@ -72,105 +69,97 @@ public class LoadBenchMarkDemo implements CommandLineRunner {
domainService.createDomain(domainReq, user);
}
public void addModel_1() {
public void addModel_1() throws Exception {
ModelReq modelReq = new ModelReq();
modelReq.setName("测评数据_音乐");
modelReq.setBizName("music");
modelReq.setDomainId(2L);
modelReq.setDomainId(3L);
modelReq.setName("艺术类型");
modelReq.setBizName("genre");
modelReq.setDescription("艺术类型");
modelReq.setDatabaseId(1L);
modelReq.setViewers(Arrays.asList("admin", "tom", "jack"));
modelReq.setViewOrgs(Collections.singletonList("admin"));
modelReq.setAdmins(Collections.singletonList("admin"));
modelReq.setAdminOrgs(Collections.emptyList());
modelService.createModel(modelReq, user);
}
public void addDatasource_1() throws Exception {
DatasourceReq datasourceReq = new DatasourceReq();
datasourceReq.setModelId(3L);
datasourceReq.setName("艺术类型");
datasourceReq.setBizName("genre");
datasourceReq.setDescription("艺术类型");
datasourceReq.setDatabaseId(1L);
ModelDetail modelDetail = new ModelDetail();
List<Dim> dimensions = new ArrayList<>();
Dim dimension1 = new Dim("", "imp_date", DimensionTypeEnum.time.name(), 0);
dimension1.setTypeParams(new DimensionTimeTypeParams());
dimensions.add(dimension1);
dimensions.add(new Dim("活跃区域", "most_popular_in", DimensionTypeEnum.categorical.name(), 1));
datasourceReq.setDimensions(dimensions);
modelDetail.setDimensions(dimensions);
List<Identify> identifiers = new ArrayList<>();
identifiers.add(new Identify("音乐类型名称", IdentifyTypeEnum.primary.name(), "g_name"));
datasourceReq.setIdentifiers(identifiers);
modelDetail.setIdentifiers(identifiers);
List<Measure> measures = new ArrayList<>();
Measure measure = new Measure("评分", "rating", AggOperatorEnum.SUM.name(), 0);
measures.add(measure);
datasourceReq.setMeasures(measures);
modelDetail.setMeasures(measures);
datasourceReq.setQueryType("sql_query");
datasourceReq.setSqlQuery("SELECT g_name, rating, most_popular_in FROM genre");
datasourceService.createDatasource(datasourceReq, user);
modelDetail.setQueryType("sql_query");
modelDetail.setSqlQuery("SELECT g_name, rating, most_popular_in FROM genre");
modelService.createModel(modelReq, user);
}
public void addDatasource_2() throws Exception {
DatasourceReq datasourceReq = new DatasourceReq();
datasourceReq.setModelId(3L);
datasourceReq.setName("艺术家");
datasourceReq.setBizName("artist");
datasourceReq.setDescription("艺术家");
datasourceReq.setDatabaseId(1L);
public void addModel_2() throws Exception {
ModelReq modelReq = new ModelReq();
modelReq.setDomainId(3L);
modelReq.setName("艺术家");
modelReq.setBizName("artist");
modelReq.setDescription("艺术家");
modelReq.setDatabaseId(1L);
ModelDetail modelDetail = new ModelDetail();
List<Dim> dimensions = new ArrayList<>();
dimensions.add(new Dim("国籍", "country", DimensionTypeEnum.categorical.name(), 1));
dimensions.add(new Dim("性别", "gender", DimensionTypeEnum.categorical.name(), 1));
datasourceReq.setDimensions(dimensions);
modelDetail.setDimensions(dimensions);
List<Identify> identifiers = new ArrayList<>();
identifiers.add(new Identify("艺术家名称", IdentifyTypeEnum.primary.name(), "artist_name"));
identifiers.add(new Identify("音乐类型名称", IdentifyTypeEnum.foreign.name(), "g_name"));
datasourceReq.setIdentifiers(identifiers);
modelDetail.setIdentifiers(identifiers);
datasourceReq.setMeasures(Collections.emptyList());
modelDetail.setMeasures(Collections.emptyList());
datasourceReq.setQueryType("sql_query");
datasourceReq.setSqlQuery("SELECT artist_name, country, gender, g_name FROM artist");
datasourceService.createDatasource(datasourceReq, user);
modelDetail.setQueryType("sql_query");
modelDetail.setSqlQuery("SELECT artist_name, country, gender, g_name FROM artist");
modelService.createModel(modelReq, user);
}
public void addDatasource_3() throws Exception {
DatasourceReq datasourceReq = new DatasourceReq();
datasourceReq.setModelId(3L);
datasourceReq.setName("文件");
datasourceReq.setBizName("files");
datasourceReq.setDescription("文件");
datasourceReq.setDatabaseId(1L);
public void addModel_3() throws Exception {
ModelReq modelReq = new ModelReq();
modelReq.setDomainId(3L);
modelReq.setName("文件");
modelReq.setBizName("files");
modelReq.setDescription("文件");
modelReq.setDatabaseId(1L);
ModelDetail modelDetail = new ModelDetail();
List<Dim> dimensions = new ArrayList<>();
dimensions.add(new Dim("持续时间", "duration", DimensionTypeEnum.categorical.name(), 1));
dimensions.add(new Dim("文件格式", "formats", DimensionTypeEnum.categorical.name(), 1));
datasourceReq.setDimensions(dimensions);
modelDetail.setDimensions(dimensions);
List<Identify> identifiers = new ArrayList<>();
identifiers.add(new Identify("歌曲ID", IdentifyTypeEnum.primary.name(), "f_id"));
identifiers.add(new Identify("艺术家名称", IdentifyTypeEnum.foreign.name(), "artist_name"));
datasourceReq.setIdentifiers(identifiers);
modelDetail.setIdentifiers(identifiers);
datasourceReq.setMeasures(Collections.emptyList());
modelDetail.setMeasures(Collections.emptyList());
datasourceReq.setQueryType("sql_query");
datasourceReq.setSqlQuery("SELECT f_id, artist_name, file_size, duration, formats FROM files");
datasourceService.createDatasource(datasourceReq, user);
modelDetail.setQueryType("sql_query");
modelDetail.setSqlQuery("SELECT f_id, artist_name, file_size, duration, formats FROM files");
modelService.createModel(modelReq, user);
}
public void addDatasource_4() throws Exception {
DatasourceReq datasourceReq = new DatasourceReq();
datasourceReq.setModelId(3L);
datasourceReq.setName("歌曲");
datasourceReq.setBizName("song");
datasourceReq.setDescription("歌曲");
datasourceReq.setDatabaseId(1L);
public void addModel_4() throws Exception {
ModelReq modelReq = new ModelReq();
modelReq.setDomainId(3L);
modelReq.setName("歌曲");
modelReq.setBizName("song");
modelReq.setDescription("歌曲");
modelReq.setDatabaseId(1L);
ModelDetail modelDetail = new ModelDetail();
List<Dim> dimensions = new ArrayList<>();
Dim dimension1 = new Dim("", "imp_date", DimensionTypeEnum.time.name(), 0);
dimension1.setTypeParams(new DimensionTimeTypeParams());
@@ -178,22 +167,22 @@ public class LoadBenchMarkDemo implements CommandLineRunner {
dimensions.add(new Dim("国家", "country", DimensionTypeEnum.categorical.name(), 1));
dimensions.add(new Dim("语种", "languages", DimensionTypeEnum.categorical.name(), 1));
dimensions.add(new Dim("发行时间", "releasedate", DimensionTypeEnum.categorical.name(), 1));
datasourceReq.setDimensions(dimensions);
modelDetail.setDimensions(dimensions);
List<Identify> identifiers = new ArrayList<>();
identifiers.add(new Identify("歌曲名称", IdentifyTypeEnum.primary.name(), "song_name"));
identifiers.add(new Identify("歌曲ID", IdentifyTypeEnum.foreign.name(), "f_id"));
datasourceReq.setIdentifiers(identifiers);
modelDetail.setIdentifiers(identifiers);
List<Measure> measures = new ArrayList<>();
measures.add(new Measure("分辨率", "resolution", AggOperatorEnum.SUM.name(), 1));
measures.add(new Measure("评分", "rating", AggOperatorEnum.SUM.name(), 1));
datasourceReq.setMeasures(measures);
modelDetail.setMeasures(measures);
datasourceReq.setQueryType("sql_query");
datasourceReq.setSqlQuery("SELECT imp_date, song_name, artist_name, country, f_id, g_name, "
modelDetail.setQueryType("sql_query");
modelDetail.setSqlQuery("SELECT imp_date, song_name, artist_name, country, f_id, g_name, "
+ " rating, languages, releasedate, resolution FROM song");
datasourceService.createDatasource(datasourceReq, user);
modelService.createModel(modelReq, user);
}
}

207
launchers/standalone/src/main/java/com/tencent/supersonic/LoadModelDataDemo.java
View File

@@ -5,30 +5,33 @@ import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthGroup;
import com.tencent.supersonic.auth.api.authorization.pojo.AuthRule;
import com.tencent.supersonic.auth.api.authorization.service.AuthService;
import com.tencent.supersonic.common.pojo.JoinCondition;
import com.tencent.supersonic.common.pojo.ModelRela;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum;
import com.tencent.supersonic.common.pojo.enums.StatusEnum;
import com.tencent.supersonic.semantic.api.model.enums.DataTypeEnum;
import com.tencent.supersonic.semantic.api.model.enums.DimensionTypeEnum;
import com.tencent.supersonic.semantic.api.model.enums.IdentifyTypeEnum;
import com.tencent.supersonic.semantic.api.model.enums.SemanticTypeEnum;
import com.tencent.supersonic.semantic.api.model.pojo.Dim;
import com.tencent.supersonic.semantic.api.model.pojo.DimensionTimeTypeParams;
import com.tencent.supersonic.semantic.api.model.pojo.Entity;
import com.tencent.supersonic.semantic.api.model.pojo.Identify;
import com.tencent.supersonic.semantic.api.model.pojo.Measure;
import com.tencent.supersonic.semantic.api.model.pojo.MetricTypeParams;
import com.tencent.supersonic.semantic.api.model.pojo.ModelDetail;
import com.tencent.supersonic.semantic.api.model.request.DatabaseReq;
import com.tencent.supersonic.semantic.api.model.request.DatasourceReq;
import com.tencent.supersonic.semantic.api.model.request.DimensionReq;
import com.tencent.supersonic.semantic.api.model.request.DomainReq;
import com.tencent.supersonic.semantic.api.model.request.MetricReq;
import com.tencent.supersonic.semantic.api.model.request.ModelReq;
import com.tencent.supersonic.semantic.model.domain.DatabaseService;
import com.tencent.supersonic.semantic.model.domain.DatasourceService;
import com.tencent.supersonic.semantic.model.domain.DimensionService;
import com.tencent.supersonic.semantic.model.domain.DomainService;
import com.tencent.supersonic.semantic.model.domain.MetricService;
import com.tencent.supersonic.semantic.model.domain.ModelRelaService;
import com.tencent.supersonic.semantic.model.domain.ModelService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
@@ -36,6 +39,7 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.CommandLineRunner;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -58,7 +62,7 @@ public class LoadModelDataDemo implements CommandLineRunner {
@Autowired
private ModelService modelService;
@Autowired
private DatasourceService datasourceService;
private ModelRelaService modelRelaService;
@Autowired
private DimensionService dimensionService;
@Autowired
@@ -75,11 +79,12 @@ public class LoadModelDataDemo implements CommandLineRunner {
addDatabase();
addDomain();
addModel_1();
addDatasource_1();
addDatasource_2();
addDatasource_3();
addModel_2();
addDatasource_4();
addModel_3();
addModelRela_1();
addModelRela_2();
addDomain_2();
addModel_4();
updateDimension();
updateMetric();
addAuthGroup_1();
@@ -106,6 +111,7 @@ public class LoadModelDataDemo implements CommandLineRunner {
domainReq.setName("超音数");
domainReq.setBizName("supersonic");
domainReq.setParentId(0L);
domainReq.setStatus(StatusEnum.ONLINE.getCode());
domainReq.setViewers(Arrays.asList("admin", "tom", "jack"));
domainReq.setViewOrgs(Collections.singletonList("admin"));
domainReq.setAdmins(Collections.singletonList("admin"));
@@ -113,51 +119,49 @@ public class LoadModelDataDemo implements CommandLineRunner {
domainService.createDomain(domainReq, user);
}
public void addModel_1() {
public void addModel_1() throws Exception {
ModelReq modelReq = new ModelReq();
modelReq.setName("超音数");
modelReq.setBizName("supersonic");
modelReq.setName("超音数用户部门");
modelReq.setBizName("user_department");
modelReq.setDescription("用户部门信息");
modelReq.setDatabaseId(1L);
modelReq.setDomainId(1L);
modelReq.setViewers(Arrays.asList("admin", "tom", "jack"));
modelReq.setViewOrgs(Collections.singletonList("admin"));
modelReq.setAdmins(Collections.singletonList("admin"));
modelReq.setAdminOrgs(Collections.emptyList());
modelService.createModel(modelReq, user);
}
public void addDatasource_1() throws Exception {
DatasourceReq datasourceReq = new DatasourceReq();
datasourceReq.setName("用户部门");
datasourceReq.setBizName("user_department");
datasourceReq.setDescription("用户部门");
datasourceReq.setDatabaseId(1L);
ModelDetail modelDetail = new ModelDetail();
List<Identify> identifiers = new ArrayList<>();
identifiers.add(new Identify("用户名", IdentifyTypeEnum.primary.name(), "user_name"));
datasourceReq.setIdentifiers(identifiers);
identifiers.add(new Identify("用户", IdentifyTypeEnum.primary.name(), "user_name"));
modelDetail.setIdentifiers(identifiers);
List<Dim> dimensions = new ArrayList<>();
dimensions.add(new Dim("部门", "department",
DimensionTypeEnum.categorical.name(), 1));
datasourceReq.setDimensions(dimensions);
modelDetail.setDimensions(dimensions);
datasourceReq.setMeasures(Collections.emptyList());
datasourceReq.setQueryType("table_query");
datasourceReq.setTableQuery("PUBLIC.s2_user_department");
datasourceReq.setModelId(1L);
datasourceService.createDatasource(datasourceReq, user);
modelDetail.setMeasures(Collections.emptyList());
modelDetail.setQueryType("table_query");
modelDetail.setSqlQuery("select user_name,department from PUBLIC.s2_user_department");
modelReq.setModelDetail(modelDetail);
modelReq.setDomainId(1L);
modelService.createModel(modelReq, user);
}
public void addDatasource_2() throws Exception {
DatasourceReq datasourceReq = new DatasourceReq();
datasourceReq.setName("PVUV统计");
datasourceReq.setBizName("s2_pv_uv_statis");
datasourceReq.setDescription("PVUV统计");
datasourceReq.setDatabaseId(1L);
public void addModel_2() throws Exception {
ModelReq modelReq = new ModelReq();
modelReq.setName("超音数PVUV统计");
modelReq.setBizName("s2_pv_uv_statis");
modelReq.setDescription("超音数PVUV统计");
modelReq.setDatabaseId(1L);
modelReq.setViewers(Arrays.asList("admin", "tom", "jack"));
modelReq.setViewOrgs(Collections.singletonList("admin"));
modelReq.setAdmins(Collections.singletonList("admin"));
modelReq.setAdminOrgs(Collections.emptyList());
List<Identify> identifiers = new ArrayList<>();
identifiers.add(new Identify("用户名", IdentifyTypeEnum.primary.name(), "user_name"));
datasourceReq.setIdentifiers(identifiers);
ModelDetail modelDetail = new ModelDetail();
identifiers.add(new Identify("用户名", IdentifyTypeEnum.primary.name(), "s2_pv_uv_statis_user_name"));
modelDetail.setIdentifiers(identifiers);
List<Dim> dimensions = new ArrayList<>();
Dim dimension1 = new Dim("", "imp_date", DimensionTypeEnum.time.name(), 0);
@@ -166,7 +170,7 @@ public class LoadModelDataDemo implements CommandLineRunner {
Dim dimension2 = new Dim("", "page", DimensionTypeEnum.categorical.name(), 0);
dimension2.setExpr("page");
dimensions.add(dimension2);
datasourceReq.setDimensions(dimensions);
modelDetail.setDimensions(dimensions);
List<Measure> measures = new ArrayList<>();
Measure measure1 = new Measure("访问次数", "pv", AggOperatorEnum.SUM.name(), 1);
@@ -175,23 +179,29 @@ public class LoadModelDataDemo implements CommandLineRunner {
Measure measure2 = new Measure("访问人数", "uv", AggOperatorEnum.COUNT_DISTINCT.name(), 1);
measures.add(measure2);
datasourceReq.setMeasures(measures);
datasourceReq.setSqlQuery("SELECT imp_date, user_name, page, 1 as pv, user_name as uv FROM s2_pv_uv_statis");
datasourceReq.setQueryType("sql_query");
datasourceReq.setModelId(1L);
datasourceService.createDatasource(datasourceReq, user);
modelDetail.setMeasures(measures);
modelDetail.setSqlQuery("SELECT imp_date, user_name as s2_pv_uv_statis_user_name, page, 1 as pv, "
+ "user_name as uv FROM s2_pv_uv_statis");
modelDetail.setQueryType("sql_query");
modelReq.setDomainId(1L);
modelReq.setModelDetail(modelDetail);
modelService.createModel(modelReq, user);
}
public void addDatasource_3() throws Exception {
DatasourceReq datasourceReq = new DatasourceReq();
datasourceReq.setName("停留时长统计");
datasourceReq.setBizName("s2_stay_time_statis");
datasourceReq.setDescription("停留时长统计");
datasourceReq.setDatabaseId(1L);
public void addModel_3() throws Exception {
ModelReq modelReq = new ModelReq();
modelReq.setName("停留时长统计");
modelReq.setBizName("s2_stay_time_statis");
modelReq.setDescription("停留时长统计");
modelReq.setDatabaseId(1L);
modelReq.setViewers(Arrays.asList("admin", "tom", "jack"));
modelReq.setViewOrgs(Collections.singletonList("admin"));
modelReq.setAdmins(Collections.singletonList("admin"));
modelReq.setAdminOrgs(Collections.emptyList());
List<Identify> identifiers = new ArrayList<>();
identifiers.add(new Identify("用户名", IdentifyTypeEnum.primary.name(), "user_name"));
datasourceReq.setIdentifiers(identifiers);
ModelDetail modelDetail = new ModelDetail();
identifiers.add(new Identify("用户名称", IdentifyTypeEnum.primary.name(), "stay_hours_user_name"));
modelDetail.setIdentifiers(identifiers);
List<Dim> dimensions = new ArrayList<>();
Dim dimension1 = new Dim("", "imp_date", DimensionTypeEnum.time.name(), 0);
@@ -200,44 +210,75 @@ public class LoadModelDataDemo implements CommandLineRunner {
Dim dimension2 = new Dim("页面", "page", DimensionTypeEnum.categorical.name(), 1);
dimension2.setExpr("page");
dimensions.add(dimension2);
datasourceReq.setDimensions(dimensions);
modelDetail.setDimensions(dimensions);
List<Measure> measures = new ArrayList<>();
Measure measure1 = new Measure("停留时长", "stay_hours", AggregateTypeEnum.SUM.name(), 1);
measures.add(measure1);
datasourceReq.setMeasures(measures);
datasourceReq.setTableQuery("PUBLIC.s2_stay_time_statis");
datasourceReq.setQueryType("table_query");
datasourceReq.setModelId(1L);
datasourceService.createDatasource(datasourceReq, user);
modelDetail.setMeasures(measures);
modelDetail.setSqlQuery(
"select imp_date,user_name as stay_hours_user_name,stay_hours,page from PUBLIC.s2_stay_time_statis");
modelDetail.setQueryType("table_query");
modelReq.setDomainId(1L);
modelReq.setModelDetail(modelDetail);
modelService.createModel(modelReq, user);
}
public void addModel_2() {
public void addModelRela_1() {
List<JoinCondition> joinConditions = Lists.newArrayList();
joinConditions.add(new JoinCondition("user_name", "s2_pv_uv_statis_user_name", FilterOperatorEnum.EQUALS));
ModelRela modelRelaReq = new ModelRela();
modelRelaReq.setDomainId(1L);
modelRelaReq.setFromModelId(1L);
modelRelaReq.setToModelId(2L);
modelRelaReq.setJoinType("left join");
modelRelaReq.setJoinConditions(joinConditions);
modelRelaService.save(modelRelaReq, user);
}
public void addModelRela_2() {
List<JoinCondition> joinConditions = Lists.newArrayList();
joinConditions.add(new JoinCondition("user_name", "stay_hours_user_name", FilterOperatorEnum.EQUALS));
ModelRela modelRelaReq = new ModelRela();
modelRelaReq.setDomainId(1L);
modelRelaReq.setFromModelId(1L);
modelRelaReq.setToModelId(3L);
modelRelaReq.setJoinType("left join");
modelRelaReq.setJoinConditions(joinConditions);
modelRelaService.save(modelRelaReq, user);
}
public void addDomain_2() {
DomainReq domainReq = new DomainReq();
domainReq.setName("艺人库");
domainReq.setBizName("supersonic");
domainReq.setParentId(0L);
domainReq.setStatus(StatusEnum.ONLINE.getCode());
domainReq.setViewers(Arrays.asList("admin", "tom", "jack"));
domainReq.setViewOrgs(Collections.singletonList("admin"));
domainReq.setAdmins(Collections.singletonList("admin"));
domainReq.setAdminOrgs(Collections.emptyList());
domainService.createDomain(domainReq, user);
}
public void addModel_4() throws Exception {
ModelReq modelReq = new ModelReq();
modelReq.setName("艺人库");
modelReq.setBizName("singer");
modelReq.setDomainId(1L);
modelReq.setDescription("艺人库");
modelReq.setDatabaseId(1L);
modelReq.setDomainId(2L);
modelReq.setViewers(Arrays.asList("admin", "tom", "jack"));
modelReq.setViewOrgs(Collections.singletonList("admin"));
modelReq.setAdmins(Collections.singletonList("admin"));
modelReq.setAdminOrgs(Collections.emptyList());
modelReq.setEntity(new Entity(7L, Arrays.asList("歌手", "艺人")));
modelService.createModel(modelReq, user);
}
public void addDatasource_4() throws Exception {
DatasourceReq datasourceReq = new DatasourceReq();
datasourceReq.setName("艺人库");
datasourceReq.setBizName("singer");
datasourceReq.setDescription("艺人库");
datasourceReq.setDatabaseId(1L);
ModelDetail modelDetail = new ModelDetail();
List<Identify> identifiers = new ArrayList<>();
Identify identify = new Identify("歌手名", IdentifyTypeEnum.primary.name(), "singer_name");
identify.setEntityNames(Lists.newArrayList("歌手", "艺人"));
identifiers.add(identify);
datasourceReq.setIdentifiers(identifiers);
modelDetail.setIdentifiers(identifiers);
List<Dim> dimensions = new ArrayList<>();
Dim dimension1 = new Dim("", "imp_date", DimensionTypeEnum.time.name(), 0);
@@ -249,26 +290,26 @@ public class LoadModelDataDemo implements CommandLineRunner {
DimensionTypeEnum.categorical.name(), 1));
dimensions.add(new Dim("风格", "genre",
DimensionTypeEnum.categorical.name(), 1, 1));
datasourceReq.setDimensions(dimensions);
modelDetail.setDimensions(dimensions);
Measure measure1 = new Measure("播放量", "js_play_cnt", "sum", 1);
Measure measure2 = new Measure("下载量", "down_cnt", "sum", 1);
Measure measure3 = new Measure("收藏量", "favor_cnt", "sum", 1);
datasourceReq.setMeasures(Lists.newArrayList(measure1, measure2, measure3));
datasourceReq.setQueryType("table_query");
datasourceReq.setTableQuery("PUBLIC.singer");
datasourceReq.setModelId(2L);
datasourceService.createDatasource(datasourceReq, user);
modelDetail.setMeasures(Lists.newArrayList(measure1, measure2, measure3));
modelDetail.setQueryType("table_query");
modelDetail.setTableQuery("PUBLIC.singer");
modelReq.setModelDetail(modelDetail);
modelService.createModel(modelReq, user);
}
public void updateDimension() throws Exception {
DimensionReq dimensionReq = new DimensionReq();
dimensionReq.setModelId(1L);
dimensionReq.setType(DimensionTypeEnum.categorical.name());
dimensionReq.setId(3L);
dimensionReq.setId(4L);
dimensionReq.setName("页面");
dimensionReq.setBizName("page");
dimensionReq.setDatasourceId(3L);
dimensionReq.setModelId(3L);
dimensionReq.setAlias("page");
dimensionReq.setSemanticType(SemanticTypeEnum.CATEGORY.name());
dimensionReq.setSensitiveLevel(2);
@@ -302,7 +343,7 @@ public class LoadModelDataDemo implements CommandLineRunner {
public void addAuthGroup_1() {
AuthGroup authGroupReq = new AuthGroup();
authGroupReq.setModelId(1L);
authGroupReq.setModelId(3L);
authGroupReq.setName("admin-permission");
List<AuthRule> authRules = new ArrayList<>();
@@ -319,7 +360,7 @@ public class LoadModelDataDemo implements CommandLineRunner {
public void addAuthGroup_2() {
AuthGroup authGroupReq = new AuthGroup();
authGroupReq.setModelId(1L);
authGroupReq.setModelId(3L);
authGroupReq.setName("tom_sales_permission");
List<AuthRule> authRules = new ArrayList<>();

6
launchers/standalone/src/main/resources/META-INF/spring.factories
View File

@@ -3,7 +3,8 @@ com.tencent.supersonic.chat.api.component.SchemaMapper=\
com.tencent.supersonic.chat.mapper.HanlpDictMapper, \
com.tencent.supersonic.chat.mapper.FuzzyNameMapper, \
com.tencent.supersonic.chat.mapper.QueryFilterMapper, \
com.tencent.supersonic.chat.mapper.EntityMapper
com.tencent.supersonic.chat.mapper.EntityMapper, \
com.tencent.supersonic.chat.mapper.ModelClusterMapper
com.tencent.supersonic.chat.api.component.SemanticParser=\
com.tencent.supersonic.chat.parser.rule.RuleBasedParser, \
@@ -17,7 +18,8 @@ com.tencent.supersonic.chat.api.component.SemanticCorrector=\
com.tencent.supersonic.chat.corrector.SelectCorrector, \
com.tencent.supersonic.chat.corrector.WhereCorrector, \
com.tencent.supersonic.chat.corrector.GroupByCorrector, \
com.tencent.supersonic.chat.corrector.HavingCorrector
com.tencent.supersonic.chat.corrector.HavingCorrector, \
com.tencent.supersonic.chat.corrector.FromCorrector
com.tencent.supersonic.chat.llm.LLMInterpreter=\
com.tencent.supersonic.chat.llm.HttpLLMInterpreter

22
launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_2.txt
View File

@@ -5,14 +5,14 @@ dean _1_2 36
john _1_2 50
jack _1_2 38
admin _1_2 70
周杰伦 _2_7 100
陈奕迅 _2_7 100
林俊杰 _2_7 100
张碧晨 _2_7 100
程响 _2_7 100
Taylor#Swift _2_7 100
内地 _2_4 100
欧美 _2_4 100
港台 _2_4 100
流行 _2_6 100
国风 _2_6 100
周杰伦 _4_9 100
陈奕迅 _4_9 100
林俊杰 _4_9 100
张碧晨 _4_9 100
程响 _4_9 100
Taylor#Swift _4_9 100
内地 _4_6 100
欧美 _4_6 100
港台 _4_6 100
流行 _4_8 100
国风 _4_8 100

12
launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_1_3.txt
View File

@@ -1,6 +1,6 @@
p1 _1_3 52
p2 _1_3 47
p3 _1_3 31
p4 _1_3 36
p5 _1_3 50
p6 _1_3 38
p1 _3_4 52
p2 _3_4 47
p3 _3_4 31
p4 _3_4 36
p5 _3_4 50
p6 _3_4 38

9
launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_2_7.txt
View File

@@ -1,9 +0,0 @@
周杰伦 _2_7 9000
周深 _2_7 8000
周传雄 _2_7 7000
周华建 _2_7 6000
陈奕迅 _2_7 8000
林俊杰 _2_7 7000
张碧晨 _2_7 7000
程响 _2_7 7000
Taylor#Swift _2_7 7000

9
launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_4_9.txt Normal file
View File

@@ -0,0 +1,9 @@
周杰伦 _4_9 9000
周深 _4_9 8000
周传雄 _4_9 7000
周华建 _4_9 6000
陈奕迅 _4_9 8000
林俊杰 _4_9 7000
张碧晨 _4_9 7000
程响 _4_9 7000
Taylor#Swift _4_9 7000

68
launchers/standalone/src/main/resources/data/dictionary/custom/benchmark_cspider.txt
View File

@@ -1,68 +0,0 @@
tagore _3_8 9000
nazrul _3_8 9000
民间 _3_8 9000
现代 _3_8 9000
蓝调 _3_8 9000
流行 _3_8 9000
孟加拉国 _3_10 9000
锡尔赫特、吉大港、库斯蒂亚 _3_10 9000
加拿大 _3_10 9000
美国 _3_10 9000
Shrikanta _3_11 9000
Prity _3_11 9000
Farida _3_11 9000
Topu _3_11 9000
Enrique _3_11 9000
Michel _3_11 9000
孟加拉国 _3_12 9000
印度 _3_12 9000
美国 _3_12 9000
英国 _3_12 9000
男性 _3_13 9000
女性 _3_13 9000
mp4 _3_19 9000
mp3 _3_19 9000
Tumi#长袍#尼罗布 _3_20 9000
舒克诺#帕塔尔#努普尔#帕埃 _3_20 9000
阿米·奥帕尔·霍伊 _3_20 9000
我的爱 _3_20 9000
打败它 _3_20 9000
阿杰伊阿卡什 _3_20 9000
孟加拉国 _3_22 9000
印度 _3_22 9000
美国 _3_22 9000
英国 _3_22 9000
孟加拉语 _3_26 9000
英文 _3_26 9000
=======
孟加拉国 _3_8 9000
锡尔赫特、吉大港、库斯蒂亚 _3_8 9000
加拿大 _3_8 9000
美国 _3_8 9000
tagore _3_9 9000
nazrul _3_9 9000
民间 _3_9 9000
现代 _3_9 9000
蓝调 _3_9 9000
流行 _3_9 9000
孟加拉国 _3_10 9000
印度 _3_10 9000
美国 _3_10 9000
英国 _3_10 9000
男性 _3_11 9000
女性 _3_11 9000
Shrikanta _3_12 9000
Prity _3_12 9000
Farida _3_12 9000
Topu _3_12 9000
Enrique _3_12 9000
Michel _3_12 9000
mp4 _3_14 9000
mp3 _3_14 9000
孟加拉语 _3_16 9000
英文 _3_16 9000
Tumi#长袍#尼罗布 _3_18 9000
舒克诺#帕塔尔#努普尔#帕埃 _3_18 9000
阿米·奥帕尔·霍伊 _3_18 9000
我的爱 _3_18 9000
打败它 _3_18 9000

38
launchers/standalone/src/main/resources/db/schema-h2.sql
View File

@@ -133,6 +133,10 @@ CREATE TABLE IF NOT EXISTS `s2_model` (
`view_org` varchar(3000) DEFAULT NULL , -- domain available organization
`entity` varchar(500) DEFAULT NULL , -- domain entity info
`drill_down_dimensions` varchar(500) DEFAULT NULL , -- drill down dimensions info
`database_id` INT NOT NULL ,
`model_detail` LONGVARCHAR NOT NULL ,
`depends` varchar(500) DEFAULT NULL ,
`filter_sql` varchar(1000) DEFAULT NULL ,
PRIMARY KEY (`id`)
);
COMMENT ON TABLE s2_model IS 'model information';
@@ -156,16 +160,12 @@ CREATE TABLE `s2_database` (
COMMENT ON TABLE s2_database IS 'database instance table';
CREATE TABLE IF NOT EXISTS `s2_datasource` (
`id` INT NOT NULL AUTO_INCREMENT,
`model_id` INT NOT NULL ,
`name` varchar(255) NOT NULL ,
`id` INT NOT NULL AUTO_INCREMENT,
`model_id` INT NOT NULL ,
`name` varchar(255) NOT NULL ,
`biz_name` varchar(255) NOT NULL ,
`description` varchar(500) DEFAULT NULL ,
`database_id` INT NOT NULL ,
`datasource_detail` LONGVARCHAR NOT NULL ,
`status` int(11) DEFAULT NULL ,
`depends` varchar(500) DEFAULT NULL ,
`filter_sql` varchar(1000) DEFAULT NULL ,
`created_at` TIMESTAMP NOT NULL ,
`created_by` varchar(100) NOT NULL ,
`updated_at` TIMESTAMP NOT NULL ,
@@ -208,7 +208,6 @@ COMMENT ON TABLE s2_metric IS 'metric information table';
CREATE TABLE IF NOT EXISTS `s2_dimension` (
`id` INT NOT NULL AUTO_INCREMENT ,
`model_id` INT NOT NULL ,
`datasource_id` INT NOT NULL ,
`name` varchar(255) NOT NULL ,
`biz_name` varchar(255) NOT NULL ,
`description` varchar(500) NOT NULL ,
@@ -231,20 +230,16 @@ CREATE TABLE IF NOT EXISTS `s2_dimension` (
);
COMMENT ON TABLE s2_dimension IS 'dimension information table';
create table s2_datasource_rela
CREATE TABLE s2_model_rela
(
id INT AUTO_INCREMENT,
model_id INT null,
datasource_from INT null,
datasource_to INT null,
join_key varchar(100) null,
created_at TIMESTAMP null,
created_by varchar(100) null,
updated_at TIMESTAMP null,
updated_by varchar(100) null,
id BIGINT AUTO_INCREMENT,
domain_id BIGINT,
from_model_id BIGINT,
to_model_id BIGINT,
join_type VARCHAR(255),
join_condition VARCHAR(255),
PRIMARY KEY (`id`)
);
COMMENT ON TABLE s2_datasource_rela IS 'data source association table';
create table s2_view_info
(
@@ -535,4 +530,5 @@ CREATE TABLE s2_sys_parameter
id INT PRIMARY KEY AUTO_INCREMENT,
admin varchar(500),
parameters text null
);
);

10
launchers/standalone/src/main/resources/db/schema-mysql.sql
View File

@@ -453,4 +453,14 @@ CREATE TABLE s2_sys_parameter
id int primary key AUTO_INCREMENT COMMENT '主键id',
admin varchar(500) COMMENT '系统管理员',
parameters text null COMMENT '配置项'
);
CREATE TABLE s2_model_rela
(
id bigint primary key AUTO_INCREMENT,
domain_id bigint,
from_model_id bigint,
to_model_id bigint,
join_type VARCHAR(255),
join_condition VARCHAR(255)
);

19
launchers/standalone/src/main/resources/db/sql-update.sql
View File

@@ -96,4 +96,21 @@ alter table s2_chat_config add column `llm_examples` text COMMENT 'llm examples'
alter table s2_datasource add column `filter_sql` varchar(1000) COMMENT 'filter_sql' after depends;
--20231120
alter table s2_dimension add column `is_tag` int(10) DEFAULT NULL;
alter table s2_dimension add column `is_tag` int(10) DEFAULT NULL;
--20231125
alter table s2_model add column `database_id` INT NOT NULL;
alter table s2_model add column `model_detail` text NOT NULL;
alter table s2_model add column `depends` varchar(500) DEFAULT NULL;
alter table s2_model add column `filter_sql` varchar(1000) DEFAULT NULL;
CREATE TABLE s2_model_rela
(
id BIGINT AUTO_INCREMENT,
domain_id BIGINT,
from_model_id BIGINT,
to_model_id BIGINT,
join_type VARCHAR(255),
join_condition VARCHAR(255),
PRIMARY KEY (`id`)
);