topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 03:58:14 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement](project) Support cspider multi-table association, fix dictionary loading, and repair InternalMetric exceptions. (#446)

Browse Source
This commit is contained in:
lexluo09
2023-11-28 21:16:02 +08:00
committed by GitHub
parent 46733d1728
commit 3a30a1a317
13 changed files with 194 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/MultiCustomDictionary.java
View File

@@ -222,7 +222,10 @@ public class MultiCustomDictionary extends DynamicCustomDictionary {
new BufferedOutputStream(IOUtil.newOutputStream(mainPath + ".bin")));
if (customNatureCollector.isEmpty()) {
for (int i = Nature.begin.ordinal() + 1; i < Nature.values().length; ++i) {
customNatureCollector.add(Nature.values()[i]);
Nature nature = Nature.values()[i];
if(Objects.nonNull(nature)) {
customNatureCollector.add(nature);
}
}
}

107
launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java
View File

@@ -263,10 +263,32 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
configService.addConfig(chatConfigBaseReq, user);
}
public void addDemoChatConfig_5(Long modelId) {
public void addDemoChatConfig_5() {
ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq();
chatConfigBaseReq.setModelId(modelId);
chatConfigBaseReq.setModelId(5L);
ChatDetailConfigReq chatDetailConfig = getChatDetailConfigReq();
List<KnowledgeInfoReq> knowledgeInfos = new ArrayList<>();
KnowledgeInfoReq knowledgeInfoReq = new KnowledgeInfoReq();
knowledgeInfoReq.setSearchEnable(true);
knowledgeInfoReq.setItemId(10L);
knowledgeInfoReq.setBizName("most_popular_in");
knowledgeInfos.add(knowledgeInfoReq);
KnowledgeInfoReq knowledgeInfoReq2 = new KnowledgeInfoReq();
knowledgeInfoReq2.setSearchEnable(true);
knowledgeInfoReq2.setItemId(11L);
knowledgeInfoReq2.setBizName("g_name");
knowledgeInfos.add(knowledgeInfoReq2);
chatDetailConfig.setKnowledgeInfos(knowledgeInfos);
chatConfigBaseReq.setChatDetailConfig(chatDetailConfig);
List<RecommendedQuestionReq> recommendedQuestions = new ArrayList<>();
chatConfigBaseReq.setRecommendedQuestions(recommendedQuestions);
configService.addConfig(chatConfigBaseReq, user);
}
private ChatDetailConfigReq getChatDetailConfigReq() {
ChatDetailConfigReq chatDetailConfig = new ChatDetailConfigReq();
ChatDefaultConfigReq chatDefaultConfigDetail = new ChatDefaultConfigReq();
chatDefaultConfigDetail.setUnit(-1);
@@ -274,6 +296,79 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
chatDetailConfig.setChatDefaultConfig(chatDefaultConfigDetail);
ItemVisibility visibility0 = new ItemVisibility();
chatDetailConfig.setVisibility(visibility0);
return chatDetailConfig;
}
public void addDemoChatConfig_6() {
ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq();
chatConfigBaseReq.setModelId(6L);
ChatDetailConfigReq chatDetailConfig = getChatDetailConfigReq();
List<KnowledgeInfoReq> knowledgeInfos = new ArrayList<>();
KnowledgeInfoReq knowledgeInfoReq = new KnowledgeInfoReq();
knowledgeInfoReq.setSearchEnable(true);
knowledgeInfoReq.setItemId(12L);
knowledgeInfoReq.setBizName("country");
knowledgeInfos.add(knowledgeInfoReq);
KnowledgeInfoReq knowledgeInfoReq2 = new KnowledgeInfoReq();
knowledgeInfoReq2.setSearchEnable(true);
knowledgeInfoReq2.setItemId(13L);
knowledgeInfoReq2.setBizName("gender");
knowledgeInfos.add(knowledgeInfoReq2);
chatDetailConfig.setKnowledgeInfos(knowledgeInfos);
chatConfigBaseReq.setChatDetailConfig(chatDetailConfig);
List<RecommendedQuestionReq> recommendedQuestions = new ArrayList<>();
chatConfigBaseReq.setRecommendedQuestions(recommendedQuestions);
configService.addConfig(chatConfigBaseReq, user);
}
public void addDemoChatConfig_7() {
ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq();
chatConfigBaseReq.setModelId(7L);
ChatDetailConfigReq chatDetailConfig = getChatDetailConfigReq();
List<KnowledgeInfoReq> knowledgeInfos = new ArrayList<>();
KnowledgeInfoReq knowledgeInfoReq = new KnowledgeInfoReq();
knowledgeInfoReq.setSearchEnable(true);
knowledgeInfoReq.setItemId(16L);
knowledgeInfoReq.setBizName("formats");
knowledgeInfos.add(knowledgeInfoReq);
chatDetailConfig.setKnowledgeInfos(knowledgeInfos);
chatConfigBaseReq.setChatDetailConfig(chatDetailConfig);
List<RecommendedQuestionReq> recommendedQuestions = new ArrayList<>();
chatConfigBaseReq.setRecommendedQuestions(recommendedQuestions);
configService.addConfig(chatConfigBaseReq, user);
}
public void addDemoChatConfig_8() {
ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq();
chatConfigBaseReq.setModelId(8L);
ChatDetailConfigReq chatDetailConfig = getChatDetailConfigReq();
List<KnowledgeInfoReq> knowledgeInfos = new ArrayList<>();
KnowledgeInfoReq knowledgeInfoReq = new KnowledgeInfoReq();
knowledgeInfoReq.setSearchEnable(true);
knowledgeInfoReq.setItemId(18L);
knowledgeInfoReq.setBizName("country");
knowledgeInfos.add(knowledgeInfoReq);
KnowledgeInfoReq knowledgeInfoReq2 = new KnowledgeInfoReq();
knowledgeInfoReq2.setSearchEnable(true);
knowledgeInfoReq2.setItemId(19L);
knowledgeInfoReq2.setBizName("languages");
knowledgeInfos.add(knowledgeInfoReq2);
KnowledgeInfoReq knowledgeInfoReq3 = new KnowledgeInfoReq();
knowledgeInfoReq3.setSearchEnable(true);
knowledgeInfoReq3.setItemId(21L);
knowledgeInfoReq3.setBizName("song_name");
knowledgeInfos.add(knowledgeInfoReq3);
chatDetailConfig.setKnowledgeInfos(knowledgeInfos);
chatConfigBaseReq.setChatDetailConfig(chatDetailConfig);
List<RecommendedQuestionReq> recommendedQuestions = new ArrayList<>();
chatConfigBaseReq.setRecommendedQuestions(recommendedQuestions);
@@ -392,10 +487,10 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
addDemoChatConfig_2();
addDemoChatConfig_3();
addDemoChatConfig_4();
addDemoChatConfig_5(5L);
addDemoChatConfig_5(6L);
addDemoChatConfig_5(7L);
addDemoChatConfig_5(8L);
addDemoChatConfig_5();
addDemoChatConfig_6();
addDemoChatConfig_7();
addDemoChatConfig_8();
addPlugin_1();
addAgent1();
addAgent2();

55
launchers/standalone/src/main/java/com/tencent/supersonic/LoadBenchMarkDemo.java
View File

@@ -62,6 +62,8 @@ public class LoadBenchMarkDemo implements CommandLineRunner {
addModelRela_1();
addModelRela_2();
addModelRela_3();
addModelRela_4();
addModelRela_5();
} catch (Exception e) {
log.error("Failed to add bench mark demo data", e);
}
@@ -187,6 +189,9 @@ public class LoadBenchMarkDemo implements CommandLineRunner {
List<Identify> identifiers = new ArrayList<>();
identifiers.add(new Identify("歌曲名称", IdentifyTypeEnum.primary.name(), "song_name"));
identifiers.add(new Identify("歌曲ID", IdentifyTypeEnum.foreign.name(), "f_id"));
identifiers.add(new Identify("艺术家名称", IdentifyTypeEnum.foreign.name(), "artist_name"));
identifiers.add(new Identify("艺术家名称", IdentifyTypeEnum.foreign.name(), "artist_name"));
modelDetail.setIdentifiers(identifiers);
List<Measure> measures = new ArrayList<>();
@@ -203,7 +208,7 @@ public class LoadBenchMarkDemo implements CommandLineRunner {
public void addModelRela_1() {
List<JoinCondition> joinConditions = Lists.newArrayList();
joinConditions.add(new JoinCondition("preferred_genre", "g_name", FilterOperatorEnum.EQUALS));
joinConditions.add(new JoinCondition("g_name", "g_name", FilterOperatorEnum.EQUALS));
ModelRela modelRelaReq = new ModelRela();
modelRelaReq.setDomainId(3L);
modelRelaReq.setFromModelId(6L);
@@ -214,6 +219,42 @@ public class LoadBenchMarkDemo implements CommandLineRunner {
}
public void addModelRela_2() {
List<JoinCondition> joinConditions = Lists.newArrayList();
joinConditions.add(new JoinCondition("artist_name", "artist_name", FilterOperatorEnum.EQUALS));
ModelRela modelRelaReq = new ModelRela();
modelRelaReq.setDomainId(3L);
modelRelaReq.setFromModelId(7L);
modelRelaReq.setToModelId(6L);
modelRelaReq.setJoinType("left join");
modelRelaReq.setJoinConditions(joinConditions);
modelRelaService.save(modelRelaReq, user);
}
public void addModelRela_3() {
List<JoinCondition> joinConditions = Lists.newArrayList();
joinConditions.add(new JoinCondition("artist_name", "artist_name", FilterOperatorEnum.EQUALS));
ModelRela modelRelaReq = new ModelRela();
modelRelaReq.setDomainId(3L);
modelRelaReq.setFromModelId(8L);
modelRelaReq.setToModelId(6L);
modelRelaReq.setJoinType("left join");
modelRelaReq.setJoinConditions(joinConditions);
modelRelaService.save(modelRelaReq, user);
}
public void addModelRela_4() {
List<JoinCondition> joinConditions = Lists.newArrayList();
joinConditions.add(new JoinCondition("g_name", "g_name", FilterOperatorEnum.EQUALS));
ModelRela modelRelaReq = new ModelRela();
modelRelaReq.setDomainId(3L);
modelRelaReq.setFromModelId(8L);
modelRelaReq.setToModelId(5L);
modelRelaReq.setJoinType("left join");
modelRelaReq.setJoinConditions(joinConditions);
modelRelaService.save(modelRelaReq, user);
}
public void addModelRela_5() {
List<JoinCondition> joinConditions = Lists.newArrayList();
joinConditions.add(new JoinCondition("f_id", "f_id", FilterOperatorEnum.EQUALS));
ModelRela modelRelaReq = new ModelRela();
@@ -224,16 +265,4 @@ public class LoadBenchMarkDemo implements CommandLineRunner {
modelRelaReq.setJoinConditions(joinConditions);
modelRelaService.save(modelRelaReq, user);
}
public void addModelRela_3() {
List<JoinCondition> joinConditions = Lists.newArrayList();
joinConditions.add(new JoinCondition("genre_is", "g_name", FilterOperatorEnum.EQUALS));
ModelRela modelRelaReq = new ModelRela();
modelRelaReq.setDomainId(3L);
modelRelaReq.setFromModelId(8L);
modelRelaReq.setToModelId(5L);
modelRelaReq.setJoinType("left join");
modelRelaReq.setJoinConditions(joinConditions);
modelRelaService.save(modelRelaReq, user);
}
}

4
launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_5_10.txt Normal file
View File

@@ -0,0 +1,4 @@
美国 _5_10 1
加拿大 _5_10 1
锡尔赫特、吉大港、库斯蒂亚 _5_10 1
孟加拉国 _5_10 3

6
launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_5_11.txt Normal file
View File

@@ -0,0 +1,6 @@
现代 _5_11 1
tagore _5_11 1
蓝调 _5_11 1
流行 _5_11 1
民间 _5_11 1
nazrul _5_11 1

4
launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_6_12.txt Normal file
View File

@@ -0,0 +1,4 @@
美国 _6_12 1
印度 _6_12 2
英国 _6_12 1
孟加拉国 _6_12 2

2
launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_6_13.txt Normal file
View File

@@ -0,0 +1,2 @@
男性 _6_13 3
女性 _6_13 3

2
launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_7_16.txt Normal file
View File

@@ -0,0 +1,2 @@
mp4 _7_16 4
mp3 _7_16 2

4
launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_18.txt Normal file
View File

@@ -0,0 +1,4 @@
美国 _8_18 1
印度 _8_18 2
英国 _8_18 1
孟加拉国 _8_18 2

2
launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_19.txt Normal file
View File

@@ -0,0 +1,2 @@
英文 _8_19 2
孟加拉语 _8_19 4

6
launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_21.txt Normal file
View File

@@ -0,0 +1,6 @@
阿米·奥帕尔·霍伊 _8_21 1
我的爱 _8_21 1
打败它 _8_21 1
阿杰伊阿卡什 _8_21 1
Tumi#长袍#尼罗布 _8_21 1
舒克诺#帕塔尔#努普尔#帕埃 _8_21 1

0
launchers/standalone/src/main/resources/data/dictionary/custom/benchmark_cspider.txt
View File

46
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryStructUtils.java
View File

@@ -1,5 +1,11 @@
package com.tencent.supersonic.semantic.query.utils;
import static com.tencent.supersonic.common.pojo.Constants.DAY;
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
import static com.tencent.supersonic.common.pojo.Constants.MONTH;
import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.WEEK;
import com.google.common.collect.Lists;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.common.pojo.Aggregator;
@@ -28,14 +34,6 @@ import com.tencent.supersonic.semantic.model.domain.Catalog;
import com.tencent.supersonic.semantic.model.domain.pojo.EngineTypeEnum;
import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
import com.tencent.supersonic.semantic.query.service.SchemaService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
@@ -49,12 +47,13 @@ import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import static com.tencent.supersonic.common.pojo.Constants.DAY;
import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
import static com.tencent.supersonic.common.pojo.Constants.MONTH;
import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
import static com.tencent.supersonic.common.pojo.Constants.WEEK;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Slf4j
@@ -254,22 +253,11 @@ public class QueryStructUtils {
public String generateInternalMetricName(Long modelId, List<String> groups) {
String internalMetricNamePrefix = "";
if (CollectionUtils.isEmpty(groups)) {
log.warn("group is empty!");
} else {
for (int i = 0; i < groups.size(); i++) {
if (groups.get(i).equalsIgnoreCase("sys_imp_date")) {
continue;
}
DimensionResp dimension = catalog.getDimension(groups.get(i), modelId);
if (Objects.nonNull(dimension) && Strings.isNotEmpty(dimension.getModelBizName())) {
internalMetricNamePrefix = dimension.getModelBizName() + UNDERLINE;
break;
}
}
List<DimensionResp> dimensions = catalog.getDimensions(Collections.singletonList(modelId));
if (!CollectionUtils.isEmpty(dimensions)) {
internalMetricNamePrefix = dimensions.get(0).getModelBizName();
}
String internalMetricName = internalMetricNamePrefix + internalMetricNameSuffix;
return internalMetricName;
return internalMetricNamePrefix + UNDERLINE + internalMetricNameSuffix;
}
public boolean isSupportWith(EngineTypeEnum engineTypeEnum, String version) {