diff --git a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/MultiCustomDictionary.java b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/MultiCustomDictionary.java index 015775c29..35ecf2b83 100644 --- a/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/MultiCustomDictionary.java +++ b/chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/MultiCustomDictionary.java @@ -222,7 +222,10 @@ public class MultiCustomDictionary extends DynamicCustomDictionary { new BufferedOutputStream(IOUtil.newOutputStream(mainPath + ".bin"))); if (customNatureCollector.isEmpty()) { for (int i = Nature.begin.ordinal() + 1; i < Nature.values().length; ++i) { - customNatureCollector.add(Nature.values()[i]); + Nature nature = Nature.values()[i]; + if(Objects.nonNull(nature)) { + customNatureCollector.add(nature); + } } } diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java b/launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java index 750a82317..fdcd676a6 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java @@ -263,10 +263,32 @@ public class ConfigureDemo implements ApplicationListener configService.addConfig(chatConfigBaseReq, user); } - public void addDemoChatConfig_5(Long modelId) { + public void addDemoChatConfig_5() { ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq(); - chatConfigBaseReq.setModelId(modelId); + chatConfigBaseReq.setModelId(5L); + ChatDetailConfigReq chatDetailConfig = getChatDetailConfigReq(); + List knowledgeInfos = new ArrayList<>(); + KnowledgeInfoReq knowledgeInfoReq = new KnowledgeInfoReq(); + knowledgeInfoReq.setSearchEnable(true); + knowledgeInfoReq.setItemId(10L); + knowledgeInfoReq.setBizName("most_popular_in"); + knowledgeInfos.add(knowledgeInfoReq); + + KnowledgeInfoReq knowledgeInfoReq2 = new KnowledgeInfoReq(); + knowledgeInfoReq2.setSearchEnable(true); + knowledgeInfoReq2.setItemId(11L); + knowledgeInfoReq2.setBizName("g_name"); + knowledgeInfos.add(knowledgeInfoReq2); + + chatDetailConfig.setKnowledgeInfos(knowledgeInfos); + chatConfigBaseReq.setChatDetailConfig(chatDetailConfig); + List recommendedQuestions = new ArrayList<>(); + chatConfigBaseReq.setRecommendedQuestions(recommendedQuestions); + configService.addConfig(chatConfigBaseReq, user); + } + + private ChatDetailConfigReq getChatDetailConfigReq() { ChatDetailConfigReq chatDetailConfig = new ChatDetailConfigReq(); ChatDefaultConfigReq chatDefaultConfigDetail = new ChatDefaultConfigReq(); chatDefaultConfigDetail.setUnit(-1); @@ -274,6 +296,79 @@ public class ConfigureDemo implements ApplicationListener chatDetailConfig.setChatDefaultConfig(chatDefaultConfigDetail); ItemVisibility visibility0 = new ItemVisibility(); chatDetailConfig.setVisibility(visibility0); + return chatDetailConfig; + } + + + public void addDemoChatConfig_6() { + ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq(); + chatConfigBaseReq.setModelId(6L); + + ChatDetailConfigReq chatDetailConfig = getChatDetailConfigReq(); + List knowledgeInfos = new ArrayList<>(); + KnowledgeInfoReq knowledgeInfoReq = new KnowledgeInfoReq(); + knowledgeInfoReq.setSearchEnable(true); + knowledgeInfoReq.setItemId(12L); + knowledgeInfoReq.setBizName("country"); + knowledgeInfos.add(knowledgeInfoReq); + + KnowledgeInfoReq knowledgeInfoReq2 = new KnowledgeInfoReq(); + knowledgeInfoReq2.setSearchEnable(true); + knowledgeInfoReq2.setItemId(13L); + knowledgeInfoReq2.setBizName("gender"); + knowledgeInfos.add(knowledgeInfoReq2); + + chatDetailConfig.setKnowledgeInfos(knowledgeInfos); + chatConfigBaseReq.setChatDetailConfig(chatDetailConfig); + List recommendedQuestions = new ArrayList<>(); + chatConfigBaseReq.setRecommendedQuestions(recommendedQuestions); + configService.addConfig(chatConfigBaseReq, user); + } + + public void addDemoChatConfig_7() { + ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq(); + chatConfigBaseReq.setModelId(7L); + + ChatDetailConfigReq chatDetailConfig = getChatDetailConfigReq(); + List knowledgeInfos = new ArrayList<>(); + KnowledgeInfoReq knowledgeInfoReq = new KnowledgeInfoReq(); + knowledgeInfoReq.setSearchEnable(true); + knowledgeInfoReq.setItemId(16L); + knowledgeInfoReq.setBizName("formats"); + knowledgeInfos.add(knowledgeInfoReq); + + chatDetailConfig.setKnowledgeInfos(knowledgeInfos); + chatConfigBaseReq.setChatDetailConfig(chatDetailConfig); + List recommendedQuestions = new ArrayList<>(); + chatConfigBaseReq.setRecommendedQuestions(recommendedQuestions); + configService.addConfig(chatConfigBaseReq, user); + } + + public void addDemoChatConfig_8() { + ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq(); + chatConfigBaseReq.setModelId(8L); + + ChatDetailConfigReq chatDetailConfig = getChatDetailConfigReq(); + List knowledgeInfos = new ArrayList<>(); + KnowledgeInfoReq knowledgeInfoReq = new KnowledgeInfoReq(); + knowledgeInfoReq.setSearchEnable(true); + knowledgeInfoReq.setItemId(18L); + knowledgeInfoReq.setBizName("country"); + knowledgeInfos.add(knowledgeInfoReq); + + KnowledgeInfoReq knowledgeInfoReq2 = new KnowledgeInfoReq(); + knowledgeInfoReq2.setSearchEnable(true); + knowledgeInfoReq2.setItemId(19L); + knowledgeInfoReq2.setBizName("languages"); + knowledgeInfos.add(knowledgeInfoReq2); + + KnowledgeInfoReq knowledgeInfoReq3 = new KnowledgeInfoReq(); + knowledgeInfoReq3.setSearchEnable(true); + knowledgeInfoReq3.setItemId(21L); + knowledgeInfoReq3.setBizName("song_name"); + knowledgeInfos.add(knowledgeInfoReq3); + + chatDetailConfig.setKnowledgeInfos(knowledgeInfos); chatConfigBaseReq.setChatDetailConfig(chatDetailConfig); List recommendedQuestions = new ArrayList<>(); chatConfigBaseReq.setRecommendedQuestions(recommendedQuestions); @@ -392,10 +487,10 @@ public class ConfigureDemo implements ApplicationListener addDemoChatConfig_2(); addDemoChatConfig_3(); addDemoChatConfig_4(); - addDemoChatConfig_5(5L); - addDemoChatConfig_5(6L); - addDemoChatConfig_5(7L); - addDemoChatConfig_5(8L); + addDemoChatConfig_5(); + addDemoChatConfig_6(); + addDemoChatConfig_7(); + addDemoChatConfig_8(); addPlugin_1(); addAgent1(); addAgent2(); diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/LoadBenchMarkDemo.java b/launchers/standalone/src/main/java/com/tencent/supersonic/LoadBenchMarkDemo.java index 8a04b5f51..60c1e679f 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/LoadBenchMarkDemo.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/LoadBenchMarkDemo.java @@ -62,6 +62,8 @@ public class LoadBenchMarkDemo implements CommandLineRunner { addModelRela_1(); addModelRela_2(); addModelRela_3(); + addModelRela_4(); + addModelRela_5(); } catch (Exception e) { log.error("Failed to add bench mark demo data", e); } @@ -187,6 +189,9 @@ public class LoadBenchMarkDemo implements CommandLineRunner { List identifiers = new ArrayList<>(); identifiers.add(new Identify("歌曲名称", IdentifyTypeEnum.primary.name(), "song_name")); identifiers.add(new Identify("歌曲ID", IdentifyTypeEnum.foreign.name(), "f_id")); + identifiers.add(new Identify("艺术家名称", IdentifyTypeEnum.foreign.name(), "artist_name")); + identifiers.add(new Identify("艺术家名称", IdentifyTypeEnum.foreign.name(), "artist_name")); + modelDetail.setIdentifiers(identifiers); List measures = new ArrayList<>(); @@ -203,7 +208,7 @@ public class LoadBenchMarkDemo implements CommandLineRunner { public void addModelRela_1() { List joinConditions = Lists.newArrayList(); - joinConditions.add(new JoinCondition("preferred_genre", "g_name", FilterOperatorEnum.EQUALS)); + joinConditions.add(new JoinCondition("g_name", "g_name", FilterOperatorEnum.EQUALS)); ModelRela modelRelaReq = new ModelRela(); modelRelaReq.setDomainId(3L); modelRelaReq.setFromModelId(6L); @@ -214,6 +219,42 @@ public class LoadBenchMarkDemo implements CommandLineRunner { } public void addModelRela_2() { + List joinConditions = Lists.newArrayList(); + joinConditions.add(new JoinCondition("artist_name", "artist_name", FilterOperatorEnum.EQUALS)); + ModelRela modelRelaReq = new ModelRela(); + modelRelaReq.setDomainId(3L); + modelRelaReq.setFromModelId(7L); + modelRelaReq.setToModelId(6L); + modelRelaReq.setJoinType("left join"); + modelRelaReq.setJoinConditions(joinConditions); + modelRelaService.save(modelRelaReq, user); + } + + public void addModelRela_3() { + List joinConditions = Lists.newArrayList(); + joinConditions.add(new JoinCondition("artist_name", "artist_name", FilterOperatorEnum.EQUALS)); + ModelRela modelRelaReq = new ModelRela(); + modelRelaReq.setDomainId(3L); + modelRelaReq.setFromModelId(8L); + modelRelaReq.setToModelId(6L); + modelRelaReq.setJoinType("left join"); + modelRelaReq.setJoinConditions(joinConditions); + modelRelaService.save(modelRelaReq, user); + } + + public void addModelRela_4() { + List joinConditions = Lists.newArrayList(); + joinConditions.add(new JoinCondition("g_name", "g_name", FilterOperatorEnum.EQUALS)); + ModelRela modelRelaReq = new ModelRela(); + modelRelaReq.setDomainId(3L); + modelRelaReq.setFromModelId(8L); + modelRelaReq.setToModelId(5L); + modelRelaReq.setJoinType("left join"); + modelRelaReq.setJoinConditions(joinConditions); + modelRelaService.save(modelRelaReq, user); + } + + public void addModelRela_5() { List joinConditions = Lists.newArrayList(); joinConditions.add(new JoinCondition("f_id", "f_id", FilterOperatorEnum.EQUALS)); ModelRela modelRelaReq = new ModelRela(); @@ -224,16 +265,4 @@ public class LoadBenchMarkDemo implements CommandLineRunner { modelRelaReq.setJoinConditions(joinConditions); modelRelaService.save(modelRelaReq, user); } - - public void addModelRela_3() { - List joinConditions = Lists.newArrayList(); - joinConditions.add(new JoinCondition("genre_is", "g_name", FilterOperatorEnum.EQUALS)); - ModelRela modelRelaReq = new ModelRela(); - modelRelaReq.setDomainId(3L); - modelRelaReq.setFromModelId(8L); - modelRelaReq.setToModelId(5L); - modelRelaReq.setJoinType("left join"); - modelRelaReq.setJoinConditions(joinConditions); - modelRelaService.save(modelRelaReq, user); - } } \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_5_10.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_5_10.txt new file mode 100644 index 000000000..26f723f61 --- /dev/null +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_5_10.txt @@ -0,0 +1,4 @@ +美国 _5_10 1 +加拿大 _5_10 1 +锡尔赫特、吉大港、库斯蒂亚 _5_10 1 +孟加拉国 _5_10 3 \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_5_11.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_5_11.txt new file mode 100644 index 000000000..faf66a48c --- /dev/null +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_5_11.txt @@ -0,0 +1,6 @@ +现代 _5_11 1 +tagore _5_11 1 +蓝调 _5_11 1 +流行 _5_11 1 +民间 _5_11 1 +nazrul _5_11 1 \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_6_12.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_6_12.txt new file mode 100644 index 000000000..d7dc8744a --- /dev/null +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_6_12.txt @@ -0,0 +1,4 @@ +美国 _6_12 1 +印度 _6_12 2 +英国 _6_12 1 +孟加拉国 _6_12 2 \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_6_13.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_6_13.txt new file mode 100644 index 000000000..f14388257 --- /dev/null +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_6_13.txt @@ -0,0 +1,2 @@ +男性 _6_13 3 +女性 _6_13 3 \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_7_16.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_7_16.txt new file mode 100644 index 000000000..9cbcab009 --- /dev/null +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_7_16.txt @@ -0,0 +1,2 @@ +mp4 _7_16 4 +mp3 _7_16 2 \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_18.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_18.txt new file mode 100644 index 000000000..a6ea79314 --- /dev/null +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_18.txt @@ -0,0 +1,4 @@ +美国 _8_18 1 +印度 _8_18 2 +英国 _8_18 1 +孟加拉国 _8_18 2 \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_19.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_19.txt new file mode 100644 index 000000000..1a5efd963 --- /dev/null +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_19.txt @@ -0,0 +1,2 @@ +英文 _8_19 2 +孟加拉语 _8_19 4 \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_21.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_21.txt new file mode 100644 index 000000000..f892b3ba6 --- /dev/null +++ b/launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_21.txt @@ -0,0 +1,6 @@ +阿米·奥帕尔·霍伊 _8_21 1 +我的爱 _8_21 1 +打败它 _8_21 1 +阿杰伊阿卡什 _8_21 1 +Tumi#长袍#尼罗布 _8_21 1 +舒克诺#帕塔尔#努普尔#帕埃 _8_21 1 \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/data/dictionary/custom/benchmark_cspider.txt b/launchers/standalone/src/main/resources/data/dictionary/custom/benchmark_cspider.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryStructUtils.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryStructUtils.java index 1c95f4a81..fe40992de 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryStructUtils.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryStructUtils.java @@ -1,5 +1,11 @@ package com.tencent.supersonic.semantic.query.utils; +import static com.tencent.supersonic.common.pojo.Constants.DAY; +import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT; +import static com.tencent.supersonic.common.pojo.Constants.MONTH; +import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE; +import static com.tencent.supersonic.common.pojo.Constants.WEEK; + import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.common.pojo.Aggregator; @@ -28,14 +34,6 @@ import com.tencent.supersonic.semantic.model.domain.Catalog; import com.tencent.supersonic.semantic.model.domain.pojo.EngineTypeEnum; import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement; import com.tencent.supersonic.semantic.query.service.SchemaService; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.apache.logging.log4j.util.Strings; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; - import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.ArrayList; @@ -49,12 +47,13 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; - -import static com.tencent.supersonic.common.pojo.Constants.DAY; -import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT; -import static com.tencent.supersonic.common.pojo.Constants.MONTH; -import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE; -import static com.tencent.supersonic.common.pojo.Constants.WEEK; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.logging.log4j.util.Strings; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; @Slf4j @@ -254,22 +253,11 @@ public class QueryStructUtils { public String generateInternalMetricName(Long modelId, List groups) { String internalMetricNamePrefix = ""; - if (CollectionUtils.isEmpty(groups)) { - log.warn("group is empty!"); - } else { - for (int i = 0; i < groups.size(); i++) { - if (groups.get(i).equalsIgnoreCase("sys_imp_date")) { - continue; - } - DimensionResp dimension = catalog.getDimension(groups.get(i), modelId); - if (Objects.nonNull(dimension) && Strings.isNotEmpty(dimension.getModelBizName())) { - internalMetricNamePrefix = dimension.getModelBizName() + UNDERLINE; - break; - } - } + List dimensions = catalog.getDimensions(Collections.singletonList(modelId)); + if (!CollectionUtils.isEmpty(dimensions)) { + internalMetricNamePrefix = dimensions.get(0).getModelBizName(); } - String internalMetricName = internalMetricNamePrefix + internalMetricNameSuffix; - return internalMetricName; + return internalMetricNamePrefix + UNDERLINE + internalMetricNameSuffix; } public boolean isSupportWith(EngineTypeEnum engineTypeEnum, String version) {