[improvement](project) Support cspider multi-table association, fix dictionary loading, and repair InternalMetric exceptions. (#446)

chat/knowledge/src/main/java/com/tencent/supersonic/knowledge/dictionary/MultiCustomDictionary.java

@@ -222,7 +222,10 @@ public class MultiCustomDictionary extends DynamicCustomDictionary {
                             new BufferedOutputStream(IOUtil.newOutputStream(mainPath + ".bin")));
                     if (customNatureCollector.isEmpty()) {
                         for (int i = Nature.begin.ordinal() + 1; i < Nature.values().length; ++i) {
-                            customNatureCollector.add(Nature.values()[i]);
+                            Nature nature = Nature.values()[i];
+                            if(Objects.nonNull(nature)) {
+                                customNatureCollector.add(nature);
+                            }
                         }
                     }
 

launchers/standalone/src/main/java/com/tencent/supersonic/ConfigureDemo.java

@@ -263,10 +263,32 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
         configService.addConfig(chatConfigBaseReq, user);
     }
 
-    public void addDemoChatConfig_5(Long modelId) {
+    public void addDemoChatConfig_5() {
         ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq();
-        chatConfigBaseReq.setModelId(modelId);
+        chatConfigBaseReq.setModelId(5L);
 
+        ChatDetailConfigReq chatDetailConfig = getChatDetailConfigReq();
+        List<KnowledgeInfoReq> knowledgeInfos = new ArrayList<>();
+        KnowledgeInfoReq knowledgeInfoReq = new KnowledgeInfoReq();
+        knowledgeInfoReq.setSearchEnable(true);
+        knowledgeInfoReq.setItemId(10L);
+        knowledgeInfoReq.setBizName("most_popular_in");
+        knowledgeInfos.add(knowledgeInfoReq);
+
+        KnowledgeInfoReq knowledgeInfoReq2 = new KnowledgeInfoReq();
+        knowledgeInfoReq2.setSearchEnable(true);
+        knowledgeInfoReq2.setItemId(11L);
+        knowledgeInfoReq2.setBizName("g_name");
+        knowledgeInfos.add(knowledgeInfoReq2);
+
+        chatDetailConfig.setKnowledgeInfos(knowledgeInfos);
+        chatConfigBaseReq.setChatDetailConfig(chatDetailConfig);
+        List<RecommendedQuestionReq> recommendedQuestions = new ArrayList<>();
+        chatConfigBaseReq.setRecommendedQuestions(recommendedQuestions);
+        configService.addConfig(chatConfigBaseReq, user);
+    }
+
+    private ChatDetailConfigReq getChatDetailConfigReq() {
         ChatDetailConfigReq chatDetailConfig = new ChatDetailConfigReq();
         ChatDefaultConfigReq chatDefaultConfigDetail = new ChatDefaultConfigReq();
         chatDefaultConfigDetail.setUnit(-1);
@@ -274,6 +296,79 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
         chatDetailConfig.setChatDefaultConfig(chatDefaultConfigDetail);
         ItemVisibility visibility0 = new ItemVisibility();
         chatDetailConfig.setVisibility(visibility0);
+        return chatDetailConfig;
+    }
+
+
+    public void addDemoChatConfig_6() {
+        ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq();
+        chatConfigBaseReq.setModelId(6L);
+
+        ChatDetailConfigReq chatDetailConfig = getChatDetailConfigReq();
+        List<KnowledgeInfoReq> knowledgeInfos = new ArrayList<>();
+        KnowledgeInfoReq knowledgeInfoReq = new KnowledgeInfoReq();
+        knowledgeInfoReq.setSearchEnable(true);
+        knowledgeInfoReq.setItemId(12L);
+        knowledgeInfoReq.setBizName("country");
+        knowledgeInfos.add(knowledgeInfoReq);
+
+        KnowledgeInfoReq knowledgeInfoReq2 = new KnowledgeInfoReq();
+        knowledgeInfoReq2.setSearchEnable(true);
+        knowledgeInfoReq2.setItemId(13L);
+        knowledgeInfoReq2.setBizName("gender");
+        knowledgeInfos.add(knowledgeInfoReq2);
+
+        chatDetailConfig.setKnowledgeInfos(knowledgeInfos);
+        chatConfigBaseReq.setChatDetailConfig(chatDetailConfig);
+        List<RecommendedQuestionReq> recommendedQuestions = new ArrayList<>();
+        chatConfigBaseReq.setRecommendedQuestions(recommendedQuestions);
+        configService.addConfig(chatConfigBaseReq, user);
+    }
+
+    public void addDemoChatConfig_7() {
+        ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq();
+        chatConfigBaseReq.setModelId(7L);
+
+        ChatDetailConfigReq chatDetailConfig = getChatDetailConfigReq();
+        List<KnowledgeInfoReq> knowledgeInfos = new ArrayList<>();
+        KnowledgeInfoReq knowledgeInfoReq = new KnowledgeInfoReq();
+        knowledgeInfoReq.setSearchEnable(true);
+        knowledgeInfoReq.setItemId(16L);
+        knowledgeInfoReq.setBizName("formats");
+        knowledgeInfos.add(knowledgeInfoReq);
+
+        chatDetailConfig.setKnowledgeInfos(knowledgeInfos);
+        chatConfigBaseReq.setChatDetailConfig(chatDetailConfig);
+        List<RecommendedQuestionReq> recommendedQuestions = new ArrayList<>();
+        chatConfigBaseReq.setRecommendedQuestions(recommendedQuestions);
+        configService.addConfig(chatConfigBaseReq, user);
+    }
+
+    public void addDemoChatConfig_8() {
+        ChatConfigBaseReq chatConfigBaseReq = new ChatConfigBaseReq();
+        chatConfigBaseReq.setModelId(8L);
+
+        ChatDetailConfigReq chatDetailConfig = getChatDetailConfigReq();
+        List<KnowledgeInfoReq> knowledgeInfos = new ArrayList<>();
+        KnowledgeInfoReq knowledgeInfoReq = new KnowledgeInfoReq();
+        knowledgeInfoReq.setSearchEnable(true);
+        knowledgeInfoReq.setItemId(18L);
+        knowledgeInfoReq.setBizName("country");
+        knowledgeInfos.add(knowledgeInfoReq);
+
+        KnowledgeInfoReq knowledgeInfoReq2 = new KnowledgeInfoReq();
+        knowledgeInfoReq2.setSearchEnable(true);
+        knowledgeInfoReq2.setItemId(19L);
+        knowledgeInfoReq2.setBizName("languages");
+        knowledgeInfos.add(knowledgeInfoReq2);
+
+        KnowledgeInfoReq knowledgeInfoReq3 = new KnowledgeInfoReq();
+        knowledgeInfoReq3.setSearchEnable(true);
+        knowledgeInfoReq3.setItemId(21L);
+        knowledgeInfoReq3.setBizName("song_name");
+        knowledgeInfos.add(knowledgeInfoReq3);
+
+        chatDetailConfig.setKnowledgeInfos(knowledgeInfos);
         chatConfigBaseReq.setChatDetailConfig(chatDetailConfig);
         List<RecommendedQuestionReq> recommendedQuestions = new ArrayList<>();
         chatConfigBaseReq.setRecommendedQuestions(recommendedQuestions);
@@ -392,10 +487,10 @@ public class ConfigureDemo implements ApplicationListener<ApplicationReadyEvent>
             addDemoChatConfig_2();
             addDemoChatConfig_3();
             addDemoChatConfig_4();
-            addDemoChatConfig_5(5L);
+            addDemoChatConfig_5();
-            addDemoChatConfig_5(6L);
+            addDemoChatConfig_6();
-            addDemoChatConfig_5(7L);
+            addDemoChatConfig_7();
-            addDemoChatConfig_5(8L);
+            addDemoChatConfig_8();
             addPlugin_1();
             addAgent1();
             addAgent2();

launchers/standalone/src/main/java/com/tencent/supersonic/LoadBenchMarkDemo.java

@@ -62,6 +62,8 @@ public class LoadBenchMarkDemo implements CommandLineRunner {
             addModelRela_1();
             addModelRela_2();
             addModelRela_3();
+            addModelRela_4();
+            addModelRela_5();
         } catch (Exception e) {
             log.error("Failed to add bench mark demo data", e);
         }
@@ -187,6 +189,9 @@ public class LoadBenchMarkDemo implements CommandLineRunner {
         List<Identify> identifiers = new ArrayList<>();
         identifiers.add(new Identify("歌曲名称", IdentifyTypeEnum.primary.name(), "song_name"));
         identifiers.add(new Identify("歌曲ID", IdentifyTypeEnum.foreign.name(), "f_id"));
+        identifiers.add(new Identify("艺术家名称", IdentifyTypeEnum.foreign.name(), "artist_name"));
+        identifiers.add(new Identify("艺术家名称", IdentifyTypeEnum.foreign.name(), "artist_name"));
+
         modelDetail.setIdentifiers(identifiers);
 
         List<Measure> measures = new ArrayList<>();
@@ -203,7 +208,7 @@ public class LoadBenchMarkDemo implements CommandLineRunner {
 
     public void addModelRela_1() {
         List<JoinCondition> joinConditions = Lists.newArrayList();
-        joinConditions.add(new JoinCondition("preferred_genre", "g_name", FilterOperatorEnum.EQUALS));
+        joinConditions.add(new JoinCondition("g_name", "g_name", FilterOperatorEnum.EQUALS));
         ModelRela modelRelaReq = new ModelRela();
         modelRelaReq.setDomainId(3L);
         modelRelaReq.setFromModelId(6L);
@@ -214,6 +219,42 @@ public class LoadBenchMarkDemo implements CommandLineRunner {
     }
 
     public void addModelRela_2() {
+        List<JoinCondition> joinConditions = Lists.newArrayList();
+        joinConditions.add(new JoinCondition("artist_name", "artist_name", FilterOperatorEnum.EQUALS));
+        ModelRela modelRelaReq = new ModelRela();
+        modelRelaReq.setDomainId(3L);
+        modelRelaReq.setFromModelId(7L);
+        modelRelaReq.setToModelId(6L);
+        modelRelaReq.setJoinType("left join");
+        modelRelaReq.setJoinConditions(joinConditions);
+        modelRelaService.save(modelRelaReq, user);
+    }
+
+    public void addModelRela_3() {
+        List<JoinCondition> joinConditions = Lists.newArrayList();
+        joinConditions.add(new JoinCondition("artist_name", "artist_name", FilterOperatorEnum.EQUALS));
+        ModelRela modelRelaReq = new ModelRela();
+        modelRelaReq.setDomainId(3L);
+        modelRelaReq.setFromModelId(8L);
+        modelRelaReq.setToModelId(6L);
+        modelRelaReq.setJoinType("left join");
+        modelRelaReq.setJoinConditions(joinConditions);
+        modelRelaService.save(modelRelaReq, user);
+    }
+
+    public void addModelRela_4() {
+        List<JoinCondition> joinConditions = Lists.newArrayList();
+        joinConditions.add(new JoinCondition("g_name", "g_name", FilterOperatorEnum.EQUALS));
+        ModelRela modelRelaReq = new ModelRela();
+        modelRelaReq.setDomainId(3L);
+        modelRelaReq.setFromModelId(8L);
+        modelRelaReq.setToModelId(5L);
+        modelRelaReq.setJoinType("left join");
+        modelRelaReq.setJoinConditions(joinConditions);
+        modelRelaService.save(modelRelaReq, user);
+    }
+
+    public void addModelRela_5() {
         List<JoinCondition> joinConditions = Lists.newArrayList();
         joinConditions.add(new JoinCondition("f_id", "f_id", FilterOperatorEnum.EQUALS));
         ModelRela modelRelaReq = new ModelRela();
@@ -224,16 +265,4 @@ public class LoadBenchMarkDemo implements CommandLineRunner {
         modelRelaReq.setJoinConditions(joinConditions);
         modelRelaService.save(modelRelaReq, user);
     }
-
-    public void addModelRela_3() {
-        List<JoinCondition> joinConditions = Lists.newArrayList();
-        joinConditions.add(new JoinCondition("genre_is", "g_name", FilterOperatorEnum.EQUALS));
-        ModelRela modelRelaReq = new ModelRela();
-        modelRelaReq.setDomainId(3L);
-        modelRelaReq.setFromModelId(8L);
-        modelRelaReq.setToModelId(5L);
-        modelRelaReq.setJoinType("left join");
-        modelRelaReq.setJoinConditions(joinConditions);
-        modelRelaService.save(modelRelaReq, user);
-    }
 }

launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_5_10.txt

@@ -0,0 +1,4 @@
+美国 _5_10 1
+加拿大 _5_10 1
+锡尔赫特、吉大港、库斯蒂亚 _5_10 1
+孟加拉国 _5_10 3

launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_5_11.txt

@@ -0,0 +1,6 @@
+现代 _5_11 1
+tagore _5_11 1
+蓝调 _5_11 1
+流行 _5_11 1
+民间 _5_11 1
+nazrul _5_11 1

launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_6_12.txt

@@ -0,0 +1,4 @@
+美国 _6_12 1
+印度 _6_12 2
+英国 _6_12 1
+孟加拉国 _6_12 2

launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_6_13.txt

@@ -0,0 +1,2 @@
+男性 _6_13 3
+女性 _6_13 3

launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_7_16.txt

@@ -0,0 +1,2 @@
+mp4 _7_16 4
+mp3 _7_16 2

launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_18.txt

@@ -0,0 +1,4 @@
+美国 _8_18 1
+印度 _8_18 2
+英国 _8_18 1
+孟加拉国 _8_18 2

launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_19.txt

@@ -0,0 +1,2 @@
+英文 _8_19 2
+孟加拉语 _8_19 4

launchers/standalone/src/main/resources/data/dictionary/custom/DimValue_8_21.txt

@@ -0,0 +1,6 @@
+阿米·奥帕尔·霍伊 _8_21 1
+我的爱 _8_21 1
+打败它 _8_21 1
+阿杰伊阿卡什 _8_21 1
+Tumi#长袍#尼罗布 _8_21 1
+舒克诺#帕塔尔#努普尔#帕埃 _8_21 1

semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryStructUtils.java

@@ -1,5 +1,11 @@
 package com.tencent.supersonic.semantic.query.utils;
 
+import static com.tencent.supersonic.common.pojo.Constants.DAY;
+import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
+import static com.tencent.supersonic.common.pojo.Constants.MONTH;
+import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
+import static com.tencent.supersonic.common.pojo.Constants.WEEK;
+
 import com.google.common.collect.Lists;
 import com.tencent.supersonic.auth.api.authentication.pojo.User;
 import com.tencent.supersonic.common.pojo.Aggregator;
@@ -28,14 +34,6 @@ import com.tencent.supersonic.semantic.model.domain.Catalog;
 import com.tencent.supersonic.semantic.model.domain.pojo.EngineTypeEnum;
 import com.tencent.supersonic.semantic.query.persistence.pojo.QueryStatement;
 import com.tencent.supersonic.semantic.query.service.SchemaService;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.apache.logging.log4j.util.Strings;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.stereotype.Component;
-import org.springframework.util.CollectionUtils;
-
 import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
@@ -49,12 +47,13 @@ import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
-
+import lombok.extern.slf4j.Slf4j;
-import static com.tencent.supersonic.common.pojo.Constants.DAY;
+import org.apache.commons.lang3.tuple.ImmutablePair;
-import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
+import org.apache.logging.log4j.util.Strings;
-import static com.tencent.supersonic.common.pojo.Constants.MONTH;
+import org.springframework.beans.factory.annotation.Autowired;
-import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
+import org.springframework.beans.factory.annotation.Value;
-import static com.tencent.supersonic.common.pojo.Constants.WEEK;
+import org.springframework.stereotype.Component;
+import org.springframework.util.CollectionUtils;
 
 
 @Slf4j
@@ -254,22 +253,11 @@ public class QueryStructUtils {
 
     public String generateInternalMetricName(Long modelId, List<String> groups) {
         String internalMetricNamePrefix = "";
-        if (CollectionUtils.isEmpty(groups)) {
+        List<DimensionResp> dimensions = catalog.getDimensions(Collections.singletonList(modelId));
-            log.warn("group is empty!");
+        if (!CollectionUtils.isEmpty(dimensions)) {
-        } else {
+            internalMetricNamePrefix = dimensions.get(0).getModelBizName();
-            for (int i = 0; i < groups.size(); i++) {
-                if (groups.get(i).equalsIgnoreCase("sys_imp_date")) {
-                    continue;
         }
-                DimensionResp dimension = catalog.getDimension(groups.get(i), modelId);
+        return internalMetricNamePrefix + UNDERLINE + internalMetricNameSuffix;
-                if (Objects.nonNull(dimension) && Strings.isNotEmpty(dimension.getModelBizName())) {
-                    internalMetricNamePrefix = dimension.getModelBizName() + UNDERLINE;
-                    break;
-                }
-            }
-        }
-        String internalMetricName = internalMetricNamePrefix + internalMetricNameSuffix;
-        return internalMetricName;
     }
 
     public boolean isSupportWith(EngineTypeEnum engineTypeEnum, String version) {