diff --git a/.gitignore b/.gitignore
index d0de5cca8..2e6f44788 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,4 +14,4 @@ assembly/runtime/*
*.umi/
/assembly/deploy
/runtime
-.flattened-pom.xml
\ No newline at end of file
+**/.flattened-pom.xml
\ No newline at end of file
diff --git a/CHANGELOG b/CHANGELOG
index 9a66340cd..88e9053ba 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -7,4 +7,15 @@ davinci 0.3.0 change log
6) modify permission process
7) optimize css style
8) optimize filter
-9) delete view module
\ No newline at end of file
+9) delete view module
+
+
+supersonic 0.6.0 change log
+1) add llm parser and llm api server
+2) support fuzzy mapping
+3) support query filter and domain filter in query and search
+4) support standalone mode
+5) add dsl query in semantic
+6) code architecture adjustment in semantic and chat
+7) add unit testing and integration testing
+8) support dimension and metric alias
\ No newline at end of file
diff --git a/README.md b/README.md
index cb589a349..d8ce42237 100644
--- a/README.md
+++ b/README.md
@@ -4,6 +4,7 @@ English | [中文](README_CN.md)
**SuperSonic is an out-of-the-box yet highly extensible framework for building a data chatbot**. SuperSonic provides a chat interface that empowers users to query data using natural language and visualize the results with suitable charts. To enable such experience, the only thing necessary is to define logical semantic models (metrics, dimensions, relationships, etc) on top of physical data models, and no data modification or copying is required. Meanwhile SuperSonic is designed to be plugable, allowing new functionalities to be added through plugins and core components to be integrated into other systems.
+
## Motivation
@@ -28,8 +29,10 @@ With these ideas in mind, we developed SuperSonic as a reference implementation
SuperSonic is composed of two layers: supersonic-chat and supersonic-semantic. The chat layer is responsible for converting **natural language query** into semantic query (also known as DSL query), whereas the semantic layer is responsible for converting DSL query into **SQL query**. The high-level architecture and main process flow is shown in below diagram:
+
+
### Chat Layer
The chat layer contains four core components:
@@ -64,4 +67,4 @@ SuperSonic comes with sample semantic models as well as chat conversations that
## How to Build
-Pull the source code and run script "assembly/bin/build-standalone.sh" to build packages in the standalone mode.
\ No newline at end of file
+Pull the source code and run script "assembly/bin/build-standalone.sh" to build packages in the standalone mode.
diff --git a/assembly/bin/build-chat.sh b/assembly/bin/build-chat.sh
new file mode 100755
index 000000000..e00ab2fa9
--- /dev/null
+++ b/assembly/bin/build-chat.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+sbinDir=$(cd "$(dirname "$0")"; pwd)
+baseDir=$(readlink -f $sbinDir/../)
+runtimeDir=$baseDir/runtime
+buildDir=$baseDir/build
+
+cd $baseDir
+
+#1. move package to build
+cp $baseDir/../launchers/chat/target/*.tar.gz ${buildDir}/supersonic-chat.tar.gz
+
diff --git a/assembly/bin/build-all.sh b/assembly/bin/build-semantic.sh
similarity index 74%
rename from assembly/bin/build-all.sh
rename to assembly/bin/build-semantic.sh
index 2259177ec..5eefb3845 100755
--- a/assembly/bin/build-all.sh
+++ b/assembly/bin/build-semantic.sh
@@ -7,18 +7,17 @@ buildDir=$baseDir/build
cd $baseDir
-#1. build semantic chat service
+#1. build semantic service
rm -fr ${buildDir}/*.tar.gz
rm -fr dist
mvn -f $baseDir/../ clean package -DskipTests
#2. move package to build
-cp $baseDir/../launchers/chat/target/*.tar.gz ${buildDir}/supersonic-chat.tar.gz
cp $baseDir/../launchers/semantic/target/*.tar.gz ${buildDir}/supersonic-semantic.tar.gz
#3. build webapp
chmod +x $baseDir/../webapp/start-fe-prod.sh
cd ../webapp
sh ./start-fe-prod.sh
-cp -fr ./supersonic-webapp.tar.gz ${buildDir}/
\ No newline at end of file
+cp -fr ./supersonic-webapp.tar.gz ${buildDir}/
diff --git a/assembly/bin/start-chat.sh b/assembly/bin/start-chat.sh
new file mode 100755
index 000000000..87e1920b4
--- /dev/null
+++ b/assembly/bin/start-chat.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+sbinDir=$(cd "$(dirname "$0")"; pwd)
+baseDir=$(readlink -f $sbinDir/../)
+runtimeDir=$baseDir/../runtime
+buildDir=$baseDir/build
+
+cd $baseDir
+
+#2. package lib
+
+tar -zxvf ${buildDir}/supersonic-chat.tar.gz -C ${runtimeDir}
+
+mv ${runtimeDir}/launchers-chat-* ${runtimeDir}/supersonic-chat
+
+tar -zxvf ${buildDir}/supersonic-webapp.tar.gz -C ${buildDir}
+
+mkdir -p ${runtimeDir}/supersonic-chat/webapp
+
+cp -fr ${buildDir}/supersonic-webapp/* ${runtimeDir}/supersonic-chat/webapp
+
+rm -fr ${buildDir}/supersonic-webapp
+
+json=$(cat ${runtimeDir}/supersonic-chat/webapp/supersonic.config.json)
+json=$(echo $json | jq '.env="chat"')
+echo $json > ${runtimeDir}/supersonic-chat/webapp/supersonic.config.json
+
+#3. start service
+#3.1 start chat service
+echo ${runtimeDir}
+sh ${runtimeDir}/supersonic-chat/bin/service.sh restart
+
+#3.2 start llm service
+sh ${runtimeDir}/supersonic-chat/llm/bin/service.sh restart
+
diff --git a/assembly/bin/start-all.sh b/assembly/bin/start-semantic.sh
similarity index 68%
rename from assembly/bin/start-all.sh
rename to assembly/bin/start-semantic.sh
index dc7c5abb5..1c070b83b 100755
--- a/assembly/bin/start-all.sh
+++ b/assembly/bin/start-semantic.sh
@@ -14,23 +14,23 @@ rm -fr ${runtimeDir}/*
#2. package lib
tar -zxvf ${buildDir}/supersonic-semantic.tar.gz -C ${runtimeDir}
-tar -zxvf ${buildDir}/supersonic-chat.tar.gz -C ${runtimeDir}
-mv ${runtimeDir}/launchers-chat-* ${runtimeDir}/supersonic-chat
mv ${runtimeDir}/launchers-semantic-* ${runtimeDir}/supersonic-semantic
tar -zxvf ${buildDir}/supersonic-webapp.tar.gz -C ${buildDir}
mkdir -p ${runtimeDir}/supersonic-semantic/webapp
-mkdir -p ${runtimeDir}/supersonic-chat/webapp
cp -fr ${buildDir}/supersonic-webapp/* ${runtimeDir}/supersonic-semantic/webapp
-cp -fr ${buildDir}/supersonic-webapp/* ${runtimeDir}/supersonic-chat/webapp
rm -fr ${buildDir}/supersonic-webapp
+
+json=$(cat ${runtimeDir}/supersonic-semantic/webapp/supersonic.config.json)
+json=$(echo $json | jq '.env="semantic"')
+echo $json > ${runtimeDir}/supersonic-semantic/webapp/supersonic.config.json
+
#3. start service
sh ${runtimeDir}/supersonic-semantic/bin/service.sh restart
-sleep 5
-sh ${runtimeDir}/supersonic-chat/bin/service.sh restart
+
diff --git a/assembly/bin/start-standalone.sh b/assembly/bin/start-standalone.sh
index e0a180eda..80fb3099d 100755
--- a/assembly/bin/start-standalone.sh
+++ b/assembly/bin/start-standalone.sh
@@ -26,5 +26,7 @@ cp -fr ${buildDir}/supersonic-webapp/* ${runtimeDir}/supersonic-standalone/weba
rm -fr ${buildDir}/supersonic-webapp
#3. start service
+#start standalone service
sh ${runtimeDir}/supersonic-standalone/bin/service.sh restart
-
+#start llm service
+sh ${runtimeDir}/supersonic-standalone/llm/bin/service.sh restart
\ No newline at end of file
diff --git a/assembly/build/build.xml b/assembly/build/build.xml
index 7165bcc00..29d0806a6 100644
--- a/assembly/build/build.xml
+++ b/assembly/build/build.xml
@@ -28,6 +28,12 @@
*.jar
+
+ ${project.basedir}/../../chat/core/src/main/python
+ llm
+ 0777
+ 0755
+
diff --git a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/interceptor/DefaultAuthenticationInterceptor.java b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/interceptor/DefaultAuthenticationInterceptor.java
index b245ece49..c925b21bd 100644
--- a/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/interceptor/DefaultAuthenticationInterceptor.java
+++ b/auth/authentication/src/main/java/com/tencent/supersonic/auth/authentication/domain/interceptor/DefaultAuthenticationInterceptor.java
@@ -2,6 +2,7 @@ package com.tencent.supersonic.auth.authentication.domain.interceptor;
import com.tencent.supersonic.auth.api.authentication.config.AuthenticationConfig;
+import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.auth.api.authentication.pojo.UserWithPassword;
import com.tencent.supersonic.auth.authentication.application.UserServiceImpl;
import com.tencent.supersonic.auth.authentication.domain.utils.UserTokenUtils;
@@ -27,11 +28,12 @@ public class DefaultAuthenticationInterceptor extends AuthenticationInterceptor
userTokenUtils = ContextUtils.getBean(UserTokenUtils.class);
s2ThreadContext = ContextUtils.getBean(S2ThreadContext.class);
if (!authenticationConfig.isEnabled()) {
+ setFakerUser(request);
return true;
}
if (isInternalRequest(request)) {
- String token = userTokenUtils.generateAdminToken();
- reflectSetparam(request, authenticationConfig.getTokenHttpHeaderKey(), token);
+ setFakerUser(request);
+ return true;
}
HandlerMethod handlerMethod = (HandlerMethod) handler;
Method method = handlerMethod.getMethod();
@@ -47,15 +49,25 @@ public class DefaultAuthenticationInterceptor extends AuthenticationInterceptor
UserWithPassword user = userTokenUtils.getUserWithPassword(request);
if (StringUtils.isNotBlank(user.getName())) {
- ThreadContext threadContext = ThreadContext.builder()
- .token(request.getHeader(authenticationConfig.getTokenHttpHeaderKey()))
- .userName(user.getName())
- .build();
- s2ThreadContext.set(threadContext);
+ setContext(user.getName(), request);
return true;
}
throw new AccessException("authentication failed, please login");
}
+ private void setFakerUser(HttpServletRequest request) {
+ String token = userTokenUtils.generateAdminToken();
+ reflectSetparam(request, authenticationConfig.getTokenHttpHeaderKey(), token);
+ setContext(User.getFakeUser().getName(), request);
+ }
+
+ private void setContext(String userName, HttpServletRequest request) {
+ ThreadContext threadContext = ThreadContext.builder()
+ .token(request.getHeader(authenticationConfig.getTokenHttpHeaderKey()))
+ .userName(userName)
+ .build();
+ s2ThreadContext.set(threadContext);
+ }
+
}
diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticLayer.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticLayer.java
index e524d8189..f3c2566f1 100644
--- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticLayer.java
+++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/component/SemanticLayer.java
@@ -1,8 +1,10 @@
package com.tencent.supersonic.chat.api.component;
+import com.github.pagehelper.PageInfo;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
-import com.tencent.supersonic.semantic.api.core.response.DomainSchemaResp;
-import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
+import com.tencent.supersonic.semantic.api.core.request.PageDimensionReq;
+import com.tencent.supersonic.semantic.api.core.request.PageMetricReq;
+import com.tencent.supersonic.semantic.api.core.response.*;
import com.tencent.supersonic.semantic.api.query.request.QuerySqlReq;
import com.tencent.supersonic.semantic.api.query.request.QueryStructReq;
@@ -26,8 +28,24 @@ public interface SemanticLayer {
QueryResultWithSchemaResp queryBySql(QuerySqlReq querySqlReq, User user);
- DomainSchemaResp getDomainSchemaInfo(Long domain);
+ DomainSchemaResp getDomainSchemaInfo(Long domain, Boolean cacheEnable);
List getDomainSchemaInfo(List ids);
+ List getDomainListForViewer();
+
+ List getDomainListForAdmin();
+
+ PageInfo queryDimensionPage(PageDimensionReq pageDimensionCmd);
+
+ PageInfo queryMetricPage(PageMetricReq pageMetricCmd);
+
+// PageInfo queryMetricPage(PageMetricReq pageMetricCmd);
+//
+// PageInfo queryDimensionPage(PageDimensionReq pageDimensionCmd);
+//
+// List getDomainListForAdmin();
+//
+// List getDomainListForViewer();
+
}
diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/ConfigServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/ConfigServiceImpl.java
index 63ba2fa20..4a21a7e0f 100644
--- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/ConfigServiceImpl.java
+++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/ConfigServiceImpl.java
@@ -3,32 +3,25 @@ package com.tencent.supersonic.chat.application;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.component.SemanticLayer;
-import com.tencent.supersonic.chat.domain.pojo.config.ChatConfig;
-import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigBase;
-import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigEditReq;
-import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigFilter;
-import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigInfo;
-import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigRichInfo;
-import com.tencent.supersonic.chat.domain.pojo.config.DefaultMetric;
-import com.tencent.supersonic.chat.domain.pojo.config.EntityRichInfo;
-import com.tencent.supersonic.chat.domain.pojo.config.ItemVisibilityInfo;
-import com.tencent.supersonic.chat.domain.pojo.config.KnowledgeInfo;
-import com.tencent.supersonic.chat.domain.repository.ChatConfigRepository;
-import com.tencent.supersonic.chat.domain.service.ConfigService;
-import com.tencent.supersonic.chat.domain.utils.ChatConfigUtils;
+import com.tencent.supersonic.chat.domain.pojo.config.*;
import com.tencent.supersonic.chat.domain.utils.ComponentFactory;
-import com.tencent.supersonic.chat.domain.utils.DefaultSemanticInternalUtils;
-import com.tencent.supersonic.common.util.json.JsonUtil;
+import com.tencent.supersonic.common.pojo.SchemaItem;
import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
import com.tencent.supersonic.semantic.api.core.response.DomainResp;
import com.tencent.supersonic.semantic.api.core.response.DomainSchemaResp;
import com.tencent.supersonic.semantic.api.core.response.MetricSchemaResp;
+import com.tencent.supersonic.chat.domain.repository.ChatConfigRepository;
+import com.tencent.supersonic.chat.domain.service.ConfigService;
+import com.tencent.supersonic.chat.domain.utils.ChatConfigUtils;
+import com.tencent.supersonic.common.util.json.JsonUtil;
+
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.Function;
import java.util.stream.Collectors;
+
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.context.annotation.Lazy;
@@ -40,33 +33,31 @@ import org.springframework.util.CollectionUtils;
@Service
public class ConfigServiceImpl implements ConfigService {
- private final ChatConfigRepository chaConfigRepository;
+ private final ChatConfigRepository chatConfigRepository;
private final ChatConfigUtils chatConfigUtils;
- private final DefaultSemanticInternalUtils defaultSemanticUtils;
+ private SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
- public ConfigServiceImpl(ChatConfigRepository chaConfigRepository,
- ChatConfigUtils chatConfigUtils,
- @Lazy DefaultSemanticInternalUtils defaultSemanticUtils) {
- this.chaConfigRepository = chaConfigRepository;
+ public ConfigServiceImpl(ChatConfigRepository chatConfigRepository,
+ ChatConfigUtils chatConfigUtils) {
+ this.chatConfigRepository = chatConfigRepository;
this.chatConfigUtils = chatConfigUtils;
- this.defaultSemanticUtils = defaultSemanticUtils;
}
@Override
- public Long addConfig(ChatConfigBase configBaseCmd, User user) {
+ public Long addConfig(ChatConfigBaseReq configBaseCmd, User user) {
log.info("[create domain extend] object:{}", JsonUtil.toString(configBaseCmd, true));
duplicateCheck(configBaseCmd.getDomainId());
permissionCheckLogic(configBaseCmd.getDomainId(), user.getName());
ChatConfig chaConfig = chatConfigUtils.newChatConfig(configBaseCmd, user);
- chaConfigRepository.createConfig(chaConfig);
- return chaConfig.getDomainId();
+ Long id = chatConfigRepository.createConfig(chaConfig);
+ return id;
}
private void duplicateCheck(Long domainId) {
ChatConfigFilter filter = new ChatConfigFilter();
filter.setDomainId(domainId);
- List chaConfigDescList = chaConfigRepository.getChatConfig(filter);
+ List chaConfigDescList = chatConfigRepository.getChatConfig(filter);
if (!CollectionUtils.isEmpty(chaConfigDescList)) {
throw new RuntimeException("chat config existed, no need to add repeatedly");
}
@@ -74,16 +65,16 @@ public class ConfigServiceImpl implements ConfigService {
@Override
- public Long editConfig(ChatConfigEditReq configEditCmd, User user) {
+ public Long editConfig(ChatConfigEditReqReq configEditCmd, User user) {
log.info("[edit domain extend] object:{}", JsonUtil.toString(configEditCmd, true));
if (Objects.isNull(configEditCmd) || Objects.isNull(configEditCmd.getId()) && Objects.isNull(
configEditCmd.getDomainId())) {
throw new RuntimeException("editConfig, id and domainId are not allowed to be empty at the same time");
}
permissionCheckLogic(configEditCmd.getDomainId(), user.getName());
- ChatConfig chaConfig = chatConfigUtils.editChaConfig(configEditCmd, user);
- chaConfigRepository.updateConfig(chaConfig);
- return configEditCmd.getDomainId();
+ ChatConfig chaConfig = chatConfigUtils.editChatConfig(configEditCmd, user);
+ chatConfigRepository.updateConfig(chaConfig);
+ return configEditCmd.getId();
}
@@ -96,98 +87,34 @@ public class ConfigServiceImpl implements ConfigService {
}
@Override
- public List search(ChatConfigFilter filter, User user) {
+ public List search(ChatConfigFilter filter, User user) {
log.info("[search domain extend] object:{}", JsonUtil.toString(filter, true));
- List chaConfigDescList = chaConfigRepository.getChatConfig(filter);
+ List chaConfigDescList = chatConfigRepository.getChatConfig(filter);
return chaConfigDescList;
}
- public ChatConfigInfo fetchConfigByDomainId(Long domainId) {
- return chaConfigRepository.getConfigByDomainId(domainId);
- }
-
- public EntityRichInfo fetchEntityDescByDomainId(Long domainId) {
- SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
- ChatConfigInfo chaConfigDesc = chaConfigRepository.getConfigByDomainId(domainId);
- DomainSchemaResp domainSchemaDesc = semanticLayer.getDomainSchemaInfo(domainId);
- return fetchEntityDescByConfig(chaConfigDesc, domainSchemaDesc);
- }
-
- public EntityRichInfo fetchEntityDescByConfig(ChatConfigInfo chatConfigDesc, DomainSchemaResp domain) {
- Long domainId = chatConfigDesc.getDomainId();
- EntityRichInfo entityDesc = new EntityRichInfo();
- if (Objects.isNull(chatConfigDesc) || Objects.isNull(chatConfigDesc.getEntity())) {
- log.info("domainId:{}, entityDesc info is null", domainId);
- return entityDesc;
- }
-
- entityDesc.setDomainId(domain.getId());
- entityDesc.setDomainBizName(domain.getBizName());
- entityDesc.setDomainName(domain.getName());
- entityDesc.setNames(chatConfigDesc.getEntity().getNames());
-
- entityDesc.setEntityIds(chatConfigUtils.generateDimDesc(chatConfigDesc.getEntity().getEntityIds(), domain));
- entityDesc.setEntityInternalDetailDesc(
- chatConfigUtils.generateEntityDetailData(chatConfigDesc.getEntity().getDetailData(), domain));
- return entityDesc;
+ @Override
+ public ChatConfigResp fetchConfigByDomainId(Long domainId) {
+ return chatConfigRepository.getConfigByDomainId(domainId);
}
- public List fetchDefaultMetricDescByDomainId(Long domainId) {
- SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
- ChatConfigInfo chatConfigDesc = chaConfigRepository.getConfigByDomainId(domainId);
- DomainSchemaResp domainSchemaDesc = semanticLayer.getDomainSchemaInfo(domainId);
- return fetchDefaultMetricDescByConfig(chatConfigDesc, domainSchemaDesc);
- }
-
- public List fetchDefaultMetricDescByConfig(ChatConfigInfo chatConfigDesc, DomainSchemaResp domain) {
- Long domainId = chatConfigDesc.getDomainId();
- List defaultMetricDescList = new ArrayList<>();
- if (Objects.isNull(chatConfigDesc) || CollectionUtils.isEmpty(chatConfigDesc.getDefaultMetrics())) {
- log.info("domainId:{}, defaultMetricDescList info is null", domainId);
- return defaultMetricDescList;
- }
- List metricIds = chatConfigDesc.getDefaultMetrics().stream()
- .map(defaultMetricInfo -> defaultMetricInfo.getMetricId()).collect(Collectors.toList());
- Map metricIdAndDescPair = chatConfigUtils.generateMetricIdAndDescPair(metricIds,
- domain);
- chatConfigDesc.getDefaultMetrics().stream().forEach(defaultMetricInfo -> {
- DefaultMetric defaultMetricDesc = new DefaultMetric();
- BeanUtils.copyProperties(defaultMetricInfo, defaultMetricDesc);
- if (metricIdAndDescPair.containsKey(defaultMetricInfo.getMetricId())) {
- MetricSchemaResp metricDesc = metricIdAndDescPair.get(defaultMetricInfo.getMetricId());
- defaultMetricDesc.setBizName(metricDesc.getBizName());
- defaultMetricDesc.setName(metricDesc.getName());
- }
- defaultMetricDescList.add(defaultMetricDesc);
- });
- return defaultMetricDescList;
- }
-
- public ItemVisibilityInfo fetchVisibilityDescByDomainId(Long domainId) {
- SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
- ChatConfigInfo chatConfigDesc = chaConfigRepository.getConfigByDomainId(domainId);
- DomainSchemaResp domainSchemaDesc = semanticLayer.getDomainSchemaInfo(domainId);
- return fetchVisibilityDescByConfig(chatConfigDesc, domainSchemaDesc);
- }
-
- private ItemVisibilityInfo fetchVisibilityDescByConfig(ChatConfigInfo chatConfigDesc,
- DomainSchemaResp domainSchemaDesc) {
+ private ItemVisibilityInfo fetchVisibilityDescByConfig(ItemVisibility visibility,
+ DomainSchemaResp domainSchemaDesc) {
ItemVisibilityInfo itemVisibilityDesc = new ItemVisibilityInfo();
- Long domainId = chatConfigDesc.getDomainId();
List dimIdAllList = chatConfigUtils.generateAllDimIdList(domainSchemaDesc);
List metricIdAllList = chatConfigUtils.generateAllMetricIdList(domainSchemaDesc);
List blackDimIdList = new ArrayList<>();
List blackMetricIdList = new ArrayList<>();
- if (Objects.nonNull(chatConfigDesc.getVisibility())) {
- if (!CollectionUtils.isEmpty(chatConfigDesc.getVisibility().getBlackDimIdList())) {
- blackDimIdList.addAll(chatConfigDesc.getVisibility().getBlackDimIdList());
+ if (Objects.nonNull(visibility)) {
+ if (!CollectionUtils.isEmpty(visibility.getBlackDimIdList())) {
+ blackDimIdList.addAll(visibility.getBlackDimIdList());
}
- if (!CollectionUtils.isEmpty(chatConfigDesc.getVisibility().getBlackMetricIdList())) {
- blackMetricIdList.addAll(chatConfigDesc.getVisibility().getBlackMetricIdList());
+ if (!CollectionUtils.isEmpty(visibility.getBlackMetricIdList())) {
+ blackMetricIdList.addAll(visibility.getBlackMetricIdList());
}
}
List whiteMetricIdList = metricIdAllList.stream().filter(id -> !blackMetricIdList.contains(id))
@@ -204,31 +131,110 @@ public class ConfigServiceImpl implements ConfigService {
}
@Override
- public ChatConfigRichInfo getConfigRichInfo(Long domainId) {
- SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
- ChatConfigRichInfo chaConfigRichDesc = new ChatConfigRichInfo();
- ChatConfigInfo chatConfigDesc = chaConfigRepository.getConfigByDomainId(domainId);
- if (Objects.isNull(chatConfigDesc)) {
+ public ChatConfigRichResp getConfigRichInfo(Long domainId) {
+ ChatConfigRichResp chatConfigRichResp = new ChatConfigRichResp();
+ ChatConfigResp chatConfigResp = chatConfigRepository.getConfigByDomainId(domainId);
+ if (Objects.isNull(chatConfigResp)) {
log.info("there is no chatConfigDesc for domainId:{}", domainId);
- return chaConfigRichDesc;
+ return chatConfigRichResp;
}
- BeanUtils.copyProperties(chatConfigDesc, chaConfigRichDesc);
+ BeanUtils.copyProperties(chatConfigResp, chatConfigRichResp);
- DomainSchemaResp domainSchemaInfo = semanticLayer.getDomainSchemaInfo(domainId);
- chaConfigRichDesc.setBizName(domainSchemaInfo.getBizName());
- chaConfigRichDesc.setName(domainSchemaInfo.getName());
+ SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
+ DomainSchemaResp domainSchemaInfo = semanticLayer.getDomainSchemaInfo(domainId, false);
+ chatConfigRichResp.setBizName(domainSchemaInfo.getBizName());
+ chatConfigRichResp.setDomainName(domainSchemaInfo.getName());
- chaConfigRichDesc.setKnowledgeInfos(
- fillKnowledgeBizName(chaConfigRichDesc.getKnowledgeInfos(), domainSchemaInfo));
- chaConfigRichDesc.setDefaultMetrics(fetchDefaultMetricDescByConfig(chatConfigDesc, domainSchemaInfo));
- chaConfigRichDesc.setVisibility(fetchVisibilityDescByConfig(chatConfigDesc, domainSchemaInfo));
- chaConfigRichDesc.setEntity(fetchEntityDescByConfig(chatConfigDesc, domainSchemaInfo));
+ chatConfigRichResp.setChatAggRichConfig(fillChatAggRichConfig(domainSchemaInfo, chatConfigResp));
+ chatConfigRichResp.setChatDetailRichConfig(fillChatDetailRichConfig(domainSchemaInfo, chatConfigRichResp, chatConfigResp));
- return chaConfigRichDesc;
+ return chatConfigRichResp;
}
+ private ChatDetailRichConfig fillChatDetailRichConfig(DomainSchemaResp domainSchemaInfo, ChatConfigRichResp chatConfigRichResp, ChatConfigResp chatConfigResp) {
+ if (Objects.isNull(chatConfigResp) || Objects.isNull(chatConfigResp.getChatDetailConfig())) {
+ return null;
+ }
+ ChatDetailRichConfig detailRichConfig = new ChatDetailRichConfig();
+ ChatDetailConfig chatDetailConfig = chatConfigResp.getChatDetailConfig();
+
+ detailRichConfig.setVisibility(fetchVisibilityDescByConfig(chatDetailConfig.getVisibility(), domainSchemaInfo));
+ detailRichConfig.setKnowledgeInfos(fillKnowledgeBizName(chatDetailConfig.getKnowledgeInfos(), domainSchemaInfo));
+ detailRichConfig.setGlobalKnowledgeConfig(chatDetailConfig.getGlobalKnowledgeConfig());
+ detailRichConfig.setChatDefaultConfig(fetchDefaultConfig(chatDetailConfig.getChatDefaultConfig(), domainSchemaInfo));
+
+ detailRichConfig.setEntity(generateRichEntity(chatDetailConfig.getEntity(), domainSchemaInfo));
+ return detailRichConfig;
+ }
+
+ private EntityRichInfo generateRichEntity(Entity entity, DomainSchemaResp domainSchemaInfo) {
+ EntityRichInfo entityRichInfo = new EntityRichInfo();
+ if (Objects.isNull(entity) || Objects.isNull(entity.getEntityId())) {
+ return entityRichInfo;
+ }
+ BeanUtils.copyProperties(entity, entityRichInfo);
+ Map dimIdAndRespPair = domainSchemaInfo.getDimensions().stream()
+ .collect(Collectors.toMap(DimSchemaResp::getId, Function.identity()));
+
+ entityRichInfo.setDimItem(dimIdAndRespPair.get(entity.getEntityId()));
+ return entityRichInfo;
+ }
+
+ private ChatAggRichConfig fillChatAggRichConfig(DomainSchemaResp domainSchemaInfo, ChatConfigResp chatConfigResp) {
+ if (Objects.isNull(chatConfigResp) || Objects.isNull(chatConfigResp.getChatAggConfig())) {
+ return null;
+ }
+ ChatAggConfig chatAggConfig = chatConfigResp.getChatAggConfig();
+ ChatAggRichConfig chatAggRichConfig = new ChatAggRichConfig();
+
+ chatAggRichConfig.setVisibility(fetchVisibilityDescByConfig(chatAggConfig.getVisibility(), domainSchemaInfo));
+ chatAggRichConfig.setKnowledgeInfos(fillKnowledgeBizName(chatAggConfig.getKnowledgeInfos(), domainSchemaInfo));
+ chatAggRichConfig.setGlobalKnowledgeConfig(chatAggConfig.getGlobalKnowledgeConfig());
+ chatAggRichConfig.setChatDefaultConfig(fetchDefaultConfig(chatAggConfig.getChatDefaultConfig(), domainSchemaInfo));
+
+ return chatAggRichConfig;
+ }
+
+ private ChatDefaultRichConfig fetchDefaultConfig(ChatDefaultConfig chatDefaultConfig, DomainSchemaResp domainSchemaInfo) {
+ ChatDefaultRichConfig defaultRichConfig = new ChatDefaultRichConfig();
+ if (Objects.isNull(chatDefaultConfig)) {
+ return defaultRichConfig;
+ }
+ BeanUtils.copyProperties(chatDefaultConfig, defaultRichConfig);
+ Map dimIdAndRespPair = domainSchemaInfo.getDimensions().stream()
+ .collect(Collectors.toMap(DimSchemaResp::getId, Function.identity()));
+
+ Map metricIdAndRespPair = domainSchemaInfo.getMetrics().stream()
+ .collect(Collectors.toMap(MetricSchemaResp::getId, Function.identity()));
+
+ List dimensions = new ArrayList<>();
+ List metrics = new ArrayList<>();
+ if (!CollectionUtils.isEmpty(chatDefaultConfig.getDimensionIds())) {
+ chatDefaultConfig.getDimensionIds().stream().forEach(dimId -> {
+ DimSchemaResp dimSchemaResp = dimIdAndRespPair.get(dimId);
+ SchemaItem dimSchema = new SchemaItem();
+ BeanUtils.copyProperties(dimSchemaResp, dimSchema);
+ dimensions.add(dimSchema);
+ });
+ }
+
+ if (!CollectionUtils.isEmpty(chatDefaultConfig.getMetricIds())) {
+ chatDefaultConfig.getMetricIds().stream().forEach(metricId -> {
+ MetricSchemaResp metricSchemaResp = metricIdAndRespPair.get(metricId);
+ SchemaItem metricSchema = new SchemaItem();
+ BeanUtils.copyProperties(metricSchemaResp, metricSchema);
+ metrics.add(metricSchema);
+ });
+ }
+
+ defaultRichConfig.setDimensions(dimensions);
+ defaultRichConfig.setMetrics(metrics);
+ return defaultRichConfig;
+ }
+
+
private List fillKnowledgeBizName(List knowledgeInfos,
- DomainSchemaResp domainSchemaInfo) {
+ DomainSchemaResp domainSchemaInfo) {
if (CollectionUtils.isEmpty(knowledgeInfos)) {
return new ArrayList<>();
}
@@ -240,30 +246,21 @@ public class ConfigServiceImpl implements ConfigService {
if (Objects.nonNull(dimSchemaResp)) {
knowledgeInfo.setBizName(dimSchemaResp.getBizName());
}
- if (CollectionUtils.isEmpty(knowledgeInfo.getBlackList())) {
- knowledgeInfo.setBlackList(new ArrayList<>());
- }
- if (CollectionUtils.isEmpty(knowledgeInfo.getRuleList())) {
- knowledgeInfo.setRuleList(new ArrayList<>());
- }
- if (CollectionUtils.isEmpty(knowledgeInfo.getWhiteList())) {
- knowledgeInfo.setWhiteList(new ArrayList<>());
- }
}
});
return knowledgeInfos;
}
@Override
- public List getAllChatRichConfig() {
- List chatConfigRichInfoList = new ArrayList<>();
- List domainRespList = defaultSemanticUtils.getDomainListForAdmin();
+ public List getAllChatRichConfig() {
+ List chatConfigRichInfoList = new ArrayList<>();
+ List domainRespList = semanticLayer.getDomainListForAdmin();
domainRespList.stream().forEach(domainResp -> {
- ChatConfigRichInfo chatConfigRichInfo = getConfigRichInfo(domainResp.getId());
+ ChatConfigRichResp chatConfigRichInfo = getConfigRichInfo(domainResp.getId());
if (Objects.nonNull(chatConfigRichInfo)) {
chatConfigRichInfoList.add(chatConfigRichInfo);
}
});
return chatConfigRichInfoList;
}
-}
\ No newline at end of file
+}
diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/DomainEntityService.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/DomainEntityService.java
index 935bb9090..41cc10d6a 100644
--- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/DomainEntityService.java
+++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/DomainEntityService.java
@@ -8,15 +8,14 @@ import com.tencent.supersonic.chat.api.pojo.DomainInfo;
import com.tencent.supersonic.chat.api.pojo.EntityInfo;
import com.tencent.supersonic.chat.api.pojo.Filter;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
-import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigRichInfo;
+import com.tencent.supersonic.chat.api.component.SemanticLayer;
+import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigRichResp;
+import com.tencent.supersonic.chat.domain.pojo.config.ChatDefaultRichConfig;
import com.tencent.supersonic.chat.domain.pojo.config.EntityRichInfo;
import com.tencent.supersonic.chat.domain.utils.ComponentFactory;
-import com.tencent.supersonic.chat.domain.utils.DefaultSemanticInternalUtils;
import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.SchemaItem;
-import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
-import com.tencent.supersonic.semantic.api.core.response.MetricSchemaResp;
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import java.util.ArrayList;
@@ -37,7 +36,7 @@ public class DomainEntityService {
private SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
@Autowired
- private DefaultSemanticInternalUtils defaultSemanticUtils;
+ private ConfigServiceImpl configService;
public EntityInfo getEntityInfo(SemanticParseInfo parseInfo, User user) {
if (parseInfo != null && parseInfo.getDomainId() > 0) {
@@ -77,42 +76,55 @@ public class DomainEntityService {
}
public EntityInfo getEntityInfo(Long domain) {
- ChatConfigRichInfo chaConfigRichDesc = defaultSemanticUtils.getChatConfigRichInfo(domain);
- return getEntityInfo(chaConfigRichDesc.getEntity());
+ ChatConfigRichResp chaConfigRichDesc = configService.getConfigRichInfo(domain);
+ if (Objects.isNull(chaConfigRichDesc) || Objects.isNull(chaConfigRichDesc.getChatDetailRichConfig())) {
+ return new EntityInfo();
+ }
+ return getEntityInfo(chaConfigRichDesc);
}
- private EntityInfo getEntityInfo(EntityRichInfo entityDesc) {
- EntityInfo entityInfo = new EntityInfo();
+ private EntityInfo getEntityInfo(ChatConfigRichResp chaConfigRichDesc) {
- if (entityDesc != null && Objects.nonNull(entityDesc.getDomainId())) {
+ EntityInfo entityInfo = new EntityInfo();
+ EntityRichInfo entityDesc = chaConfigRichDesc.getChatDetailRichConfig().getEntity();
+ if (entityDesc != null && Objects.nonNull(chaConfigRichDesc.getDomainId())) {
DomainInfo domainInfo = new DomainInfo();
- domainInfo.setItemId(Integer.valueOf(entityDesc.getDomainId().intValue()));
- domainInfo.setName(entityDesc.getDomainName());
+ domainInfo.setItemId(Integer.valueOf(chaConfigRichDesc.getDomainId().intValue()));
+ domainInfo.setName(chaConfigRichDesc.getDomainName());
domainInfo.setWords(entityDesc.getNames());
- domainInfo.setBizName(entityDesc.getDomainBizName());
- if (entityDesc.getEntityIds().size() > 0) {
- domainInfo.setPrimaryEntityBizName(entityDesc.getEntityIds().get(0).getBizName());
+ domainInfo.setBizName(chaConfigRichDesc.getBizName());
+ if (Objects.nonNull(entityDesc.getDimItem())) {
+ domainInfo.setPrimaryEntityBizName(entityDesc.getDimItem().getBizName());
}
+
entityInfo.setDomainInfo(domainInfo);
List dimensions = new ArrayList<>();
List metrics = new ArrayList<>();
- if (entityDesc.getEntityInternalDetailDesc() != null) {
- for (DimSchemaResp dimensionDesc : entityDesc.getEntityInternalDetailDesc().getDimensionList()) {
- DataInfo mainEntityDimension = new DataInfo();
- mainEntityDimension.setItemId(dimensionDesc.getId().intValue());
- mainEntityDimension.setName(dimensionDesc.getName());
- mainEntityDimension.setBizName(dimensionDesc.getBizName());
- dimensions.add(mainEntityDimension);
+
+ if (Objects.nonNull(chaConfigRichDesc) && Objects.nonNull(chaConfigRichDesc.getChatDetailRichConfig())
+ && Objects.nonNull(chaConfigRichDesc.getChatDetailRichConfig().getChatDefaultConfig())) {
+ ChatDefaultRichConfig chatDefaultConfig = chaConfigRichDesc.getChatDetailRichConfig().getChatDefaultConfig();
+ if(!CollectionUtils.isEmpty(chatDefaultConfig.getDimensions())){
+ for (SchemaItem dimensionDesc : chatDefaultConfig.getDimensions()) {
+ DataInfo mainEntityDimension = new DataInfo();
+ mainEntityDimension.setItemId(dimensionDesc.getId().intValue());
+ mainEntityDimension.setName(dimensionDesc.getName());
+ mainEntityDimension.setBizName(dimensionDesc.getBizName());
+ dimensions.add(mainEntityDimension);
+ }
+ entityInfo.setDimensions(dimensions);
}
- entityInfo.setDimensions(dimensions);
- for (MetricSchemaResp metricDesc : entityDesc.getEntityInternalDetailDesc().getMetricList()) {
- DataInfo dataInfo = new DataInfo();
- dataInfo.setName(metricDesc.getName());
- dataInfo.setBizName(metricDesc.getBizName());
- dataInfo.setItemId(metricDesc.getId().intValue());
- metrics.add(dataInfo);
+
+ if(!CollectionUtils.isEmpty(chatDefaultConfig.getMetrics())){
+ for (SchemaItem metricDesc : chatDefaultConfig.getMetrics()) {
+ DataInfo dataInfo = new DataInfo();
+ dataInfo.setName(metricDesc.getName());
+ dataInfo.setBizName(metricDesc.getBizName());
+ dataInfo.setItemId(metricDesc.getId().intValue());
+ metrics.add(dataInfo);
+ }
+ entityInfo.setMetrics(metrics);
}
- entityInfo.setMetrics(metrics);
}
}
return entityInfo;
diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/QueryServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/QueryServiceImpl.java
index 46d2b8a2a..4fdde6e91 100644
--- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/QueryServiceImpl.java
+++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/QueryServiceImpl.java
@@ -14,6 +14,7 @@ import com.tencent.supersonic.chat.domain.service.QueryService;
import com.tencent.supersonic.chat.domain.service.ChatService;
import com.tencent.supersonic.chat.domain.utils.ComponentFactory;
import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter;
+import com.tencent.supersonic.common.util.json.JsonUtil;
import com.tencent.supersonic.semantic.api.core.response.QueryResultWithSchemaResp;
import java.util.List;
import java.util.stream.Collectors;
@@ -50,7 +51,6 @@ public class QueryServiceImpl implements QueryService {
log.info("semanticParser processing:[{}]", semanticParser.getClass().getName());
semanticParser.parse(queryCtx, chatCtx);
}
-
if (queryCtx.getCandidateQueries().size() > 0) {
log.info("pick before [{}]", queryCtx.getCandidateQueries().stream().collect(
Collectors.toList()));
diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/RecommendServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/RecommendServiceImpl.java
index 6044b08d0..b88ce07e4 100644
--- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/RecommendServiceImpl.java
+++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/RecommendServiceImpl.java
@@ -29,7 +29,7 @@ public class RecommendServiceImpl implements RecommendService {
}
DomainSchemaResp domainSchemaDesc = semanticLayer.getDomainSchemaInfo(
- Long.valueOf(domainId));
+ Long.valueOf(domainId), true);
List dimensions = domainSchemaDesc.getDimensions().stream().map(dimSchemaDesc -> {
RecommendResponse.Item item = new RecommendResponse.Item();
diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/SearchServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/SearchServiceImpl.java
index f9361d1fa..47ba6529a 100644
--- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/SearchServiceImpl.java
+++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/SearchServiceImpl.java
@@ -2,6 +2,8 @@ package com.tencent.supersonic.chat.application;
import com.google.common.collect.Lists;
import com.hankcs.hanlp.seg.common.Term;
+import com.tencent.supersonic.chat.api.pojo.Filter;
+import com.tencent.supersonic.chat.api.pojo.QueryFilter;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.request.QueryContextReq;
import com.tencent.supersonic.chat.application.knowledge.NatureHelper;
@@ -64,8 +66,7 @@ public class SearchServiceImpl implements SearchService {
List metricsDb = domainInfosDb.getMetrics();
final Map domainToName = domainInfosDb.getDomainToName();
// 2.detect by segment
- List originals = HanlpHelper.getSegment().seg(queryText.toLowerCase()).stream()
- .collect(Collectors.toList());
+ List originals = HanlpHelper.getTerms(queryText);
Map> regTextMap = searchMatchStrategy.match(queryText, originals,
queryCtx.getDomainId());
regTextMap.entrySet().stream().forEach(m -> HanlpHelper.transLetterOriginal(m.getValue()));
@@ -101,7 +102,7 @@ public class SearchServiceImpl implements SearchService {
for (Map.Entry natureToNameEntry : natureToNameMap.entrySet()) {
searchDimensionValue(metricsDb, domainToName, domainStat.getMetricDomainCount(), searchResults,
- existMetricAndDimension, matchText, natureToNameMap, natureToNameEntry);
+ existMetricAndDimension, matchText, natureToNameMap, natureToNameEntry, queryCtx.getQueryFilter());
}
return searchResults.stream().limit(RESULT_SIZE).collect(Collectors.toList());
}
@@ -140,13 +141,14 @@ public class SearchServiceImpl implements SearchService {
}
private void searchDimensionValue(List metricsDb,
- Map domainToName,
- long metricDomainCount,
- Set searchResults,
- boolean existMetricAndDimension,
- MatchText matchText,
- Map natureToNameMap,
- Map.Entry natureToNameEntry) {
+ Map domainToName,
+ long metricDomainCount,
+ Set searchResults,
+ boolean existMetricAndDimension,
+ MatchText matchText,
+ Map natureToNameMap,
+ Map.Entry natureToNameEntry,
+ QueryFilter queryFilter) {
String nature = natureToNameEntry.getKey();
String wordName = natureToNameEntry.getValue();
@@ -158,6 +160,9 @@ public class SearchServiceImpl implements SearchService {
}
// If there are no metric/dimension, complete the metric information
if (metricDomainCount <= 0 && !existMetricAndDimension) {
+ if (filterByQueryFilter(matchText.getRegText(), queryFilter)) {
+ return;
+ }
searchResults.add(
new SearchResult(matchText.getRegText() + wordName, wordName, domainToName.get(domain), domain,
schemaElementType));
@@ -167,7 +172,6 @@ public class SearchServiceImpl implements SearchService {
}
List metrics = filerMetricsByDomain(metricsDb, domain).stream().limit(metricSize).collect(
Collectors.toList());
- ;
for (String metric : metrics) {
String subRecommend = matchText.getRegText() + wordName + NatureType.SPACE + metric;
searchResults.add(
@@ -181,6 +185,19 @@ public class SearchServiceImpl implements SearchService {
}
}
+ private boolean filterByQueryFilter(String regText, QueryFilter queryFilter) {
+ if (queryFilter == null || CollectionUtils.isEmpty(queryFilter.getFilters())) {
+ return false;
+ }
+ List filters = queryFilter.getFilters();
+ for (Filter filter : filters) {
+ if (regText.equalsIgnoreCase(String.valueOf(filter.getValue()))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
protected List filerMetricsByDomain(List metricsDb, Integer domain) {
if (CollectionUtils.isEmpty(metricsDb)) {
return Lists.newArrayList();
diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/DatabaseSchemaMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/DatabaseSchemaMapper.java
index d9b5173b8..f22f1ac2c 100644
--- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/DatabaseSchemaMapper.java
+++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/DatabaseSchemaMapper.java
@@ -31,8 +31,7 @@ public class DatabaseSchemaMapper implements SchemaMapper {
log.debug("before db mapper,mapInfo:{}", queryContext.getMapInfo());
- List terms = HanlpHelper.getSegment().seg(queryContext.getQueryText().toLowerCase()).stream()
- .collect(Collectors.toList());
+ List terms = HanlpHelper.getTerms(queryContext.getQueryText());
WordNatureService wordNatureService = ContextUtils.getBean(WordNatureService.class);
diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/HanlpSchemaMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/HanlpSchemaMapper.java
index 56fc6cb02..13f1709b4 100644
--- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/HanlpSchemaMapper.java
+++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/HanlpSchemaMapper.java
@@ -30,8 +30,7 @@ public class HanlpSchemaMapper implements SchemaMapper {
@Override
public void map(QueryContextReq queryContext) {
- List terms = HanlpHelper.getSegment().seg(queryContext.getQueryText().toLowerCase()).stream()
- .collect(Collectors.toList());
+ List terms = HanlpHelper.getTerms(queryContext.getQueryText());
terms.forEach(
item -> log.info("word:{},nature:{},frequency:{}", item.word, item.nature.toString(),
@@ -56,6 +55,8 @@ public class HanlpSchemaMapper implements SchemaMapper {
convertTermsToSchemaMapInfo(matches, queryContext.getMapInfo(), terms);
}
+
+
private void convertTermsToSchemaMapInfo(List mapResults, SchemaMapInfo schemaMap, List terms) {
if (CollectionUtils.isEmpty(mapResults)) {
return;
diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/QueryFilterMapper.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/QueryFilterMapper.java
index dccf8290b..5d4977a65 100644
--- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/QueryFilterMapper.java
+++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/mapper/QueryFilterMapper.java
@@ -4,9 +4,12 @@ import com.google.common.collect.Lists;
import com.tencent.supersonic.chat.api.component.SchemaMapper;
import com.tencent.supersonic.chat.api.pojo.*;
import com.tencent.supersonic.chat.api.request.QueryContextReq;
+import com.tencent.supersonic.common.constant.Constants;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.CollectionUtils;
import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
@Slf4j
@@ -25,6 +28,7 @@ public class QueryFilterMapper implements SchemaMapper {
QueryFilter queryFilter = queryContext.getQueryFilter();
SchemaMapInfo schemaMapInfo = queryContext.getMapInfo();
List schemaElementMatches = schemaMapInfo.getMatchedElements(domainId);
+ clearOtherSchemaElementMatch(domainId, schemaMapInfo);
convertFilterToSchemaMapInfo(queryFilter.getFilters(), schemaElementMatches);
}
@@ -33,6 +37,7 @@ public class QueryFilterMapper implements SchemaMapper {
if (CollectionUtils.isEmpty(schemaElementMatches)) {
schemaElementMatches = Lists.newArrayList();
}
+ List words = schemaElementMatches.stream().map(SchemaElementMatch::getWord).collect(Collectors.toList());
for (Filter filter : filters) {
SchemaElementMatch schemaElementMatch = SchemaElementMatch.builder()
.elementType(SchemaElementType.VALUE)
@@ -40,11 +45,22 @@ public class QueryFilterMapper implements SchemaMapper {
.frequency(FREQUENCY)
.word(String.valueOf(filter.getValue()))
.similarity(SIMILARITY)
- .detectWord(filter.getName())
+ .detectWord(Constants.EMPTY)
.build();
+ if (words.contains(schemaElementMatch.getWord())) {
+ continue;
+ }
schemaElementMatches.add(schemaElementMatch);
}
log.info("schemaElementMatches after queryFilerMapper:{}", schemaElementMatches);
}
+ private void clearOtherSchemaElementMatch(Integer domainId, SchemaMapInfo schemaMapInfo) {
+ for (Map.Entry> entry : schemaMapInfo.getDomainElementMatches().entrySet()) {
+ if (!entry.getKey().equals(domainId)) {
+ entry.getValue().clear();
+ }
+ }
+ }
+
}
diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/DomainSemanticParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/DomainSemanticParser.java
index 32cb52923..b24be50fc 100644
--- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/DomainSemanticParser.java
+++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/DomainSemanticParser.java
@@ -2,24 +2,28 @@ package com.tencent.supersonic.chat.application.parser;
import com.tencent.supersonic.chat.api.component.SemanticLayer;
import com.tencent.supersonic.chat.api.component.SemanticParser;
-import com.tencent.supersonic.chat.api.component.SemanticQuery;
-import com.tencent.supersonic.chat.api.pojo.*;
+import com.tencent.supersonic.chat.api.pojo.ChatContext;
+import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
+import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
+import com.tencent.supersonic.chat.api.pojo.SchemaMapInfo;
+import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.request.QueryContextReq;
-import com.tencent.supersonic.chat.application.query.*;
+import com.tencent.supersonic.chat.application.query.EntitySemanticQuery;
+import com.tencent.supersonic.chat.application.query.MetricSemanticQuery;
+import com.tencent.supersonic.chat.application.query.RuleSemanticQuery;
+import com.tencent.supersonic.chat.application.query.RuleSemanticQueryManager;
import com.tencent.supersonic.chat.domain.pojo.chat.DomainInfos;
-import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigRichInfo;
-import com.tencent.supersonic.chat.domain.utils.*;
-import com.tencent.supersonic.common.pojo.SchemaItem;
+import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigResp;
+import com.tencent.supersonic.chat.domain.service.ConfigService;
+import com.tencent.supersonic.chat.domain.utils.ComponentFactory;
+import com.tencent.supersonic.chat.domain.utils.ContextHelper;
+import com.tencent.supersonic.chat.domain.utils.DefaultMetricUtils;
+import com.tencent.supersonic.chat.domain.utils.SchemaInfoConverter;
import com.tencent.supersonic.common.util.context.ContextUtils;
import java.util.*;
-import java.util.function.Function;
import java.util.stream.Collectors;
-import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
-import com.tencent.supersonic.semantic.api.core.response.DomainSchemaResp;
-import com.tencent.supersonic.semantic.api.core.response.MetricSchemaResp;
-import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.CollectionUtils;
@@ -37,10 +41,15 @@ public class DomainSemanticParser implements SemanticParser {
// iterate all schemaElementMatches to resolve semantic query
for (Integer domainId : mapInfo.getMatchedDomains()) {
List elementMatches = mapInfo.getMatchedElements(domainId);
- Map> queryMatches = resolveQuery(elementMatches, queryContext);
- for (Map.Entry> match : queryMatches.entrySet()) {
+ List queries = resolveQuery(elementMatches, queryContext);
+ for (RuleSemanticQuery query : queries) {
+
+ if (useBlackItem(query, domainId)) {
+ log.info("useBlackItem, skip query:{}", query);
+ continue;
+ }
addCandidateQuery(queryContext, chatContext, domainId.longValue(),
- domainToName.get(domainId), match.getKey(), match.getValue());
+ domainToName.get(domainId), query);
}
}
@@ -50,27 +59,95 @@ public class DomainSemanticParser implements SemanticParser {
Integer chatDomainId = Integer.valueOf(chatContext.getParseInfo().getDomainId().intValue());
if (mapInfo.getMatchedDomains().contains(chatDomainId)) {
List elementMatches = mapInfo.getMatchedElements(chatDomainId);
- detectionContext(chatContext);
- Map> queryMatches = tryParseByContext(elementMatches,
- chatContext, queryContext);
- for (Map.Entry> match : queryMatches.entrySet()) {
+
+ List queries = tryParseByContext(elementMatches, chatContext, queryContext);
+ for (RuleSemanticQuery query : queries) {
addCandidateQuery(queryContext, chatContext, chatDomainId.longValue(),
- domainToName.get(chatDomainId), match.getKey(), match.getValue());
+ domainToName.get(chatDomainId), query);
}
}
}
}
}
+ private boolean useBlackItem(RuleSemanticQuery query, Integer domainId) {
+ if (Objects.isNull(domainId)) {
+ return false;
+ }
+ ConfigService configService = ContextUtils.getBean(ConfigService.class);
+ ChatConfigResp chatConfigResp = configService.fetchConfigByDomainId(domainId.longValue());
+ if (Objects.nonNull(chatConfigResp) && Objects.nonNull(query) && Objects.nonNull(query.getParseInfo())) {
+ List elementMatches = query.getParseInfo().getElementMatches();
+ if (!CollectionUtils.isEmpty(elementMatches)) {
+ return useBlackItemInternal(elementMatches, chatConfigResp, query);
+
+ }
+ }
+ return false;
+ }
+
+ private boolean useBlackItemInternal(List elementMatches, ChatConfigResp chatConfigResp, RuleSemanticQuery query) {
+ if (Objects.isNull(chatConfigResp)) {
+ return false;
+ }
+ List blackDimIdList = new ArrayList<>();
+ List blackMetricIdList = new ArrayList<>();
+ if (query instanceof EntitySemanticQuery
+ && Objects.nonNull(chatConfigResp.getChatDetailConfig())
+ && Objects.nonNull(chatConfigResp.getChatDetailConfig().getVisibility())) {
+ log.info("useBlackItem, handle EntitySemanticQuery blackList logic");
+ blackDimIdList = chatConfigResp.getChatDetailConfig().getVisibility().getBlackDimIdList();
+ blackMetricIdList = chatConfigResp.getChatDetailConfig().getVisibility().getBlackMetricIdList();
+ }
+
+ if (query instanceof MetricSemanticQuery
+ && Objects.nonNull(chatConfigResp.getChatAggConfig())
+ && Objects.nonNull(chatConfigResp.getChatAggConfig().getVisibility())) {
+ log.info("useBlackItem, handle MetricSemanticQuery blackList logic");
+ blackDimIdList = chatConfigResp.getChatAggConfig().getVisibility().getBlackDimIdList();
+ blackMetricIdList = chatConfigResp.getChatAggConfig().getVisibility().getBlackMetricIdList();
+ }
+ return useBlackItemWithElementMatches(elementMatches, blackDimIdList, blackMetricIdList);
+ }
+
+ private boolean useBlackItemWithElementMatches(List elementMatches, List blackDimIdList, List blackMetricIdList) {
+
+ Set dimIds = elementMatches.stream()
+ .filter(element -> SchemaElementType.VALUE.equals(element.getElementType()) || SchemaElementType.DIMENSION.equals(element.getElementType()))
+ .map(element -> Long.valueOf(element.getElementID())).collect(Collectors.toSet());
+
+ Set metricIds = elementMatches.stream()
+ .filter(element -> SchemaElementType.METRIC.equals(element.getElementType()))
+ .map(element -> Long.valueOf(element.getElementID())).collect(Collectors.toSet());
+
+
+ return useBlackItemWithIds(dimIds, metricIds, blackDimIdList, blackMetricIdList);
+ }
+
+ private boolean useBlackItemWithIds(Set dimIds, Set metricIds, List blackDimIdList, List blackMetricIdList) {
+
+ if (!CollectionUtils.isEmpty(blackDimIdList) && !CollectionUtils.isEmpty(dimIds)) {
+ if (blackDimIdList.stream().anyMatch(dimIds::contains)) {
+ log.info("useBlackItem, blackDimIdList:{}", blackDimIdList.stream().filter(dimIds::contains).collect(Collectors.toList()));
+ return true;
+ }
+ }
+ if (!CollectionUtils.isEmpty(blackMetricIdList) && !CollectionUtils.isEmpty(metricIds)) {
+ if (blackMetricIdList.stream().anyMatch(metricIds::contains)) {
+ log.info("useBlackItem, blackMetricIdList:{}", blackMetricIdList.stream().filter(metricIds::contains).collect(Collectors.toList()));
+ return true;
+ }
+ }
+ return false;
+ }
+
private void addCandidateQuery(QueryContextReq queryContext, ChatContext chatContext,
- Long domainId, String domainName,
- RuleSemanticQuery semanticQuery, List elementMatches) {
+ Long domainId, String domainName, RuleSemanticQuery semanticQuery) {
if (semanticQuery != null) {
- fillParseInfo(semanticQuery, domainId, domainName, elementMatches);
- // inherit from context
+ DefaultMetricUtils defaultMetricUtils = ContextUtils.getBean(DefaultMetricUtils.class);
+ defaultMetricUtils.fillParseInfo(semanticQuery, domainId, domainName);
inheritContext(semanticQuery, chatContext);
- // default metric, date, dimension
- injectDefaultMetric(semanticQuery, queryContext, chatContext);
+ defaultMetricUtils.fillDefaultMetric(semanticQuery.getParseInfo(), queryContext, chatContext);
queryContext.getCandidateQueries().add(semanticQuery);
}
}
@@ -84,38 +161,15 @@ public class DomainSemanticParser implements SemanticParser {
}
}
- protected void injectDefaultMetric(RuleSemanticQuery semanticQuery, QueryContextReq queryContext,
- ChatContext chatContext) {
- DefaultMetricUtils defaultMetricUtils = ContextUtils.getBean(DefaultMetricUtils.class);
- defaultMetricUtils.injectDefaultMetric(semanticQuery.getParseInfo(), queryContext, chatContext);
- }
-
/**
- * get the chatContext for the tryParseByContext
- *
- * @param chatContext
- */
- protected void detectionContext(ChatContext chatContext) {
- if (chatContext.getParseInfo() != null) {
- SemanticParseInfo semanticParseInfo = chatContext.getParseInfo();
- if (semanticParseInfo.getQueryMode().equals(EntityDetail.QUERY_MODE)) {
- // EntityDetail model will unset some items
- semanticParseInfo.setDateInfo(null);
- semanticParseInfo.setMetrics(new HashSet<>());
- semanticParseInfo.setDimensions(new HashSet<>());
- }
- }
- }
-
- /**
- * try to add ChatContext to SchemaElementMatch and look if match QueryMode
+ * try to add ChatContext to SchemaMatch and look if match QueryMode
*
* @param elementMatches
* @param chatCtx
* @return
*/
- private Map> tryParseByContext(List elementMatches,
- ChatContext chatCtx, QueryContextReq queryCtx) {
+ private List tryParseByContext(List elementMatches,
+ ChatContext chatCtx, QueryContextReq queryCtx) {
if (chatCtx.getParseInfo() != null && chatCtx.getParseInfo().getEntity() > 0) {
Long entityCount = elementMatches.stream().filter(i -> SchemaElementType.ENTITY.equals(i.getElementType()))
.count();
@@ -125,15 +179,14 @@ public class DomainSemanticParser implements SemanticParser {
// try entity parse
SchemaElementMatch entityElementMatch = SchemaElementMatch.builder()
.elementType(SchemaElementType.ENTITY).build();
- List newSchemaElementMatch = new ArrayList<>();
+ List newSchemaMatches = new ArrayList<>();
if (!CollectionUtils.isEmpty(elementMatches)) {
- newSchemaElementMatch.addAll(elementMatches);
+ newSchemaMatches.addAll(elementMatches);
}
- newSchemaElementMatch.add(entityElementMatch);
- Map> queryMatches = doParseByContext(newSchemaElementMatch,
- chatCtx, queryCtx);
- if (queryMatches.size() > 0) {
- return queryMatches;
+ newSchemaMatches.add(entityElementMatch);
+ List queries = doParseByContext(newSchemaMatches, chatCtx, queryCtx);
+ if (queries.size() > 0) {
+ return queries;
}
}
}
@@ -141,8 +194,8 @@ public class DomainSemanticParser implements SemanticParser {
}
- private Map> doParseByContext(List elementMatches,
- ChatContext chatCtx, QueryContextReq queryCtx) {
+ private List doParseByContext(List elementMatches,
+ ChatContext chatCtx, QueryContextReq queryContext) {
SemanticParseInfo contextSemanticParse = chatCtx.getParseInfo();
if (contextSemanticParse != null) {
List newElementMatches = new ArrayList<>();
@@ -162,123 +215,36 @@ public class DomainSemanticParser implements SemanticParser {
trySchemaElementTypes.add(new ArrayList<>(Arrays.asList(SchemaElementType.VALUE)));
trySchemaElementTypes.add(new ArrayList<>(Arrays.asList(SchemaElementType.DIMENSION)));
- for (List schemaElementTypes : trySchemaElementTypes) {
+ for (List schemaTypes : trySchemaElementTypes) {
newElementMatches.clear();
if (!CollectionUtils.isEmpty(elementMatches)) {
newElementMatches.addAll(elementMatches);
}
- ContextHelper.mergeContextSchemaElementMatch(newElementMatches, elementMatches, schemaElementTypes,
+ ContextHelper.mergeContextSchemaElementMatch(newElementMatches, elementMatches, schemaTypes,
contextSemanticParse);
- Map> queryMatches = resolveQuery(newElementMatches,
- queryCtx);
- if (queryMatches.size() > 0) {
- return queryMatches;
+ List queries = resolveQuery(newElementMatches, queryContext);
+ if (queries.size() > 0) {
+ return queries;
}
}
}
- return new HashMap<>();
+ return new ArrayList<>();
}
- private Map> resolveQuery(List elementMatches,
- QueryContextReq queryCtx) {
- Map> matchMap = new HashMap<>();
-
+ private List resolveQuery(List candidateElementMatches,
+ QueryContextReq queryContext) {
+ List matchedQueries = new ArrayList<>();
for (RuleSemanticQuery semanticQuery : RuleSemanticQueryManager.getSemanticQueries()) {
- List matches = semanticQuery.match(elementMatches, queryCtx);
+ List matches = semanticQuery.match(candidateElementMatches, queryContext);
if (matches.size() > 0) {
log.info("resolve match [{}:{}] ", semanticQuery.getQueryMode(), matches.size());
- matchMap.put(RuleSemanticQueryManager.create(semanticQuery.getQueryMode()), matches);
+ RuleSemanticQuery query = RuleSemanticQueryManager.create(semanticQuery.getQueryMode());
+ query.getParseInfo().getElementMatches().addAll(matches);
+ matchedQueries.add(query);
}
}
- return matchMap;
+ return matchedQueries;
}
-
- public void fillParseInfo(SemanticQuery query, Long domainId, String domainName,
- List elementMatches) {
- SemanticParseInfo parseInfo = query.getParseInfo();
- parseInfo.setDomainId(domainId);
- parseInfo.setDomainName(domainName);
- parseInfo.setQueryMode(query.getQueryMode());
- parseInfo.getElementMatches().addAll(elementMatches);
-
- DefaultSemanticInternalUtils defaultSemanticUtils = ContextUtils.getBean(DefaultSemanticInternalUtils.class);
- SemanticLayer semanticLayer = ComponentFactory.getSemanticLayer();
-
- DomainSchemaResp domainSchemaDesc = semanticLayer.getDomainSchemaInfo(parseInfo.getDomainId());
- ChatConfigRichInfo chaConfigRichDesc = defaultSemanticUtils.getChatConfigRichInfo(parseInfo.getDomainId());
- Map dimensionDescMap = domainSchemaDesc.getDimensions().stream()
- .collect(Collectors.toMap(DimSchemaResp::getId, Function.identity()));
- Map metricDescMap = domainSchemaDesc.getMetrics().stream()
- .collect(Collectors.toMap(MetricSchemaResp::getId, Function.identity()));
- Map> dim2Values = new HashMap<>();
-
- for (SchemaElementMatch schemaElementMatch : elementMatches) {
- Long elementID = Long.valueOf(schemaElementMatch.getElementID());
- switch (schemaElementMatch.getElementType()) {
- case ID:
- case VALUE:
- if (dimensionDescMap.containsKey(elementID)) {
- if (dim2Values.containsKey(elementID)) {
- dim2Values.get(elementID).add(schemaElementMatch);
- } else {
- dim2Values.put(elementID, new ArrayList<>(Arrays.asList(schemaElementMatch)));
- }
- }
- break;
- case DIMENSION:
- DimSchemaResp dimensionDesc = dimensionDescMap.get(elementID);
- if (dimensionDesc != null) {
- SchemaItem dimensionParseInfo = new SchemaItem();
- dimensionParseInfo.setBizName(dimensionDesc.getBizName());
- dimensionParseInfo.setName(dimensionDesc.getName());
- dimensionParseInfo.setId(dimensionDesc.getId());
- parseInfo.getDimensions().add(dimensionParseInfo);
- }
- break;
- case METRIC:
- MetricSchemaResp metricDesc = metricDescMap.get(elementID);
- if (metricDesc != null) {
- SchemaItem metricItem = new SchemaItem();
- metricItem.setBizName(metricDesc.getBizName());
- metricItem.setName(metricDesc.getName());
- metricItem.setId(metricDesc.getId());
- metricItem.setCreatedAt(null);
- metricItem.setUpdatedAt(null);
- parseInfo.getMetrics().add(metricItem);
- }
- break;
- default:
- }
- }
-
- if (!dim2Values.isEmpty()) {
- for (Map.Entry> entry : dim2Values.entrySet()) {
- DimSchemaResp dimensionDesc = dimensionDescMap.get(entry.getKey());
- if (entry.getValue().size() == 1) {
- SchemaElementMatch schemaElementMatch = entry.getValue().get(0);
- Filter dimensionFilter = new Filter();
- dimensionFilter.setValue(schemaElementMatch.getWord());
- dimensionFilter.setBizName(dimensionDesc.getBizName());
- dimensionFilter.setName(dimensionDesc.getName());
- dimensionFilter.setOperator(FilterOperatorEnum.EQUALS);
- dimensionFilter.setElementID(Long.valueOf(schemaElementMatch.getElementID()));
- parseInfo.getDimensionFilters().add(dimensionFilter);
- ContextHelper.setEntityId(entry.getKey(), schemaElementMatch.getWord(), chaConfigRichDesc,
- parseInfo);
- } else {
- Filter dimensionFilter = new Filter();
- List vals = new ArrayList<>();
- entry.getValue().stream().forEach(i -> vals.add(i.getWord()));
- dimensionFilter.setValue(vals);
- dimensionFilter.setBizName(dimensionDesc.getBizName());
- dimensionFilter.setName(dimensionDesc.getName());
- dimensionFilter.setOperator(FilterOperatorEnum.IN);
- dimensionFilter.setElementID(entry.getKey());
- parseInfo.getDimensionFilters().add(dimensionFilter);
- }
- }
- }
- }
-}
\ No newline at end of file
+}
diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/LLMSemanticParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/LLMSemanticParser.java
index ebb633a1d..b3aa00471 100644
--- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/LLMSemanticParser.java
+++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/LLMSemanticParser.java
@@ -20,6 +20,7 @@ import com.tencent.supersonic.chat.domain.utils.SemanticSatisfactionChecker;
import com.tencent.supersonic.common.nlp.ItemDO;
import com.tencent.supersonic.common.util.context.ContextUtils;
import com.tencent.supersonic.common.util.json.JsonUtil;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -28,6 +29,7 @@ import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
+import org.apache.calcite.sql.parser.SqlParseException;
import org.apache.commons.lang3.StringUtils;
import org.springframework.core.io.support.SpringFactoriesLoader;
import org.springframework.http.HttpEntity;
@@ -40,90 +42,100 @@ import org.springframework.web.client.RestTemplate;
@Slf4j
public class LLMSemanticParser implements SemanticParser {
+ private DslToSemanticInfo dslToSemanticInfo = new DslToSemanticInfo();
+
@Override
public void parse(QueryContextReq queryContext, ChatContext chatCtx) {
+ String queryText = queryContext.getQueryText();
+
if (SemanticSatisfactionChecker.check(queryContext)) {
- log.info("There is no need parse by llm , queryText:{}", queryContext.getQueryText());
+ log.info("There is no need parse by llm , queryText:{}", queryText);
return;
}
- Integer domainId = getDomainId(queryContext, chatCtx);
- LLMResp llmResp = requestLLM(queryContext, domainId);
- if (Objects.isNull(llmResp)) {
+ try {
+ Integer domainId = getDomainId(queryContext, chatCtx);
+ LLMResp llmResp = requestLLM(queryContext, domainId);
+ if (Objects.isNull(llmResp)) {
+ return;
+ }
+ LLMSemanticQuery semanticQuery = new LLMSemanticQuery();
+ SemanticParseInfo parseInfo = semanticQuery.getParseInfo();
+ String sql = convertToSql(llmResp, parseInfo, domainId);
+
+ parseInfo.setInfo(sql);
+ parseInfo.setDomainId(Long.valueOf(domainId));
+ parseInfo.setBonus(queryText.length() * 1.0);
+ parseInfo.setQueryMode(LLMSemanticQuery.QUERY_MODE);
+ queryContext.getCandidateQueries().add(semanticQuery);
return;
+ } catch (Exception e) {
+ log.error("llm parse error , skip the parser. error:", e);
}
- LLMSemanticQuery semanticQuery = new LLMSemanticQuery();
- SemanticParseInfo parseInfo = semanticQuery.getParseInfo();
- String sql = convertToSql(llmResp, parseInfo);
- parseInfo.setDomainId(Long.valueOf(domainId));
- parseInfo.setBonus(queryContext.getQueryText().length() * 1.0);
- parseInfo.setQueryMode(LLMSemanticQuery.QUERY_MODE);
- parseInfo.setInfo(sql);
- queryContext.getCandidateQueries().add(semanticQuery);
- return;
}
- protected String convertToSql(LLMResp llmResp, SemanticParseInfo parseInfo) {
- return DslToSemanticInfo.convert(parseInfo, llmResp);
+ protected String convertToSql(LLMResp llmResp, SemanticParseInfo parseInfo, Integer domainId)
+ throws SqlParseException {
+ return dslToSemanticInfo.convert(parseInfo, llmResp, domainId);
}
protected LLMResp requestLLM(QueryContextReq queryContext, Integer domainId) {
- try {
- final LLMConfig llmConfig = ContextUtils.getBean(LLMConfig.class);
+ final LLMConfig llmConfig = ContextUtils.getBean(LLMConfig.class);
- DomainInfos domainInfos = ContextUtils.getBean(WordNatureService.class).getCache().getUnchecked("");
-
- Map domainIdToName = domainInfos.getDomains().stream()
- .collect(Collectors.toMap(ItemDO::getDomain, a -> a.getName(), (k1, k2) -> k1));
-
- Map itemIdToName = domainInfos.getDimensions().stream()
- .filter(entry -> domainId.equals(entry.getDomain()))
- .collect(Collectors.toMap(ItemDO::getItemId, ItemDO::getName, (value1, value2) -> value2));
-
- String domainName = domainIdToName.get(domainId);
- LLMReq llmReq = new LLMReq();
- llmReq.setQueryText(queryContext.getQueryText());
-
- List matchedElements = queryContext.getMapInfo().getMatchedElements(domainId);
-
- List fieldNameList = matchedElements.stream()
- .filter(schemaElementMatch ->
- SchemaElementType.METRIC.equals(schemaElementMatch.getElementType()) ||
- SchemaElementType.DIMENSION.equals(schemaElementMatch.getElementType()) ||
- SchemaElementType.VALUE.equals(schemaElementMatch.getElementType()))
- .map(schemaElementMatch -> {
- if (!SchemaElementType.VALUE.equals(schemaElementMatch.getElementType())) {
- return schemaElementMatch.getWord();
- }
- return itemIdToName.get(schemaElementMatch.getElementID());
- })
- .filter(name -> StringUtils.isNotEmpty(name) && !name.contains("%"))
- .collect(Collectors.toList());
-
- LLMSchema llmSchema = new LLMSchema();
- llmSchema.setDomainName(domainName);
- llmSchema.setFieldNameList(fieldNameList);
- llmReq.setSchema(llmSchema);
-
- log.info("domainId:{},llmReq:{}", domainId, llmReq);
- String questUrl = llmConfig.getUrl() + llmConfig.getQueryToSqlPath();
-
- RestTemplate restTemplate = ContextUtils.getBean(RestTemplate.class);
-
- HttpHeaders headers = new HttpHeaders();
- headers.setContentType(MediaType.APPLICATION_JSON);
- HttpEntity entity = new HttpEntity<>(JsonUtil.toString(llmReq), headers);
-
- log.info("requestLLM request:{},entity:{}", questUrl, entity);
- ResponseEntity responseEntity = restTemplate.exchange(questUrl, HttpMethod.POST, entity,
- LLMResp.class);
-
- log.info("requestLLM result:{}", responseEntity);
- return responseEntity.getBody();
- } catch (Exception e) {
- log.error("requestLLM error", e);
+ if (StringUtils.isEmpty(llmConfig.getUrl())) {
+ log.warn("llmConfig url is null, skip llm parser");
+ return null;
}
- return null;
+
+ DomainInfos domainInfos = ContextUtils.getBean(WordNatureService.class).getCache().getUnchecked("");
+
+ Map domainIdToName = domainInfos.getDomains().stream()
+ .collect(Collectors.toMap(ItemDO::getDomain, a -> a.getName(), (k1, k2) -> k1));
+
+ Map itemIdToName = domainInfos.getDimensions().stream()
+ .filter(entry -> domainId.equals(entry.getDomain()))
+ .collect(Collectors.toMap(ItemDO::getItemId, ItemDO::getName, (value1, value2) -> value2));
+
+ String domainName = domainIdToName.get(domainId);
+ LLMReq llmReq = new LLMReq();
+ llmReq.setQueryText(queryContext.getQueryText());
+
+ List matchedElements = queryContext.getMapInfo().getMatchedElements(domainId);
+
+ Set fieldNameList = matchedElements.stream()
+ .filter(schemaElementMatch ->
+ SchemaElementType.METRIC.equals(schemaElementMatch.getElementType()) ||
+ SchemaElementType.DIMENSION.equals(schemaElementMatch.getElementType()) ||
+ SchemaElementType.VALUE.equals(schemaElementMatch.getElementType()))
+ .map(schemaElementMatch -> {
+ if (!SchemaElementType.VALUE.equals(schemaElementMatch.getElementType())) {
+ return schemaElementMatch.getWord();
+ }
+ return itemIdToName.get(schemaElementMatch.getElementID());
+ })
+ .filter(name -> StringUtils.isNotEmpty(name) && !name.contains("%"))
+ .collect(Collectors.toSet());
+
+ LLMSchema llmSchema = new LLMSchema();
+ llmSchema.setDomainName(domainName);
+ llmSchema.setFieldNameList(new ArrayList<>(fieldNameList));
+ llmReq.setSchema(llmSchema);
+
+ log.info("requestLLM request, domainId:{},llmReq:{}", domainId, llmReq);
+ String questUrl = llmConfig.getUrl() + llmConfig.getQueryToSqlPath();
+
+ RestTemplate restTemplate = ContextUtils.getBean(RestTemplate.class);
+
+ HttpHeaders headers = new HttpHeaders();
+ headers.setContentType(MediaType.APPLICATION_JSON);
+ HttpEntity entity = new HttpEntity<>(JsonUtil.toString(llmReq), headers);
+
+ ResponseEntity responseEntity = restTemplate.exchange(questUrl, HttpMethod.POST, entity,
+ LLMResp.class);
+
+ log.info("requestLLM response, questUrl:{} \n entity:{} \n body:{}", questUrl, entity,
+ responseEntity.getBody());
+ return responseEntity.getBody();
}
protected Integer getDomainId(QueryContextReq queryContext, ChatContext chatCtx) {
diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/TimeSemanticParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/TimeSemanticParser.java
index e63abf831..4190c7012 100644
--- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/TimeSemanticParser.java
+++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/parser/TimeSemanticParser.java
@@ -66,19 +66,18 @@ public class TimeSemanticParser implements SemanticParser {
DateConf info = new DateConf();
String zhPeriod = m.group("zhPeriod");
int days;
- info.setPeriod(Constants.DAY);
switch (zhPeriod) {
case "周":
days = 7;
- //info.setPeriod(Constants.WEEK);
+ info.setPeriod(Constants.WEEK);
break;
case "月":
days = 30;
- //info.setPeriod(Constants.MONTH);
+ info.setPeriod(Constants.MONTH);
break;
case "年":
days = 365;
- //info.setPeriod(Constants.YEAR);
+ info.setPeriod(Constants.YEAR);
break;
default:
days = 1;
@@ -92,7 +91,7 @@ public class TimeSemanticParser implements SemanticParser {
}
info.setText(text);
info.setStartDate(LocalDate.now().minusDays(days).toString());
- info.setUnit(days);
+ info.setUnit(num);
//queryContext.getParseInfo().setDateInfo(info);
for (SemanticQuery query : queryContext.getCandidateQueries()) {
if (query instanceof MetricSemanticQuery) {
diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/query/EntityListFilter.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/query/EntityListFilter.java
index 1bf46041f..aadc64412 100644
--- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/query/EntityListFilter.java
+++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/query/EntityListFilter.java
@@ -1,31 +1,33 @@
package com.tencent.supersonic.chat.application.query;
-import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.VALUE;
-import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.AT_LEAST;
-import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.REQUIRED;
+import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.*;
+import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.*;
+import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.*;
import static com.tencent.supersonic.common.constant.Constants.DAY;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
-import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigRichInfo;
+import com.tencent.supersonic.chat.domain.pojo.config.ChatConfigRichResp;
+import com.tencent.supersonic.chat.domain.pojo.config.ChatDefaultRichConfig;
import com.tencent.supersonic.chat.domain.pojo.config.EntityRichInfo;
+import com.tencent.supersonic.chat.domain.service.ConfigService;
import com.tencent.supersonic.chat.domain.utils.ContextHelper;
-import com.tencent.supersonic.chat.domain.utils.DefaultSemanticInternalUtils;
import com.tencent.supersonic.common.constant.Constants;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.SchemaItem;
import com.tencent.supersonic.common.util.context.ContextUtils;
import com.tencent.supersonic.semantic.api.core.response.DimSchemaResp;
-import com.tencent.supersonic.semantic.api.core.response.MetricSchemaResp;
+
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.Objects;
import java.util.Set;
+
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Component;
-import org.springframework.util.CollectionUtils;
+
@Slf4j
@Component
@@ -34,11 +36,14 @@ public class EntityListFilter extends EntitySemanticQuery {
public static String QUERY_MODE = "ENTITY_LIST_FILTER";
private static Long entityListLimit = 200L;
+
public EntityListFilter() {
super();
- queryMatcher.addOption(VALUE, REQUIRED, AT_LEAST, 1);
+ queryMatcher.addOption(VALUE, REQUIRED, AT_LEAST, 1)
+ .addOption(ENTITY, REQUIRED, AT_LEAST, 1);
}
+
@Override
public String getQueryMode() {
return QUERY_MODE;
@@ -67,33 +72,32 @@ public class EntityListFilter extends EntitySemanticQuery {
private void addEntityDetailAndOrderByMetric(SemanticParseInfo semanticParseInfo) {
if (semanticParseInfo.getDomainId() > 0L) {
- DefaultSemanticInternalUtils defaultSemanticUtils = ContextUtils.getBean(
- DefaultSemanticInternalUtils.class);
- ChatConfigRichInfo chaConfigRichDesc = defaultSemanticUtils.getChatConfigRichInfo(
+ ConfigService configService = ContextUtils.getBean(ConfigService.class);
+ ChatConfigRichResp chaConfigRichDesc = configService.getConfigRichInfo(
semanticParseInfo.getDomainId());
- if (chaConfigRichDesc != null) {
- //SemanticParseInfo semanticParseInfo = queryContext.getParseInfo();
+ if (chaConfigRichDesc != null && chaConfigRichDesc.getChatDetailRichConfig() != null
+ && chaConfigRichDesc.getChatDetailRichConfig().getEntity() != null) {
+// SemanticParseInfo semanticParseInfo = queryContext.getParseInfo();
+// EntityRichInfo entity = chaConfigRichDesc.getChatDetailRichConfig().getEntity();
Set dimensions = new LinkedHashSet();
- Set primaryDimensions = this.addPrimaryDimension(chaConfigRichDesc.getEntity(), dimensions);
+// Set primaryDimensions = this.addPrimaryDimension(entity, dimensions);
Set metrics = new LinkedHashSet();
- if (chaConfigRichDesc.getEntity() != null
- && chaConfigRichDesc.getEntity().getEntityInternalDetailDesc() != null) {
- chaConfigRichDesc.getEntity().getEntityInternalDetailDesc().getMetricList().stream()
- .forEach((m) -> metrics.add(this.getMetric(m)));
- chaConfigRichDesc.getEntity().getEntityInternalDetailDesc().getDimensionList().stream()
- .filter((m) -> !primaryDimensions.contains(m.getBizName()))
- .forEach((m) -> dimensions.add(this.getDimension(m)));
+ Set orders = new LinkedHashSet();
+ ChatDefaultRichConfig chatDefaultConfig = chaConfigRichDesc.getChatDetailRichConfig().getChatDefaultConfig();
+ if (chatDefaultConfig != null) {
+ chatDefaultConfig.getMetrics().stream()
+ .forEach(metric -> {
+ metrics.add(metric);
+ orders.add(new Order(metric.getBizName(), Constants.DESC_UPPER));
+ });
+ chatDefaultConfig.getDimensions().stream()
+// .filter((m) -> !primaryDimensions.contains(m.getBizName()))
+ .forEach(dimension -> dimensions.add(dimension));
+
}
semanticParseInfo.setDimensions(dimensions);
semanticParseInfo.setMetrics(metrics);
- Set orders = new LinkedHashSet();
- if (chaConfigRichDesc.getEntity() != null
- && chaConfigRichDesc.getEntity().getEntityInternalDetailDesc() != null) {
- chaConfigRichDesc.getEntity().getEntityInternalDetailDesc().getMetricList().stream()
- .forEach((metric) -> orders.add(new Order(metric.getBizName(), Constants.DESC_UPPER)));
- }
-
semanticParseInfo.setOrders(orders);
}
}
@@ -102,35 +106,18 @@ public class EntityListFilter extends EntitySemanticQuery {
private Set addPrimaryDimension(EntityRichInfo entity, Set dimensions) {
Set primaryDimensions = new HashSet();
- if (!Objects.isNull(entity) && !CollectionUtils.isEmpty(entity.getEntityIds())) {
- entity.getEntityIds().stream().forEach((dimSchemaDesc) -> {
- SchemaItem dimension = new SchemaItem();
- BeanUtils.copyProperties(dimSchemaDesc, dimension);
- dimensions.add(dimension);
- primaryDimensions.add(dimSchemaDesc.getBizName());
- });
+ DimSchemaResp dimItem = entity.getDimItem();
+ if (Objects.nonNull(entity) && Objects.nonNull(dimItem)) {
+ SchemaItem dimension = new SchemaItem();
+ BeanUtils.copyProperties(dimItem, dimension);
+ dimensions.add(dimension);
+ primaryDimensions.add(dimItem.getBizName());
return primaryDimensions;
} else {
return primaryDimensions;
}
}
- private SchemaItem getMetric(MetricSchemaResp metricSchemaDesc) {
- SchemaItem queryMeta = new SchemaItem();
- queryMeta.setId(metricSchemaDesc.getId());
- queryMeta.setBizName(metricSchemaDesc.getBizName());
- queryMeta.setName(metricSchemaDesc.getName());
- return queryMeta;
- }
-
- private SchemaItem getDimension(DimSchemaResp dimSchemaDesc) {
- SchemaItem queryMeta = new SchemaItem();
- queryMeta.setId(dimSchemaDesc.getId());
- queryMeta.setBizName(dimSchemaDesc.getBizName());
- queryMeta.setName(dimSchemaDesc.getName());
- return queryMeta;
- }
-
private void dealNativeQuery(SemanticParseInfo semanticParseInfo, boolean isNativeQuery) {
if (Objects.nonNull(semanticParseInfo)) {
semanticParseInfo.setNativeQuery(isNativeQuery);
diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/query/EntityListTopN.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/query/EntityListTopN.java
index 4f7f9769c..aba98b7a5 100644
--- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/query/EntityListTopN.java
+++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/query/EntityListTopN.java
@@ -1,6 +1,6 @@
package com.tencent.supersonic.chat.application.query;
-import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.METRIC;
+import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.*;
import static com.tencent.supersonic.chat.application.query.QueryMatchOption.RequireNumberType.AT_LEAST;
import static com.tencent.supersonic.chat.domain.pojo.chat.SchemaElementOption.REQUIRED;
diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/application/query/MetricCompare.java b/chat/core/src/main/java/com/tencent/supersonic/chat/application/query/MetricCompare.java
index 3a215dd18..927b4e2c9 100644
--- a/chat/core/src/main/java/com/tencent/supersonic/chat/application/query/MetricCompare.java
+++ b/chat/core/src/main/java/com/tencent/supersonic/chat/application/query/MetricCompare.java
@@ -16,6 +16,7 @@ import com.tencent.supersonic.chat.domain.utils.ContextHelper;
import com.tencent.supersonic.common.pojo.SchemaItem;
import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
@@ -95,7 +96,7 @@ public class MetricCompare extends MetricSemanticQuery {
if (toAdd.isPresent()) {
if (FilterOperatorEnum.EQUALS.equals(toAdd.get().getOperator()) || FilterOperatorEnum.IN.equals(
toAdd.get().getOperator())) {
- List