From b753eda9b93cb58e8210b257a602954e2ec981f9 Mon Sep 17 00:00:00 2001 From: lexluo09 <39718951+lexluo09@users.noreply.github.com> Date: Thu, 12 Oct 2023 11:38:48 +0800 Subject: [PATCH] (improvement)(chat) set native value in parser and execute , optimized the executeDirectQuery code (#196) --- .../execute/EntityInfoExecuteResponder.java | 10 +- .../parse/EntityInfoParseResponder.java | 21 +- .../chat/service/SemanticService.java | 7 + .../chat/service/impl/QueryServiceImpl.java | 247 +++++++++--------- .../query/utils/QueryStructUtils.java | 16 +- 5 files changed, 165 insertions(+), 136 deletions(-) diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/responder/execute/EntityInfoExecuteResponder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/responder/execute/EntityInfoExecuteResponder.java index c827071e1..6ec83bec6 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/responder/execute/EntityInfoExecuteResponder.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/responder/execute/EntityInfoExecuteResponder.java @@ -12,6 +12,7 @@ import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; public class EntityInfoExecuteResponder implements ExecuteResponder { @@ -25,12 +26,9 @@ public class EntityInfoExecuteResponder implements ExecuteResponder { EntityInfo entityInfo = semanticService.getEntityInfo(semanticParseInfo, user); queryResult.setEntityInfo(entityInfo); - if (Objects.isNull(entityInfo) || Objects.isNull(entityInfo.getModelInfo()) - || Objects.isNull(entityInfo.getModelInfo().getPrimaryEntityName())) { - return; - } - String primaryEntityBizName = entityInfo.getModelInfo().getPrimaryEntityBizName(); - if (CollectionUtils.isEmpty(queryResult.getQueryColumns())) { + String primaryEntityBizName = semanticService.getPrimaryEntityBizName(entityInfo); + if (StringUtils.isEmpty(primaryEntityBizName) + || CollectionUtils.isEmpty(queryResult.getQueryColumns())) { return; } boolean existPrimaryEntityName = queryResult.getQueryColumns().stream() diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/EntityInfoParseResponder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/EntityInfoParseResponder.java index 9074c9dbc..ad073c30e 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/EntityInfoParseResponder.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/EntityInfoParseResponder.java @@ -8,8 +8,9 @@ import com.tencent.supersonic.chat.api.pojo.response.ParseResp; import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.chat.service.SemanticService; import com.tencent.supersonic.common.util.ContextUtils; -import org.springframework.util.CollectionUtils; import java.util.List; +import org.apache.commons.lang3.StringUtils; +import org.springframework.util.CollectionUtils; public class EntityInfoParseResponder implements ParseResponder { @@ -21,13 +22,21 @@ public class EntityInfoParseResponder implements ParseResponder { } QueryReq queryReq = queryContext.getRequest(); selectedParses.forEach(parseInfo -> { - String queryMode = parseInfo.getQueryMode(); - if (QueryManager.isEntityQuery(queryMode)) { - EntityInfo entityInfo = ContextUtils.getBean(SemanticService.class) - .getEntityInfo(parseInfo, queryReq.getUser()); + //1. set entity info + SemanticService semanticService = ContextUtils.getBean(SemanticService.class); + EntityInfo entityInfo = semanticService.getEntityInfo(parseInfo, queryReq.getUser()); + + if (QueryManager.isEntityQuery(parseInfo.getQueryMode())) { parseInfo.setEntityInfo(entityInfo); } + //2. set native value + String primaryEntityBizName = semanticService.getPrimaryEntityBizName(entityInfo); + if (StringUtils.isNotEmpty(primaryEntityBizName)) { + //if exist primaryEntityBizName in parseInfo's dimensions, set nativeQuery to true + boolean existPrimaryEntityBizName = parseInfo.getDimensions().stream() + .anyMatch(schemaElement -> primaryEntityBizName.equalsIgnoreCase(schemaElement.getBizName())); + parseInfo.setNativeQuery(existPrimaryEntityBizName); + } }); } - } \ No newline at end of file diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/SemanticService.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/SemanticService.java index 09bca998f..5a0a4d286 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/SemanticService.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/SemanticService.java @@ -189,6 +189,13 @@ public class SemanticService { return entityInfo; } + public String getPrimaryEntityBizName(EntityInfo entityInfo) { + if (Objects.isNull(entityInfo) || Objects.isNull(entityInfo.getModelInfo())) { + return null; + } + return entityInfo.getModelInfo().getPrimaryEntityBizName(); + } + public void setMainModel(EntityInfo modelInfo, Long model, String entity, User user) { if (StringUtils.isEmpty(entity)) { return; diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java index 2a73b1289..e8bc8bf43 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java @@ -1,66 +1,63 @@ package com.tencent.supersonic.chat.service.impl; -import com.google.common.collect.Lists; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.component.SchemaMapper; import com.tencent.supersonic.chat.api.component.SemanticInterpreter; -import com.tencent.supersonic.chat.api.component.SemanticQuery; import com.tencent.supersonic.chat.api.component.SemanticParser; +import com.tencent.supersonic.chat.api.component.SemanticQuery; import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.QueryContext; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.chat.api.pojo.request.QueryDataReq; -import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.request.DimensionValueReq; import com.tencent.supersonic.chat.api.pojo.request.ExecuteQueryReq; +import com.tencent.supersonic.chat.api.pojo.request.QueryDataReq; +import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.request.SolvedQueryReq; -import com.tencent.supersonic.chat.api.pojo.response.QueryState; import com.tencent.supersonic.chat.api.pojo.response.ParseResp; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; +import com.tencent.supersonic.chat.api.pojo.response.QueryState; import com.tencent.supersonic.chat.parser.llm.dsl.DSLParseResult; import com.tencent.supersonic.chat.persistence.dataobject.ChatParseDO; import com.tencent.supersonic.chat.persistence.dataobject.ChatQueryDO; import com.tencent.supersonic.chat.persistence.dataobject.CostType; import com.tencent.supersonic.chat.persistence.dataobject.StatisticsDO; -import com.tencent.supersonic.chat.query.QuerySelector; import com.tencent.supersonic.chat.query.QueryManager; +import com.tencent.supersonic.chat.query.QuerySelector; import com.tencent.supersonic.chat.query.llm.dsl.DslQuery; import com.tencent.supersonic.chat.query.llm.dsl.LLMResp; import com.tencent.supersonic.chat.responder.execute.ExecuteResponder; import com.tencent.supersonic.chat.responder.parse.ParseResponder; import com.tencent.supersonic.chat.service.ChatService; import com.tencent.supersonic.chat.service.QueryService; -import com.tencent.supersonic.common.pojo.QueryColumn; -import com.tencent.supersonic.knowledge.dictionary.MapResult; -import com.tencent.supersonic.knowledge.service.SearchService; import com.tencent.supersonic.chat.service.StatisticsService; import com.tencent.supersonic.chat.utils.ComponentFactory; - -import java.util.List; -import java.util.ArrayList; -import java.util.Map; -import java.util.HashMap; -import java.util.Objects; -import java.util.Set; -import java.util.HashSet; -import java.util.Comparator; - import com.tencent.supersonic.chat.utils.SolvedQueryManager; - -import com.tencent.supersonic.semantic.api.model.response.ExplainResp; -import com.tencent.supersonic.common.util.jsqlparser.FilterExpression; -import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; - -import java.util.stream.Collectors; - import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.DateConf; +import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.util.JsonUtil; +import com.tencent.supersonic.common.util.jsqlparser.FilterExpression; +import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserUpdateHelper; +import com.tencent.supersonic.knowledge.dictionary.MapResult; +import com.tencent.supersonic.knowledge.service.SearchService; +import com.tencent.supersonic.semantic.api.model.enums.TimeDimensionEnum; +import com.tencent.supersonic.semantic.api.model.response.ExplainResp; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; +import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum; import com.tencent.supersonic.semantic.api.query.request.QueryStructReq; +import com.tencent.supersonic.semantic.query.utils.QueryStructUtils; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.calcite.sql.parser.SqlParseException; import org.apache.commons.collections.CollectionUtils; @@ -83,8 +80,6 @@ public class QueryServiceImpl implements QueryService { private StatisticsService statisticsService; @Autowired private SolvedQueryManager solvedQueryManager; - @Autowired - private SearchService searchService; @Value("${time.threshold: 100}") private Integer timeThreshold; @@ -205,8 +200,8 @@ public class QueryServiceImpl implements QueryService { } public void saveInfo(List timeCostDOList, - String queryText, Long queryId, - String userName, Long chatId) { + String queryText, Long queryId, + String userName, Long chatId) { List list = timeCostDOList.stream() .filter(o -> o.getCost() > timeThreshold).collect(Collectors.toList()); list.forEach(o -> { @@ -273,110 +268,35 @@ public class QueryServiceImpl implements QueryService { public QueryResult executeDirectQuery(QueryDataReq queryData, User user) throws SqlParseException { ChatParseDO chatParseDO = chatService.getParseInfo(queryData.getQueryId(), queryData.getUser().getName(), queryData.getParseId()); - SemanticParseInfo parseInfo = JsonUtil.toObject(chatParseDO.getParseInfo(), SemanticParseInfo.class); + SemanticParseInfo parseInfo = getSemanticParseInfo(queryData, chatParseDO); + SemanticQuery semanticQuery = QueryManager.createQuery(parseInfo.getQueryMode()); - if (!parseInfo.getQueryMode().equals(DslQuery.QUERY_MODE)) { - if (CollectionUtils.isNotEmpty(queryData.getDimensions())) { - parseInfo.setDimensions(queryData.getDimensions()); - } - if (CollectionUtils.isNotEmpty(queryData.getMetrics())) { - parseInfo.setMetrics(queryData.getMetrics()); - } - if (CollectionUtils.isNotEmpty(queryData.getDimensionFilters())) { - parseInfo.setDimensionFilters(queryData.getDimensionFilters()); - } - if (Objects.nonNull(queryData.getDateInfo())) { - parseInfo.setDateInfo(queryData.getDateInfo()); - } - } - if (parseInfo.getQueryMode().equals(DslQuery.QUERY_MODE) - && (CollectionUtils.isNotEmpty(queryData.getDimensionFilters()) - || CollectionUtils.isNotEmpty(queryData.getMetricFilters()))) { + + if (DslQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) { Map> filedNameToValueMap = new HashMap<>(); String json = JsonUtil.toString(parseInfo.getProperties().get(Constants.CONTEXT)); DSLParseResult dslParseResult = JsonUtil.toObject(json, DSLParseResult.class); LLMResp llmResp = dslParseResult.getLlmResp(); String correctorSql = llmResp.getCorrectorSql(); log.info("correctorSql before replacing:{}", correctorSql); + List filterExpressionList = SqlParserSelectHelper.getFilterExpression(correctorSql); - for (QueryFilter dslQueryFilter : queryData.getDimensionFilters()) { - Map map = new HashMap<>(); - for (FilterExpression filterExpression : filterExpressionList) { - if (filterExpression.getFieldName() != null - && filterExpression.getFieldName().equals(dslQueryFilter.getName()) - && dslQueryFilter.getOperator().getValue().equals(filterExpression.getOperator())) { - map.put(filterExpression.getFieldValue().toString(), dslQueryFilter.getValue().toString()); - parseInfo.getDimensionFilters().stream().forEach(o -> { - if (o.getName().equals(dslQueryFilter.getName())) { - o.setValue(dslQueryFilter.getValue()); - } - }); - break; - } - } - filedNameToValueMap.put(dslQueryFilter.getName(), map); - } - for (QueryFilter dslQueryFilter : queryData.getMetricFilters()) { - Map map = new HashMap<>(); - for (FilterExpression filterExpression : filterExpressionList) { - if (filterExpression.getFieldName() != null - && filterExpression.getFieldName().equals(dslQueryFilter.getName()) - && dslQueryFilter.getOperator().getValue().equals(filterExpression.getOperator())) { - map.put(filterExpression.getFieldValue().toString(), dslQueryFilter.getValue().toString()); - parseInfo.getMetricFilters().stream().forEach(o -> { - if (o.getName().equals(dslQueryFilter.getName())) { - o.setValue(dslQueryFilter.getValue()); - } - }); - break; - } - } - filedNameToValueMap.put(dslQueryFilter.getName(), map); - } - String dateField = "sys_imp_date"; - if (Objects.nonNull(queryData.getDateInfo())) { - Map map = new HashMap<>(); - List dateFields = Lists.newArrayList("dayno", "sys_imp_date", "sys_imp_week", "sys_imp_month"); - if (queryData.getDateInfo().getStartDate().equals(queryData.getDateInfo().getEndDate())) { - for (FilterExpression filterExpression : filterExpressionList) { - if (filterExpression.getFieldName() != null - && dateFields.contains(filterExpression.getFieldName())) { - dateField = filterExpression.getFieldName(); - map.put(filterExpression.getFieldValue().toString(), - queryData.getDateInfo().getStartDate()); - break; - } - } - } else { - for (FilterExpression filterExpression : filterExpressionList) { - if (dateFields.contains(filterExpression.getFieldName())) { - dateField = filterExpression.getFieldName(); - if (filterExpression.getOperator().equals(">=") - || filterExpression.getOperator().equals(">")) { - map.put(filterExpression.getFieldValue().toString(), - queryData.getDateInfo().getStartDate()); - } - if (filterExpression.getOperator().equals("<=") - || filterExpression.getOperator().equals("<")) { - map.put(filterExpression.getFieldValue().toString(), - queryData.getDateInfo().getEndDate()); - } - } - } - } - filedNameToValueMap.put(dateField, map); - parseInfo.setDateInfo(queryData.getDateInfo()); - } + + updateFilters(filedNameToValueMap, filterExpressionList, queryData.getDimensionFilters(), + parseInfo.getDimensionFilters()); + + updateFilters(filedNameToValueMap, filterExpressionList, queryData.getMetricFilters(), + parseInfo.getMetricFilters()); + + updateDateInfo(queryData, parseInfo, filedNameToValueMap, filterExpressionList); + log.info("filedNameToValueMap:{}", filedNameToValueMap); correctorSql = SqlParserUpdateHelper.replaceValue(correctorSql, filedNameToValueMap); log.info("correctorSql after replacing:{}", correctorSql); llmResp.setCorrectorSql(correctorSql); - dslParseResult.setLlmResp(llmResp); - Map properties = new HashMap<>(); - properties.put(Constants.CONTEXT, dslParseResult); - parseInfo.setProperties(properties); + parseInfo.getSqlInfo().setLogicSql(correctorSql); - semanticQuery.setParseInfo(parseInfo); + ExplainResp explain = semanticQuery.explain(user); if (!Objects.isNull(explain)) { parseInfo.getSqlInfo().setQuerySql(explain.getSql()); @@ -388,6 +308,91 @@ public class QueryServiceImpl implements QueryService { return queryResult; } + private void updateDateInfo(QueryDataReq queryData, SemanticParseInfo parseInfo, + Map> filedNameToValueMap, List filterExpressionList) { + if (Objects.isNull(queryData.getDateInfo())) { + return; + } + Map map = new HashMap<>(); + List dateFields = new ArrayList<>(QueryStructUtils.internalTimeCols); + String dateField = TimeDimensionEnum.DAY.getName(); + if (queryData.getDateInfo().getStartDate().equals(queryData.getDateInfo().getEndDate())) { + for (FilterExpression filterExpression : filterExpressionList) { + if (filterExpression.getFieldName() != null + && dateFields.contains(filterExpression.getFieldName())) { + dateField = filterExpression.getFieldName(); + map.put(filterExpression.getFieldValue().toString(), + queryData.getDateInfo().getStartDate()); + break; + } + } + } else { + for (FilterExpression filterExpression : filterExpressionList) { + if (dateFields.contains(filterExpression.getFieldName())) { + dateField = filterExpression.getFieldName(); + if (FilterOperatorEnum.GREATER_THAN_EQUALS.getValue().equals(filterExpression.getOperator()) + || FilterOperatorEnum.GREATER_THAN.getValue().equals(filterExpression.getOperator())) { + map.put(filterExpression.getFieldValue().toString(), + queryData.getDateInfo().getStartDate()); + } + if (FilterOperatorEnum.MINOR_THAN_EQUALS.getValue().equals(filterExpression.getOperator()) + || FilterOperatorEnum.MINOR_THAN.getValue().equals(filterExpression.getOperator())) { + map.put(filterExpression.getFieldValue().toString(), + queryData.getDateInfo().getEndDate()); + } + } + } + } + filedNameToValueMap.put(dateField, map); + parseInfo.setDateInfo(queryData.getDateInfo()); + } + + private void updateFilters(Map> filedNameToValueMap, + List filterExpressionList, Set metricFilters, + Set contextMetricFilters) { + if (CollectionUtils.isEmpty(metricFilters)) { + return; + } + for (QueryFilter dslQueryFilter : metricFilters) { + Map map = new HashMap<>(); + for (FilterExpression filterExpression : filterExpressionList) { + if (filterExpression.getFieldName() != null + && filterExpression.getFieldName().equals(dslQueryFilter.getName()) + && dslQueryFilter.getOperator().getValue().equals(filterExpression.getOperator())) { + map.put(filterExpression.getFieldValue().toString(), dslQueryFilter.getValue().toString()); + contextMetricFilters.stream().forEach(o -> { + if (o.getName().equals(dslQueryFilter.getName())) { + o.setValue(dslQueryFilter.getValue()); + } + }); + break; + } + } + filedNameToValueMap.put(dslQueryFilter.getName(), map); + } + } + + + private SemanticParseInfo getSemanticParseInfo(QueryDataReq queryData, ChatParseDO chatParseDO) { + SemanticParseInfo parseInfo = JsonUtil.toObject(chatParseDO.getParseInfo(), SemanticParseInfo.class); + if (DslQuery.QUERY_MODE.equals(parseInfo.getQueryMode())) { + return parseInfo; + } + if (CollectionUtils.isNotEmpty(queryData.getDimensions())) { + parseInfo.setDimensions(queryData.getDimensions()); + } + if (CollectionUtils.isNotEmpty(queryData.getMetrics())) { + parseInfo.setMetrics(queryData.getMetrics()); + } + if (CollectionUtils.isNotEmpty(queryData.getDimensionFilters())) { + parseInfo.setDimensionFilters(queryData.getDimensionFilters()); + } + if (Objects.nonNull(queryData.getDateInfo())) { + parseInfo.setDateInfo(queryData.getDateInfo()); + } + return parseInfo; + } + @Override public Object queryDimensionValue(DimensionValueReq dimensionValueReq, User user) throws Exception { QueryStructReq queryStructReq = new QueryStructReq(); diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryStructUtils.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryStructUtils.java index e7a5141a0..ad8c67232 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryStructUtils.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/QueryStructUtils.java @@ -44,8 +44,16 @@ import org.springframework.util.CollectionUtils; @Slf4j @Component public class QueryStructUtils { - public static Set internalCols = new HashSet<>( - Arrays.asList("dayno", "plat_sys_var", "sys_imp_date", "sys_imp_week", "sys_imp_month")); + + public static Set internalTimeCols = new HashSet<>( + Arrays.asList("dayno", "sys_imp_date", "sys_imp_week", "sys_imp_month")); + public static Set internalCols; + + static { + internalCols = new HashSet<>(Arrays.asList("plat_sys_var")); + internalCols.addAll(internalTimeCols); + } + private final DateUtils dateUtils; private final SqlFilterUtils sqlFilterUtils; private final Catalog catalog; @@ -160,11 +168,13 @@ public class QueryStructUtils { sqlFilterUtils.getFiltersCol(queryStructCmd.getOriginalFilter()).stream().forEach(col -> resNameEnSet.add(col)); return resNameEnSet; } + public Set getResName(QueryDslReq queryDslReq) { Set resNameSet = SqlParserSelectHelper.getAllFields(queryDslReq.getSql()) - .stream().collect(Collectors.toSet()); + .stream().collect(Collectors.toSet()); return resNameSet; } + public Set getResNameEnExceptInternalCol(QueryStructReq queryStructCmd) { Set resNameEnSet = getResNameEn(queryStructCmd); return resNameEnSet.stream().filter(res -> !internalCols.contains(res)).collect(Collectors.toSet());