From e4280e551601668996fc2ddc1d4c0d8482938f65 Mon Sep 17 00:00:00 2001 From: LXW <1264174498@qq.com> Date: Fri, 13 Oct 2023 11:33:03 +0800 Subject: [PATCH] (improvment)(chat) semantic parse info dimension filters support UNION and AND splicing methods (#203) Co-authored-by: jolunoluo --- .../chat/api/pojo/SemanticParseInfo.java | 2 ++ .../chat/query/rule/RuleSemanticQuery.java | 7 ++++- .../query/rule/metric/MetricEntityQuery.java | 3 ++- .../query/rule/metric/MetricFilterQuery.java | 3 ++- .../common/pojo/enums/FilterType.java | 8 ++++++ .../query/service/QueryServiceImpl.java | 26 ++++++++++++------- 6 files changed, 37 insertions(+), 12 deletions(-) create mode 100644 common/src/main/java/com/tencent/supersonic/common/pojo/enums/FilterType.java diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java index 1a6c52d2f..010e777bb 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java @@ -15,6 +15,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeSet; +import com.tencent.supersonic.common.pojo.enums.FilterType; import lombok.Data; @Data @@ -27,6 +28,7 @@ public class SemanticParseInfo { private Set dimensions = new LinkedHashSet(); private SchemaElement entity; private AggregateTypeEnum aggType = AggregateTypeEnum.NONE; + private FilterType filterType = FilterType.UNION; private Set dimensionFilters = new LinkedHashSet(); private Set metricFilters = new LinkedHashSet(); private Set orders = new LinkedHashSet(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java index 509fda2ab..1c1189a94 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/RuleSemanticQuery.java @@ -220,7 +220,8 @@ public abstract class RuleSemanticQuery implements SemanticQuery, Serializable { try { explainSqlReq = ExplainSqlReq.builder() .queryTypeEnum(QueryTypeEnum.STRUCT) - .queryReq(convertQueryStruct()) + .queryReq(isMultiStructQuery() + ? convertQueryMultiStruct() : convertQueryStruct()) .build(); return semanticInterpreter.explain(explainSqlReq, user); } catch (Exception e) { @@ -229,6 +230,10 @@ public abstract class RuleSemanticQuery implements SemanticQuery, Serializable { return null; } + protected boolean isMultiStructQuery() { + return false; + } + public QueryResult multiStructExecute(User user) { String queryMode = parseInfo.getQueryMode(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricEntityQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricEntityQuery.java index f07749a93..1fd72808e 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricEntityQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricEntityQuery.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.chat.query.rule.metric; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; +import com.tencent.supersonic.common.pojo.enums.FilterType; import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum; import com.tencent.supersonic.semantic.api.query.pojo.Filter; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; @@ -52,7 +53,7 @@ public class MetricEntityQuery extends MetricSemanticQuery { parseInfo.getDimensionFilters().stream() .filter(filter -> filter.getElementID() != null) .forEach(filter -> filterBizName.add(filter.getBizName())); - return filterBizName.size() > 1; + return FilterType.UNION.equals(parseInfo.getFilterType()) && filterBizName.size() > 1; } @Override diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricFilterQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricFilterQuery.java index eaefcbb1c..7270cc052 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricFilterQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/metric/MetricFilterQuery.java @@ -6,6 +6,7 @@ import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNum import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; +import com.tencent.supersonic.common.pojo.enums.FilterType; import com.tencent.supersonic.semantic.api.query.enums.FilterOperatorEnum; import com.tencent.supersonic.semantic.api.query.pojo.Filter; import com.tencent.supersonic.semantic.api.query.request.QueryMultiStructReq; @@ -48,7 +49,7 @@ public class MetricFilterQuery extends MetricSemanticQuery { Set filterBizName = new HashSet<>(); parseInfo.getDimensionFilters().forEach(filter -> filterBizName.add(filter.getBizName())); - return filterBizName.size() > 1; + return FilterType.UNION.equals(parseInfo.getFilterType()) && filterBizName.size() > 1; } @Override diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/FilterType.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/FilterType.java new file mode 100644 index 000000000..a92524b29 --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/FilterType.java @@ -0,0 +1,8 @@ +package com.tencent.supersonic.common.pojo.enums; + +public enum FilterType { + //filtering between different dimensions will directly splice the AND clause + AND, + //filtering between different dimensions will generate multiple sql clauses and splice them together using union + UNION +} diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java index 00fbaded2..3e98778b9 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/service/QueryServiceImpl.java @@ -172,15 +172,7 @@ public class QueryServiceImpl implements QueryService { } log.info("stat queryByStructWithoutCache, queryMultiStructReq:{}", queryMultiStructReq); try { - List sqlParsers = new ArrayList<>(); - for (QueryStructReq queryStructCmd : queryMultiStructReq.getQueryStructReqs()) { - QueryStatement queryStatement = semanticQueryEngine.plan(queryStructCmd); - queryUtils.checkSqlParse(queryStatement); - sqlParsers.add(queryStatement); - } - log.info("multi sqlParser:{}", sqlParsers); - - QueryStatement sqlParser = queryUtils.sqlParserUnion(queryMultiStructReq, sqlParsers); + QueryStatement sqlParser = getQueryStatementByMultiStruct(queryMultiStructReq); queryResultWithColumns = semanticQueryEngine.execute(sqlParser); if (queryResultWithColumns != null) { statUtils.statInfo2DbAsync(TaskStatusEnum.SUCCESS); @@ -194,6 +186,17 @@ public class QueryServiceImpl implements QueryService { } } + private QueryStatement getQueryStatementByMultiStruct(QueryMultiStructReq queryMultiStructReq) throws Exception { + List sqlParsers = new ArrayList<>(); + for (QueryStructReq queryStructCmd : queryMultiStructReq.getQueryStructReqs()) { + QueryStatement queryStatement = semanticQueryEngine.plan(queryStructCmd); + queryUtils.checkSqlParse(queryStatement); + sqlParsers.add(queryStatement); + } + log.info("multi sqlParser:{}", sqlParsers); + return queryUtils.sqlParserUnion(queryMultiStructReq, sqlParsers); + } + @Override @SneakyThrows public QueryResultWithSchemaResp queryDimValue(QueryDimValueReq queryDimValueReq, User user) { @@ -236,6 +239,11 @@ public class QueryServiceImpl implements QueryService { QueryStatement queryStatement = semanticQueryEngine.plan((QueryStructReq) queryReq); return getExplainResp(queryStatement); } + if (QueryTypeEnum.STRUCT.equals(queryTypeEnum) && queryReq instanceof QueryMultiStructReq) { + QueryMultiStructReq queryMultiStructReq = (QueryMultiStructReq) queryReq; + QueryStatement queryStatement = getQueryStatementByMultiStruct(queryMultiStructReq); + return getExplainResp(queryStatement); + } throw new IllegalArgumentException("Parameters are invalid, explainSqlReq: " + explainSqlReq); }