diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java index 57aeb2e39..221897edc 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java +++ b/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/SemanticParseInfo.java @@ -6,6 +6,7 @@ import com.tencent.supersonic.chat.api.pojo.response.EntityInfo; import com.tencent.supersonic.chat.api.pojo.response.SqlInfo; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.Order; +import com.tencent.supersonic.common.pojo.QueryType; import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; import java.util.ArrayList; import java.util.Comparator; @@ -34,7 +35,6 @@ public class SemanticParseInfo { private Set orders = new LinkedHashSet(); private DateConf dateInfo; private Long limit; - private Boolean nativeQuery = false; private double score; private List elementMatches = new ArrayList<>(); private Map properties = new HashMap<>(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/QueryTypeParser.java b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/QueryTypeParser.java index bb62d8af0..af930b89a 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/parser/QueryTypeParser.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/parser/QueryTypeParser.java @@ -5,7 +5,7 @@ import com.tencent.supersonic.chat.api.component.SemanticParser; import com.tencent.supersonic.chat.api.component.SemanticQuery; import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.QueryContext; -import com.tencent.supersonic.chat.api.pojo.QueryType; +import com.tencent.supersonic.common.pojo.QueryType; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; @@ -26,7 +26,7 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; /** - * Query type parser, determine if the query is a metric query, a entity query, + * Query type parser, determine if the query is a metric query, an entity query, * or another type of query. */ @Slf4j diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/interpret/MetricInterpretQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/interpret/MetricInterpretQuery.java index 5f41b7244..3b5604c00 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/interpret/MetricInterpretQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/llm/interpret/MetricInterpretQuery.java @@ -17,6 +17,7 @@ import com.tencent.supersonic.chat.utils.ComponentFactory; import com.tencent.supersonic.chat.utils.QueryReqBuilder; import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.QueryColumn; +import com.tencent.supersonic.common.pojo.QueryType; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp; @@ -89,7 +90,7 @@ public class MetricInterpretQuery extends LLMSemanticQuery { protected QueryStructReq convertQueryStruct() { QueryStructReq queryStructReq = QueryReqBuilder.buildStructReq(parseInfo); fillAggregator(queryStructReq, parseInfo.getMetrics()); - queryStructReq.setNativeQuery(true); + queryStructReq.setQueryType(QueryType.ENTITY); return queryStructReq; } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/entity/EntitySemanticQuery.java b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/entity/EntitySemanticQuery.java index 83b0f2f04..1145781fc 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/entity/EntitySemanticQuery.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/query/rule/entity/EntitySemanticQuery.java @@ -1,7 +1,12 @@ package com.tencent.supersonic.chat.query.rule.entity; +import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ENTITY; +import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.REQUIRED; +import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; + import com.tencent.supersonic.chat.api.pojo.ChatContext; import com.tencent.supersonic.chat.api.pojo.QueryContext; +import com.tencent.supersonic.common.pojo.QueryType; import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch; import com.tencent.supersonic.chat.api.pojo.SchemaElementType; import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; @@ -11,17 +16,12 @@ import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery; import com.tencent.supersonic.chat.service.ConfigService; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.util.ContextUtils; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections.CollectionUtils; - import java.time.LocalDate; import java.util.ArrayList; import java.util.List; import java.util.Objects; - -import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ENTITY; -import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST; -import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.REQUIRED; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; @Slf4j public abstract class EntitySemanticQuery extends RuleSemanticQuery { @@ -81,7 +81,7 @@ public abstract class EntitySemanticQuery extends RuleSemanticQuery { public void fillParseInfo(Long modelId, QueryContext queryContext, ChatContext chatContext) { super.fillParseInfo(modelId, queryContext, chatContext); - parseInfo.setNativeQuery(true); + parseInfo.setQueryType(QueryType.ENTITY); parseInfo.setLimit(ENTITY_MAX_RESULTS); if (parseInfo.getDateInfo() == null) { ConfigService configService = ContextUtils.getBean(ConfigService.class); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/responder/execute/EntityInfoExecuteResponder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/responder/execute/EntityInfoExecuteResponder.java index 05418c896..59194a69f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/responder/execute/EntityInfoExecuteResponder.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/responder/execute/EntityInfoExecuteResponder.java @@ -9,12 +9,6 @@ import com.tencent.supersonic.chat.query.QueryManager; import com.tencent.supersonic.chat.query.llm.interpret.MetricInterpretQuery; import com.tencent.supersonic.chat.service.SemanticService; import com.tencent.supersonic.common.util.ContextUtils; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; public class EntityInfoExecuteResponder implements ExecuteResponder { @@ -33,29 +27,6 @@ public class EntityInfoExecuteResponder implements ExecuteResponder { EntityInfo entityInfo = semanticService.getEntityInfo(semanticParseInfo, user); queryResult.setEntityInfo(entityInfo); - String primaryEntityBizName = semanticService.getPrimaryEntityBizName(entityInfo); - if (StringUtils.isEmpty(primaryEntityBizName) - || CollectionUtils.isEmpty(queryResult.getQueryColumns())) { - return; - } - boolean existPrimaryEntityName = queryResult.getQueryColumns().stream() - .anyMatch(queryColumn -> primaryEntityBizName.equals(queryColumn.getNameEn())); - - semanticParseInfo.setNativeQuery(existPrimaryEntityName); - - if (!existPrimaryEntityName) { - return; - } - List> queryResults = queryResult.getQueryResults(); - List entities = queryResults.stream() - .map(entry -> entry.get(primaryEntityBizName)) - .filter(Objects::nonNull) - .map(String::valueOf) - .collect(Collectors.toList()); - - if (CollectionUtils.isEmpty(entities)) { - return; - } } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/EntityInfoParseResponder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/EntityInfoParseResponder.java index b563884e4..b45318b80 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/EntityInfoParseResponder.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/responder/parse/EntityInfoParseResponder.java @@ -11,7 +11,6 @@ import com.tencent.supersonic.chat.query.llm.interpret.MetricInterpretQuery; import com.tencent.supersonic.chat.service.SemanticService; import com.tencent.supersonic.common.util.ContextUtils; import java.util.List; -import org.apache.commons.lang3.StringUtils; import org.springframework.util.CollectionUtils; public class EntityInfoParseResponder implements ParseResponder { @@ -37,14 +36,6 @@ public class EntityInfoParseResponder implements ParseResponder { || QueryManager.isMetricQuery(queryMode)) { parseInfo.setEntityInfo(entityInfo); } - //2. set native value - String primaryEntityBizName = semanticService.getPrimaryEntityBizName(entityInfo); - if (StringUtils.isNotEmpty(primaryEntityBizName)) { - //if exist primaryEntityBizName in parseInfo's dimensions, set nativeQuery to true - boolean existPrimaryEntityBizName = parseInfo.getDimensions().stream() - .anyMatch(schemaElement -> primaryEntityBizName.equalsIgnoreCase(schemaElement.getBizName())); - parseInfo.setNativeQuery(existPrimaryEntityBizName); - } }); } } diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/SemanticService.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/SemanticService.java index fc91493af..d0c89f72f 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/SemanticService.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/SemanticService.java @@ -14,6 +14,7 @@ import static com.tencent.supersonic.common.pojo.Constants.WEEK; import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.component.SemanticInterpreter; import com.tencent.supersonic.chat.api.pojo.ModelSchema; +import com.tencent.supersonic.common.pojo.QueryType; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq; @@ -233,7 +234,7 @@ public class SemanticService { modelInfo.setEntityId(entities.get(0)); SemanticParseInfo semanticParseInfo = new SemanticParseInfo(); semanticParseInfo.setModel(modelSchema.getModel()); - semanticParseInfo.setNativeQuery(true); + semanticParseInfo.setQueryType(QueryType.ENTITY); semanticParseInfo.setMetrics(getMetrics(modelInfo)); semanticParseInfo.setDimensions(getDimensions(modelInfo)); DateConf dateInfo = new DateConf(); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ParserInfoServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ParserInfoServiceImpl.java index f3746219c..07206b413 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ParserInfoServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/ParserInfoServiceImpl.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.chat.service.impl; import com.google.common.collect.Lists; +import com.tencent.supersonic.common.pojo.QueryType; import com.tencent.supersonic.chat.api.pojo.SchemaElement; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.SemanticSchema; @@ -79,13 +80,13 @@ public class ParserInfoServiceImpl implements ParseInfoService { parseInfo.setMetrics(metrics); if (SqlParserSelectFunctionHelper.hasAggregateFunction(sqlInfo.getCorrectS2SQL())) { - parseInfo.setNativeQuery(false); + parseInfo.setQueryType(QueryType.METRIC); List groupByFields = SqlParserSelectHelper.getGroupByFields(sqlInfo.getCorrectS2SQL()); List groupByDimensions = getFieldsExceptDate(groupByFields); parseInfo.setDimensions( getElements(parseInfo.getModelId(), groupByDimensions, semanticSchema.getDimensions())); } else { - parseInfo.setNativeQuery(true); + parseInfo.setQueryType(QueryType.ENTITY); List selectFields = SqlParserSelectHelper.getSelectFields(sqlInfo.getCorrectS2SQL()); List selectDimensions = getFieldsExceptDate(selectFields); parseInfo.setDimensions( diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java index 134d5c548..99a9b7ab0 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/service/impl/QueryServiceImpl.java @@ -42,6 +42,7 @@ import com.tencent.supersonic.chat.utils.ComponentFactory; import com.tencent.supersonic.chat.utils.SolvedQueryManager; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.QueryColumn; +import com.tencent.supersonic.common.pojo.QueryType; import com.tencent.supersonic.common.pojo.enums.DictWordType; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; @@ -609,7 +610,7 @@ public class QueryServiceImpl implements QueryService { queryStructReq.setDateInfo(dateConf); queryStructReq.setLimit(20L); queryStructReq.setModelId(dimensionValueReq.getModelId()); - queryStructReq.setNativeQuery(false); + queryStructReq.setQueryType(QueryType.OTHER); List groups = new ArrayList<>(); groups.add(dimensionValueReq.getBizName()); queryStructReq.setGroups(groups); diff --git a/chat/core/src/main/java/com/tencent/supersonic/chat/utils/QueryReqBuilder.java b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/QueryReqBuilder.java index 378fe5677..2f7e81dfd 100644 --- a/chat/core/src/main/java/com/tencent/supersonic/chat/utils/QueryReqBuilder.java +++ b/chat/core/src/main/java/com/tencent/supersonic/chat/utils/QueryReqBuilder.java @@ -9,6 +9,7 @@ import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.Filter; import com.tencent.supersonic.common.pojo.Order; +import com.tencent.supersonic.common.pojo.QueryType; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; @@ -36,7 +37,7 @@ public class QueryReqBuilder { public static QueryStructReq buildStructReq(SemanticParseInfo parseInfo) { QueryStructReq queryStructCmd = new QueryStructReq(); queryStructCmd.setModelId(parseInfo.getModelId()); - queryStructCmd.setNativeQuery(parseInfo.getNativeQuery()); + queryStructCmd.setQueryType(parseInfo.getQueryType()); queryStructCmd.setDateInfo(rewrite2Between(parseInfo.getDateInfo())); List dimensionFilters = parseInfo.getDimensionFilters().stream() @@ -231,7 +232,7 @@ public class QueryReqBuilder { public static QueryStructReq buildStructRatioReq(SemanticParseInfo parseInfo, SchemaElement metric, AggOperatorEnum aggOperatorEnum) { QueryStructReq queryStructCmd = buildStructReq(parseInfo); - queryStructCmd.setNativeQuery(false); + queryStructCmd.setQueryType(QueryType.METRIC); queryStructCmd.setOrders(new ArrayList<>()); List aggregators = new ArrayList<>(); Aggregator ratioRoll = new Aggregator(metric.getBizName(), aggOperatorEnum); diff --git a/chat/core/src/test/java/com/tencent/supersonic/chat/utils/QueryReqBuilderTest.java b/chat/core/src/test/java/com/tencent/supersonic/chat/utils/QueryReqBuilderTest.java index a9ce920bc..d4ca3e2c2 100644 --- a/chat/core/src/test/java/com/tencent/supersonic/chat/utils/QueryReqBuilderTest.java +++ b/chat/core/src/test/java/com/tencent/supersonic/chat/utils/QueryReqBuilderTest.java @@ -5,6 +5,7 @@ import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.DateConf.DateMode; import com.tencent.supersonic.common.pojo.Order; +import com.tencent.supersonic.common.pojo.QueryType; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.DateModeUtils; @@ -29,7 +30,7 @@ class QueryReqBuilderTest { init(); QueryStructReq queryStructReq = new QueryStructReq(); queryStructReq.setModelId(1L); - queryStructReq.setNativeQuery(false); + queryStructReq.setQueryType(QueryType.METRIC); queryStructReq.setModelName("内容库"); Aggregator aggregator = new Aggregator(); @@ -55,7 +56,7 @@ class QueryReqBuilderTest { "SELECT department, SUM(pv) FROM 内容库 WHERE (sys_imp_date IN ('2023-08-01')) " + "GROUP BY department ORDER BY uv LIMIT 2000", queryS2SQLReq.getSql()); - queryStructReq.setNativeQuery(true); + queryStructReq.setQueryType(QueryType.ENTITY); queryS2SQLReq = queryStructReq.convert(queryStructReq); Assert.assertEquals( "SELECT department, pv FROM 内容库 WHERE (sys_imp_date IN ('2023-08-01')) " diff --git a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/QueryType.java b/common/src/main/java/com/tencent/supersonic/common/pojo/QueryType.java similarity index 66% rename from chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/QueryType.java rename to common/src/main/java/com/tencent/supersonic/common/pojo/QueryType.java index 191961714..fb66ce210 100644 --- a/chat/api/src/main/java/com/tencent/supersonic/chat/api/pojo/QueryType.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/QueryType.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.chat.api.pojo; +package com.tencent.supersonic.common.pojo; /*** * Query Type @@ -15,5 +15,9 @@ public enum QueryType { /** * the other queries */ - OTHER + OTHER; + + public boolean isNativeAggQuery() { + return ENTITY.equals(this); + } } diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/EntityQueryTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/EntityQueryTest.java index 87b64adf1..3976b7e5d 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/EntityQueryTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/EntityQueryTest.java @@ -10,6 +10,7 @@ import com.tencent.supersonic.chat.query.rule.entity.EntityFilterQuery; import com.tencent.supersonic.chat.query.rule.metric.MetricEntityQuery; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.DateConf.DateMode; +import com.tencent.supersonic.common.pojo.QueryType; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.util.DataUtils; import java.util.ArrayList; @@ -36,7 +37,7 @@ public class EntityQueryTest extends BaseQueryTest { expectedParseInfo.getMetrics().add(metric); expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateMode.BETWEEN, 1, period, startDay, endDay)); - expectedParseInfo.setNativeQuery(false); + expectedParseInfo.setQueryType(QueryType.METRIC); assertQueryResult(expectedResult, actualResult); } @@ -71,7 +72,7 @@ public class EntityQueryTest extends BaseQueryTest { expectedParseInfo.getDimensions().add(dim4); expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.BETWEEN, startDay, startDay)); - expectedParseInfo.setNativeQuery(true); + expectedParseInfo.setQueryType(QueryType.ENTITY); assertQueryResult(expectedResult, actualResult); } diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/MetricQueryTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/MetricQueryTest.java index 9575cef5b..0c1a79a86 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/MetricQueryTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/MetricQueryTest.java @@ -1,33 +1,34 @@ package com.tencent.supersonic.integration; +import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE; +import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.SUM; + import com.tencent.supersonic.auth.api.authentication.pojo.User; import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq; +import com.tencent.supersonic.chat.api.pojo.request.ItemVisibility; import com.tencent.supersonic.chat.api.pojo.request.QueryFilter; +import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; import com.tencent.supersonic.chat.api.pojo.response.ParseResp; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; -import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq; -import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp; -import com.tencent.supersonic.chat.api.pojo.request.ItemVisibility; -import com.tencent.supersonic.chat.query.rule.metric.MetricModelQuery; import com.tencent.supersonic.chat.query.rule.metric.MetricFilterQuery; import com.tencent.supersonic.chat.query.rule.metric.MetricGroupByQuery; +import com.tencent.supersonic.chat.query.rule.metric.MetricModelQuery; import com.tencent.supersonic.chat.query.rule.metric.MetricTopNQuery; import com.tencent.supersonic.common.pojo.DateConf; +import com.tencent.supersonic.common.pojo.QueryType; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.util.DataUtils; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; import org.junit.Assert; import org.junit.Test; import org.springframework.beans.BeanUtils; -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.List; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.stream.Collectors; -import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.SUM; -import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE; - public class MetricQueryTest extends BaseQueryTest { @@ -48,7 +49,7 @@ public class MetricQueryTest extends BaseQueryTest { FilterOperatorEnum.EQUALS, "alice", "用户名", 2L)); expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); - expectedParseInfo.setNativeQuery(false); + expectedParseInfo.setQueryType(QueryType.METRIC); assertQueryResult(expectedResult, actualResult); } @@ -78,7 +79,7 @@ public class MetricQueryTest extends BaseQueryTest { expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("访问次数")); expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); - expectedParseInfo.setNativeQuery(false); + expectedParseInfo.setQueryType(QueryType.METRIC); assertQueryResult(expectedResult, actualResult); } @@ -108,7 +109,7 @@ public class MetricQueryTest extends BaseQueryTest { expectedParseInfo.getDimensions().add(DataUtils.getSchemaElement("部门")); expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); - expectedParseInfo.setNativeQuery(false); + expectedParseInfo.setQueryType(QueryType.METRIC); assertQueryResult(expectedResult, actualResult); } @@ -133,7 +134,7 @@ public class MetricQueryTest extends BaseQueryTest { expectedParseInfo.getDimensionFilters().add(dimensionFilter); expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); - expectedParseInfo.setNativeQuery(false); + expectedParseInfo.setQueryType(QueryType.METRIC); assertQueryResult(expectedResult, actualResult); } @@ -153,7 +154,7 @@ public class MetricQueryTest extends BaseQueryTest { expectedParseInfo.getDimensions().add(DataUtils.getSchemaElement("用户名")); expectedParseInfo.setDateInfo(DataUtils.getDateConf(3, DateConf.DateMode.RECENT, "DAY")); - expectedParseInfo.setNativeQuery(false); + expectedParseInfo.setQueryType(QueryType.METRIC); assertQueryResult(expectedResult, actualResult); } @@ -172,7 +173,7 @@ public class MetricQueryTest extends BaseQueryTest { expectedParseInfo.getDimensions().add(DataUtils.getSchemaElement("部门")); expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); - expectedParseInfo.setNativeQuery(false); + expectedParseInfo.setQueryType(QueryType.METRIC); assertQueryResult(expectedResult, actualResult); } @@ -198,7 +199,7 @@ public class MetricQueryTest extends BaseQueryTest { FilterOperatorEnum.EQUALS, "alice", "用户名", 2L)); expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.BETWEEN, 1, period, startDay, startDay)); - expectedParseInfo.setNativeQuery(false); + expectedParseInfo.setQueryType(QueryType.METRIC); assertQueryResult(expectedResult, actualResult); } @@ -225,7 +226,7 @@ public class MetricQueryTest extends BaseQueryTest { expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("访问次数")); expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); - expectedParseInfo.setNativeQuery(false); + expectedParseInfo.setQueryType(QueryType.METRIC); assertQueryResult(expectedResult, actualResult); diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/MultiTurnsTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/MultiTurnsTest.java index 5e4015437..c6f9645c9 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/MultiTurnsTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/MultiTurnsTest.java @@ -1,19 +1,19 @@ package com.tencent.supersonic.integration; +import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE; + import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.query.rule.metric.MetricFilterQuery; import com.tencent.supersonic.chat.query.rule.metric.MetricGroupByQuery; import com.tencent.supersonic.common.pojo.DateConf; +import com.tencent.supersonic.common.pojo.QueryType; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.util.DataUtils; -import org.junit.Test; -import org.junit.jupiter.api.Order; - import java.text.DateFormat; import java.text.SimpleDateFormat; - -import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE; +import org.junit.Test; +import org.junit.jupiter.api.Order; public class MultiTurnsTest extends BaseQueryTest { @@ -35,7 +35,7 @@ public class MultiTurnsTest extends BaseQueryTest { FilterOperatorEnum.EQUALS, "alice", "用户名", 2L)); expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); - expectedParseInfo.setNativeQuery(false); + expectedParseInfo.setQueryType(QueryType.METRIC); assertQueryResult(expectedResult, actualResult); } @@ -58,7 +58,7 @@ public class MultiTurnsTest extends BaseQueryTest { FilterOperatorEnum.EQUALS, "alice", "用户名", 2L)); expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); - expectedParseInfo.setNativeQuery(false); + expectedParseInfo.setQueryType(QueryType.METRIC); assertQueryResult(expectedResult, actualResult); } @@ -81,7 +81,7 @@ public class MultiTurnsTest extends BaseQueryTest { FilterOperatorEnum.EQUALS, "lucy", "用户名", 2L)); expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); - expectedParseInfo.setNativeQuery(false); + expectedParseInfo.setQueryType(QueryType.METRIC); assertQueryResult(expectedResult, actualResult); } @@ -102,7 +102,7 @@ public class MultiTurnsTest extends BaseQueryTest { expectedParseInfo.getDimensions().add(DataUtils.getSchemaElement("部门")); expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay)); - expectedParseInfo.setNativeQuery(false); + expectedParseInfo.setQueryType(QueryType.METRIC); assertQueryResult(expectedResult, actualResult); } @@ -125,7 +125,7 @@ public class MultiTurnsTest extends BaseQueryTest { expectedParseInfo.getDimensions().add(DataUtils.getSchemaElement("部门")); expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.BETWEEN, 1, period, startDay, startDay)); - expectedParseInfo.setNativeQuery(false); + expectedParseInfo.setQueryType(QueryType.METRIC); assertQueryResult(expectedResult, actualResult); } @@ -146,7 +146,7 @@ public class MultiTurnsTest extends BaseQueryTest { expectedParseInfo.getDimensions().add(DataUtils.getSchemaElement("部门")); expectedParseInfo.setDateInfo(DataUtils.getDateConf(30, DateConf.DateMode.RECENT, "DAY")); - expectedParseInfo.setNativeQuery(false); + expectedParseInfo.setQueryType(QueryType.METRIC); assertQueryResult(expectedResult, actualResult); } diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/mapper/MapperTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/mapper/MapperTest.java index d9cd0ec3e..34c3c8095 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/integration/mapper/MapperTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/integration/mapper/MapperTest.java @@ -9,6 +9,7 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryReq; import com.tencent.supersonic.chat.api.pojo.response.QueryResult; import com.tencent.supersonic.chat.query.rule.metric.MetricEntityQuery; import com.tencent.supersonic.common.pojo.DateConf; +import com.tencent.supersonic.common.pojo.QueryType; import com.tencent.supersonic.integration.BaseQueryTest; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.util.DataUtils; @@ -38,7 +39,7 @@ public class MapperTest extends BaseQueryTest { expectedParseInfo.getMetrics().add(metric); expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, 7, period, startDay, endDay)); - expectedParseInfo.setNativeQuery(false); + expectedParseInfo.setQueryType(QueryType.METRIC); assertQueryResult(expectedResult, actualResult); } diff --git a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/QueryStructReq.java b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/QueryStructReq.java index 71047e1d9..2b6e1d097 100644 --- a/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/QueryStructReq.java +++ b/semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/QueryStructReq.java @@ -6,6 +6,7 @@ import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.Filter; import com.tencent.supersonic.common.pojo.Order; +import com.tencent.supersonic.common.pojo.QueryType; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.DateModeUtils; @@ -57,7 +58,7 @@ public class QueryStructReq { private List params = new ArrayList<>(); private DateConf dateInfo; private Long limit = 2000L; - private Boolean nativeQuery = false; + private QueryType queryType = QueryType.OTHER; private Cache cacheInfo; /** @@ -121,8 +122,6 @@ public class QueryStructReq { .append(params); stringBuilder.append(",\"limit\":") .append(limit); - stringBuilder.append(",\"nativeQuery\":") - .append(nativeQuery); stringBuilder.append('}'); return stringBuilder.toString(); } @@ -157,8 +156,6 @@ public class QueryStructReq { .append(dateInfo); sb.append(",\"limit\":") .append(limit); - sb.append(",\"nativeQuery\":") - .append(nativeQuery); sb.append(",\"cacheInfo\":") .append(cacheInfo); sb.append('}'); @@ -202,7 +199,7 @@ public class QueryStructReq { if (!CollectionUtils.isEmpty(aggregators)) { for (Aggregator aggregator : aggregators) { String columnName = aggregator.getColumn(); - if (queryStructReq.getNativeQuery()) { + if (queryStructReq.getQueryType().isNativeAggQuery()) { selectItems.add(new SelectExpressionItem(new Column(columnName))); } else { Function sumFunction = new Function(); @@ -247,7 +244,7 @@ public class QueryStructReq { } //4.Set the group by clause - if (!CollectionUtils.isEmpty(groups) && !queryStructReq.getNativeQuery()) { + if (!CollectionUtils.isEmpty(groups) && !queryStructReq.getQueryType().isNativeAggQuery()) { GroupByElement groupByElement = new GroupByElement(); for (String group : groups) { groupByElement.addGroupByExpression(new Column(group)); diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/optimizer/DetailQuery.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/optimizer/DetailQuery.java index 1aced5aff..ae17c14ef 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/optimizer/DetailQuery.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/optimizer/DetailQuery.java @@ -32,7 +32,7 @@ public class DetailQuery implements QueryOptimizer { } public boolean isDetailQuery(QueryStructReq queryStructCmd) { - return Objects.nonNull(queryStructCmd) && queryStructCmd.getNativeQuery() && CollectionUtils.isEmpty( - queryStructCmd.getMetrics()); + return Objects.nonNull(queryStructCmd) && queryStructCmd.getQueryType().isNativeAggQuery() + && CollectionUtils.isEmpty(queryStructCmd.getMetrics()); } } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/optimizer/MaterializationQuery.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/optimizer/MaterializationQuery.java index f9345cacf..9896ca73b 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/optimizer/MaterializationQuery.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/optimizer/MaterializationQuery.java @@ -266,7 +266,8 @@ public class MaterializationQuery implements QueryOptimizer { getTimeRanges(queryStructReq, queryStatement); removeDefaultMetric(queryStructReq, queryStatement.getMetricReq()); MaterializationPlanner materializationPlanner = new MaterializationPlanner(schema); - materializationPlanner.explain(queryStatement, AggOption.getAggregation(queryStructReq.getNativeQuery())); + materializationPlanner.explain(queryStatement, + AggOption.getAggregation(queryStructReq.getQueryType().isNativeAggQuery())); log.info("optimize {}", materializationPlanner.findBest().getDatasource()); SemanticSchema semanticSchema = materializationPlanner.findBest(); if (!CollectionUtils.isEmpty(semanticSchema.getDatasource())) { @@ -278,7 +279,8 @@ public class MaterializationQuery implements QueryOptimizer { doSingleZipperSource(queryStructReq, queryStatement); } AggPlanner aggBuilder = new AggPlanner(semanticSchema); - aggBuilder.explain(queryStatement, AggOption.getAggregation(queryStructReq.getNativeQuery())); + aggBuilder.explain(queryStatement, + AggOption.getAggregation(queryStructReq.getQueryType().isNativeAggQuery())); log.debug("optimize before {} sql {}", queryStatement.getSourceId(), queryStatement.getSql()); log.debug("optimize after {} sql {}", aggBuilder.getSourceId(), aggBuilder.getSql()); queryStatement.setSourceId(aggBuilder.getSourceId()); diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/QueryParser.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/QueryParser.java index 6fe9780e4..efc70635e 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/QueryParser.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/QueryParser.java @@ -44,7 +44,8 @@ public class QueryParser { if (!parseSqlReq.getSql().isEmpty()) { return parser(parseSqlReq); } - metricReq.setNativeQuery(queryStructReq.getNativeQuery()); + + metricReq.setNativeQuery(queryStructReq.getQueryType().isNativeAggQuery()); return parser(metricReq); } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/CalculateAggConverter.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/CalculateAggConverter.java index 866b60ae1..4ce00e4a1 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/CalculateAggConverter.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/CalculateAggConverter.java @@ -95,7 +95,7 @@ public class CalculateAggConverter implements SemanticConverter { @Override public boolean accept(QueryStructReq queryStructCmd) { - if (queryStructCmd.getNativeQuery()) { + if (queryStructCmd.getQueryType().isNativeAggQuery()) { return false; } if (CollectionUtils.isEmpty(queryStructCmd.getAggregators())) { diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/MetricCheckConverter.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/MetricCheckConverter.java index 0f6e11eee..9853427c0 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/MetricCheckConverter.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/MetricCheckConverter.java @@ -24,7 +24,7 @@ public class MetricCheckConverter implements SemanticConverter { @Override public boolean accept(QueryStructReq queryStructCmd) { - if (queryStructCmd.getNativeQuery()) { + if (queryStructCmd.getQueryType().isNativeAggQuery()) { return false; } return !CollectionUtils.isEmpty(queryStructCmd.getAggregators()); diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/ParserDefaultConverter.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/ParserDefaultConverter.java index 3329d7847..bbc77ed6a 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/ParserDefaultConverter.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/ParserDefaultConverter.java @@ -62,7 +62,7 @@ public class ParserDefaultConverter implements SemanticConverter { // todo tmp delete // support detail query - if (queryStructCmd.getNativeQuery() && CollectionUtils.isEmpty(sqlCommend.getMetrics())) { + if (queryStructCmd.getQueryType().isNativeAggQuery() && CollectionUtils.isEmpty(sqlCommend.getMetrics())) { String internalMetricName = queryStructUtils.generateInternalMetricName( queryStructCmd.getModelId(), queryStructCmd.getGroups()); sqlCommend.getMetrics().add(internalMetricName); diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/QueryReqConverter.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/QueryReqConverter.java index df5e05d64..a9ed84119 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/QueryReqConverter.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/QueryReqConverter.java @@ -3,6 +3,7 @@ package com.tencent.supersonic.semantic.query.parser.convert; import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.Constants; +import com.tencent.supersonic.common.pojo.QueryType; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper; @@ -112,7 +113,7 @@ public class QueryReqConverter { //5.physicalSql by ParseSqlReq queryStructCmd.setDateInfo(queryStructUtils.getDateConfBySql(databaseReq.getSql())); queryStructCmd.setModelId(databaseReq.getModelId()); - queryStructCmd.setNativeQuery(!AggOption.isAgg(aggOption)); + queryStructCmd.setQueryType(getQueryType(aggOption)); log.info("QueryReqConverter queryStructCmd[{}]", queryStructCmd); QueryStatement queryStatement = parserService.physicalSql(queryStructCmd, result); queryStatement.setSql(String.format(SqlExecuteReq.LIMIT_WRAPPER, queryStatement.getSql())); @@ -146,18 +147,16 @@ public class QueryReqConverter { Map internalLowerToNameMap = QueryStructUtils.internalCols.stream() .collect(Collectors.toMap(a -> a.toLowerCase(), a -> a)); dimensionLowerToNameMap.putAll(internalLowerToNameMap); - Set collect = allFields.stream() + return allFields.stream() .filter(entry -> dimensionLowerToNameMap.containsKey(entry.toLowerCase())) .map(entry -> dimensionLowerToNameMap.get(entry.toLowerCase())).collect(Collectors.toSet()); - return collect; } private List getMetrics(ModelSchemaResp modelSchemaResp, List allFields) { Map metricLowerToNameMap = modelSchemaResp.getMetrics().stream() .collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(), entry -> entry.getBizName())); - List metrics = allFields.stream().filter(entry -> metricLowerToNameMap.containsKey(entry.toLowerCase())) + return allFields.stream().filter(entry -> metricLowerToNameMap.containsKey(entry.toLowerCase())) .map(entry -> metricLowerToNameMap.get(entry.toLowerCase())).collect(Collectors.toList()); - return metrics; } private void functionNameCorrector(QueryS2SQLReq databaseReq) { @@ -175,7 +174,6 @@ public class QueryReqConverter { } } - protected Map getFieldNameToBizNameMap(ModelSchemaResp modelSchemaResp) { // support fieldName and field alias to bizName Map dimensionResults = modelSchemaResp.getDimensions().stream() @@ -216,4 +214,13 @@ public class QueryReqConverter { databaseReq.setSql(sql); } + private QueryType getQueryType(AggOption aggOption) { + boolean isAgg = AggOption.isAgg(aggOption); + QueryType queryType = QueryType.ENTITY; + if (isAgg) { + queryType = QueryType.METRIC; + } + return queryType; + } + } diff --git a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/StatUtils.java b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/StatUtils.java index 0831ccab4..ba8892c62 100644 --- a/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/StatUtils.java +++ b/semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/StatUtils.java @@ -144,7 +144,7 @@ public class StatUtils { .setQueryStructCmd(queryStructCmd.toString()) .setQueryStructCmdMd5(DigestUtils.md5Hex(queryStructCmd.toString())) .setStartTime(System.currentTimeMillis()) - .setNativeQuery(queryStructCmd.getNativeQuery()) + .setNativeQuery(queryStructCmd.getQueryType().isNativeAggQuery()) .setGroupByCols(objectMapper.writeValueAsString(queryStructCmd.getGroups())) .setAggCols(objectMapper.writeValueAsString(queryStructCmd.getAggregators())) .setOrderByCols(objectMapper.writeValueAsString(queryStructCmd.getOrders()))