mirror of
https://github.com/tencentmusic/supersonic.git
synced 2025-12-10 19:38:13 +00:00
[improvement](chat) remove nativeQuery config in chat (#394)
This commit is contained in:
@@ -6,6 +6,7 @@ import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.pojo.Order;
|
||||
import com.tencent.supersonic.common.pojo.QueryType;
|
||||
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
@@ -34,7 +35,6 @@ public class SemanticParseInfo {
|
||||
private Set<Order> orders = new LinkedHashSet();
|
||||
private DateConf dateInfo;
|
||||
private Long limit;
|
||||
private Boolean nativeQuery = false;
|
||||
private double score;
|
||||
private List<SchemaElementMatch> elementMatches = new ArrayList<>();
|
||||
private Map<String, Object> properties = new HashMap<>();
|
||||
|
||||
@@ -5,7 +5,7 @@ import com.tencent.supersonic.chat.api.component.SemanticParser;
|
||||
import com.tencent.supersonic.chat.api.component.SemanticQuery;
|
||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
||||
import com.tencent.supersonic.chat.api.pojo.QueryType;
|
||||
import com.tencent.supersonic.common.pojo.QueryType;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
||||
@@ -26,7 +26,7 @@ import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
/**
|
||||
* Query type parser, determine if the query is a metric query, a entity query,
|
||||
* Query type parser, determine if the query is a metric query, an entity query,
|
||||
* or another type of query.
|
||||
*/
|
||||
@Slf4j
|
||||
|
||||
@@ -17,6 +17,7 @@ import com.tencent.supersonic.chat.utils.ComponentFactory;
|
||||
import com.tencent.supersonic.chat.utils.QueryReqBuilder;
|
||||
import com.tencent.supersonic.common.pojo.Aggregator;
|
||||
import com.tencent.supersonic.common.pojo.QueryColumn;
|
||||
import com.tencent.supersonic.common.pojo.QueryType;
|
||||
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
|
||||
import com.tencent.supersonic.common.util.ContextUtils;
|
||||
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
|
||||
@@ -89,7 +90,7 @@ public class MetricInterpretQuery extends LLMSemanticQuery {
|
||||
protected QueryStructReq convertQueryStruct() {
|
||||
QueryStructReq queryStructReq = QueryReqBuilder.buildStructReq(parseInfo);
|
||||
fillAggregator(queryStructReq, parseInfo.getMetrics());
|
||||
queryStructReq.setNativeQuery(true);
|
||||
queryStructReq.setQueryType(QueryType.ENTITY);
|
||||
return queryStructReq;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,12 @@
|
||||
package com.tencent.supersonic.chat.query.rule.entity;
|
||||
|
||||
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ENTITY;
|
||||
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.REQUIRED;
|
||||
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.ChatContext;
|
||||
import com.tencent.supersonic.chat.api.pojo.QueryContext;
|
||||
import com.tencent.supersonic.common.pojo.QueryType;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
|
||||
@@ -11,17 +16,12 @@ import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery;
|
||||
import com.tencent.supersonic.chat.service.ConfigService;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.util.ContextUtils;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
|
||||
import java.time.LocalDate;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ENTITY;
|
||||
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
|
||||
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.REQUIRED;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
|
||||
@Slf4j
|
||||
public abstract class EntitySemanticQuery extends RuleSemanticQuery {
|
||||
@@ -81,7 +81,7 @@ public abstract class EntitySemanticQuery extends RuleSemanticQuery {
|
||||
public void fillParseInfo(Long modelId, QueryContext queryContext, ChatContext chatContext) {
|
||||
super.fillParseInfo(modelId, queryContext, chatContext);
|
||||
|
||||
parseInfo.setNativeQuery(true);
|
||||
parseInfo.setQueryType(QueryType.ENTITY);
|
||||
parseInfo.setLimit(ENTITY_MAX_RESULTS);
|
||||
if (parseInfo.getDateInfo() == null) {
|
||||
ConfigService configService = ContextUtils.getBean(ConfigService.class);
|
||||
|
||||
@@ -9,12 +9,6 @@ import com.tencent.supersonic.chat.query.QueryManager;
|
||||
import com.tencent.supersonic.chat.query.llm.interpret.MetricInterpretQuery;
|
||||
import com.tencent.supersonic.chat.service.SemanticService;
|
||||
import com.tencent.supersonic.common.util.ContextUtils;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.stream.Collectors;
|
||||
import org.apache.commons.collections.CollectionUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
public class EntityInfoExecuteResponder implements ExecuteResponder {
|
||||
|
||||
@@ -33,29 +27,6 @@ public class EntityInfoExecuteResponder implements ExecuteResponder {
|
||||
EntityInfo entityInfo = semanticService.getEntityInfo(semanticParseInfo, user);
|
||||
queryResult.setEntityInfo(entityInfo);
|
||||
|
||||
String primaryEntityBizName = semanticService.getPrimaryEntityBizName(entityInfo);
|
||||
if (StringUtils.isEmpty(primaryEntityBizName)
|
||||
|| CollectionUtils.isEmpty(queryResult.getQueryColumns())) {
|
||||
return;
|
||||
}
|
||||
boolean existPrimaryEntityName = queryResult.getQueryColumns().stream()
|
||||
.anyMatch(queryColumn -> primaryEntityBizName.equals(queryColumn.getNameEn()));
|
||||
|
||||
semanticParseInfo.setNativeQuery(existPrimaryEntityName);
|
||||
|
||||
if (!existPrimaryEntityName) {
|
||||
return;
|
||||
}
|
||||
List<Map<String, Object>> queryResults = queryResult.getQueryResults();
|
||||
List<String> entities = queryResults.stream()
|
||||
.map(entry -> entry.get(primaryEntityBizName))
|
||||
.filter(Objects::nonNull)
|
||||
.map(String::valueOf)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
if (CollectionUtils.isEmpty(entities)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -11,7 +11,6 @@ import com.tencent.supersonic.chat.query.llm.interpret.MetricInterpretQuery;
|
||||
import com.tencent.supersonic.chat.service.SemanticService;
|
||||
import com.tencent.supersonic.common.util.ContextUtils;
|
||||
import java.util.List;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
|
||||
public class EntityInfoParseResponder implements ParseResponder {
|
||||
@@ -37,14 +36,6 @@ public class EntityInfoParseResponder implements ParseResponder {
|
||||
|| QueryManager.isMetricQuery(queryMode)) {
|
||||
parseInfo.setEntityInfo(entityInfo);
|
||||
}
|
||||
//2. set native value
|
||||
String primaryEntityBizName = semanticService.getPrimaryEntityBizName(entityInfo);
|
||||
if (StringUtils.isNotEmpty(primaryEntityBizName)) {
|
||||
//if exist primaryEntityBizName in parseInfo's dimensions, set nativeQuery to true
|
||||
boolean existPrimaryEntityBizName = parseInfo.getDimensions().stream()
|
||||
.anyMatch(schemaElement -> primaryEntityBizName.equalsIgnoreCase(schemaElement.getBizName()));
|
||||
parseInfo.setNativeQuery(existPrimaryEntityBizName);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ import static com.tencent.supersonic.common.pojo.Constants.WEEK;
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.chat.api.component.SemanticInterpreter;
|
||||
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
|
||||
import com.tencent.supersonic.common.pojo.QueryType;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq;
|
||||
@@ -233,7 +234,7 @@ public class SemanticService {
|
||||
modelInfo.setEntityId(entities.get(0));
|
||||
SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
|
||||
semanticParseInfo.setModel(modelSchema.getModel());
|
||||
semanticParseInfo.setNativeQuery(true);
|
||||
semanticParseInfo.setQueryType(QueryType.ENTITY);
|
||||
semanticParseInfo.setMetrics(getMetrics(modelInfo));
|
||||
semanticParseInfo.setDimensions(getDimensions(modelInfo));
|
||||
DateConf dateInfo = new DateConf();
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
package com.tencent.supersonic.chat.service.impl;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import com.tencent.supersonic.common.pojo.QueryType;
|
||||
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
|
||||
@@ -79,13 +80,13 @@ public class ParserInfoServiceImpl implements ParseInfoService {
|
||||
parseInfo.setMetrics(metrics);
|
||||
|
||||
if (SqlParserSelectFunctionHelper.hasAggregateFunction(sqlInfo.getCorrectS2SQL())) {
|
||||
parseInfo.setNativeQuery(false);
|
||||
parseInfo.setQueryType(QueryType.METRIC);
|
||||
List<String> groupByFields = SqlParserSelectHelper.getGroupByFields(sqlInfo.getCorrectS2SQL());
|
||||
List<String> groupByDimensions = getFieldsExceptDate(groupByFields);
|
||||
parseInfo.setDimensions(
|
||||
getElements(parseInfo.getModelId(), groupByDimensions, semanticSchema.getDimensions()));
|
||||
} else {
|
||||
parseInfo.setNativeQuery(true);
|
||||
parseInfo.setQueryType(QueryType.ENTITY);
|
||||
List<String> selectFields = SqlParserSelectHelper.getSelectFields(sqlInfo.getCorrectS2SQL());
|
||||
List<String> selectDimensions = getFieldsExceptDate(selectFields);
|
||||
parseInfo.setDimensions(
|
||||
|
||||
@@ -42,6 +42,7 @@ import com.tencent.supersonic.chat.utils.ComponentFactory;
|
||||
import com.tencent.supersonic.chat.utils.SolvedQueryManager;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.pojo.QueryColumn;
|
||||
import com.tencent.supersonic.common.pojo.QueryType;
|
||||
import com.tencent.supersonic.common.pojo.enums.DictWordType;
|
||||
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
|
||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||
@@ -609,7 +610,7 @@ public class QueryServiceImpl implements QueryService {
|
||||
queryStructReq.setDateInfo(dateConf);
|
||||
queryStructReq.setLimit(20L);
|
||||
queryStructReq.setModelId(dimensionValueReq.getModelId());
|
||||
queryStructReq.setNativeQuery(false);
|
||||
queryStructReq.setQueryType(QueryType.OTHER);
|
||||
List<String> groups = new ArrayList<>();
|
||||
groups.add(dimensionValueReq.getBizName());
|
||||
queryStructReq.setGroups(groups);
|
||||
|
||||
@@ -9,6 +9,7 @@ import com.tencent.supersonic.common.pojo.Constants;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.pojo.Filter;
|
||||
import com.tencent.supersonic.common.pojo.Order;
|
||||
import com.tencent.supersonic.common.pojo.QueryType;
|
||||
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
|
||||
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
|
||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||
@@ -36,7 +37,7 @@ public class QueryReqBuilder {
|
||||
public static QueryStructReq buildStructReq(SemanticParseInfo parseInfo) {
|
||||
QueryStructReq queryStructCmd = new QueryStructReq();
|
||||
queryStructCmd.setModelId(parseInfo.getModelId());
|
||||
queryStructCmd.setNativeQuery(parseInfo.getNativeQuery());
|
||||
queryStructCmd.setQueryType(parseInfo.getQueryType());
|
||||
queryStructCmd.setDateInfo(rewrite2Between(parseInfo.getDateInfo()));
|
||||
|
||||
List<Filter> dimensionFilters = parseInfo.getDimensionFilters().stream()
|
||||
@@ -231,7 +232,7 @@ public class QueryReqBuilder {
|
||||
public static QueryStructReq buildStructRatioReq(SemanticParseInfo parseInfo, SchemaElement metric,
|
||||
AggOperatorEnum aggOperatorEnum) {
|
||||
QueryStructReq queryStructCmd = buildStructReq(parseInfo);
|
||||
queryStructCmd.setNativeQuery(false);
|
||||
queryStructCmd.setQueryType(QueryType.METRIC);
|
||||
queryStructCmd.setOrders(new ArrayList<>());
|
||||
List<Aggregator> aggregators = new ArrayList<>();
|
||||
Aggregator ratioRoll = new Aggregator(metric.getBizName(), aggOperatorEnum);
|
||||
|
||||
@@ -5,6 +5,7 @@ import com.tencent.supersonic.common.pojo.Aggregator;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.pojo.DateConf.DateMode;
|
||||
import com.tencent.supersonic.common.pojo.Order;
|
||||
import com.tencent.supersonic.common.pojo.QueryType;
|
||||
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
|
||||
import com.tencent.supersonic.common.util.ContextUtils;
|
||||
import com.tencent.supersonic.common.util.DateModeUtils;
|
||||
@@ -29,7 +30,7 @@ class QueryReqBuilderTest {
|
||||
init();
|
||||
QueryStructReq queryStructReq = new QueryStructReq();
|
||||
queryStructReq.setModelId(1L);
|
||||
queryStructReq.setNativeQuery(false);
|
||||
queryStructReq.setQueryType(QueryType.METRIC);
|
||||
queryStructReq.setModelName("内容库");
|
||||
|
||||
Aggregator aggregator = new Aggregator();
|
||||
@@ -55,7 +56,7 @@ class QueryReqBuilderTest {
|
||||
"SELECT department, SUM(pv) FROM 内容库 WHERE (sys_imp_date IN ('2023-08-01')) "
|
||||
+ "GROUP BY department ORDER BY uv LIMIT 2000", queryS2SQLReq.getSql());
|
||||
|
||||
queryStructReq.setNativeQuery(true);
|
||||
queryStructReq.setQueryType(QueryType.ENTITY);
|
||||
queryS2SQLReq = queryStructReq.convert(queryStructReq);
|
||||
Assert.assertEquals(
|
||||
"SELECT department, pv FROM 内容库 WHERE (sys_imp_date IN ('2023-08-01')) "
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package com.tencent.supersonic.chat.api.pojo;
|
||||
package com.tencent.supersonic.common.pojo;
|
||||
|
||||
/***
|
||||
* Query Type
|
||||
@@ -15,5 +15,9 @@ public enum QueryType {
|
||||
/**
|
||||
* the other queries
|
||||
*/
|
||||
OTHER
|
||||
OTHER;
|
||||
|
||||
public boolean isNativeAggQuery() {
|
||||
return ENTITY.equals(this);
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import com.tencent.supersonic.chat.query.rule.entity.EntityFilterQuery;
|
||||
import com.tencent.supersonic.chat.query.rule.metric.MetricEntityQuery;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.pojo.DateConf.DateMode;
|
||||
import com.tencent.supersonic.common.pojo.QueryType;
|
||||
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
|
||||
import com.tencent.supersonic.util.DataUtils;
|
||||
import java.util.ArrayList;
|
||||
@@ -36,7 +37,7 @@ public class EntityQueryTest extends BaseQueryTest {
|
||||
expectedParseInfo.getMetrics().add(metric);
|
||||
|
||||
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateMode.BETWEEN, 1, period, startDay, endDay));
|
||||
expectedParseInfo.setNativeQuery(false);
|
||||
expectedParseInfo.setQueryType(QueryType.METRIC);
|
||||
|
||||
assertQueryResult(expectedResult, actualResult);
|
||||
}
|
||||
@@ -71,7 +72,7 @@ public class EntityQueryTest extends BaseQueryTest {
|
||||
expectedParseInfo.getDimensions().add(dim4);
|
||||
|
||||
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.BETWEEN, startDay, startDay));
|
||||
expectedParseInfo.setNativeQuery(true);
|
||||
expectedParseInfo.setQueryType(QueryType.ENTITY);
|
||||
|
||||
assertQueryResult(expectedResult, actualResult);
|
||||
}
|
||||
|
||||
@@ -1,33 +1,34 @@
|
||||
package com.tencent.supersonic.integration;
|
||||
|
||||
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE;
|
||||
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.SUM;
|
||||
|
||||
import com.tencent.supersonic.auth.api.authentication.pojo.User;
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.ItemVisibility;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
|
||||
import com.tencent.supersonic.chat.api.pojo.request.ItemVisibility;
|
||||
import com.tencent.supersonic.chat.query.rule.metric.MetricModelQuery;
|
||||
import com.tencent.supersonic.chat.query.rule.metric.MetricFilterQuery;
|
||||
import com.tencent.supersonic.chat.query.rule.metric.MetricGroupByQuery;
|
||||
import com.tencent.supersonic.chat.query.rule.metric.MetricModelQuery;
|
||||
import com.tencent.supersonic.chat.query.rule.metric.MetricTopNQuery;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.pojo.QueryType;
|
||||
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
|
||||
import com.tencent.supersonic.util.DataUtils;
|
||||
import java.text.DateFormat;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.springframework.beans.BeanUtils;
|
||||
|
||||
import java.text.DateFormat;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.List;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.stream.Collectors;
|
||||
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.SUM;
|
||||
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE;
|
||||
|
||||
|
||||
public class MetricQueryTest extends BaseQueryTest {
|
||||
|
||||
@@ -48,7 +49,7 @@ public class MetricQueryTest extends BaseQueryTest {
|
||||
FilterOperatorEnum.EQUALS, "alice", "用户名", 2L));
|
||||
|
||||
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay));
|
||||
expectedParseInfo.setNativeQuery(false);
|
||||
expectedParseInfo.setQueryType(QueryType.METRIC);
|
||||
|
||||
assertQueryResult(expectedResult, actualResult);
|
||||
}
|
||||
@@ -78,7 +79,7 @@ public class MetricQueryTest extends BaseQueryTest {
|
||||
expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("访问次数"));
|
||||
|
||||
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay));
|
||||
expectedParseInfo.setNativeQuery(false);
|
||||
expectedParseInfo.setQueryType(QueryType.METRIC);
|
||||
|
||||
assertQueryResult(expectedResult, actualResult);
|
||||
}
|
||||
@@ -108,7 +109,7 @@ public class MetricQueryTest extends BaseQueryTest {
|
||||
expectedParseInfo.getDimensions().add(DataUtils.getSchemaElement("部门"));
|
||||
|
||||
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay));
|
||||
expectedParseInfo.setNativeQuery(false);
|
||||
expectedParseInfo.setQueryType(QueryType.METRIC);
|
||||
|
||||
assertQueryResult(expectedResult, actualResult);
|
||||
}
|
||||
@@ -133,7 +134,7 @@ public class MetricQueryTest extends BaseQueryTest {
|
||||
expectedParseInfo.getDimensionFilters().add(dimensionFilter);
|
||||
|
||||
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay));
|
||||
expectedParseInfo.setNativeQuery(false);
|
||||
expectedParseInfo.setQueryType(QueryType.METRIC);
|
||||
|
||||
assertQueryResult(expectedResult, actualResult);
|
||||
}
|
||||
@@ -153,7 +154,7 @@ public class MetricQueryTest extends BaseQueryTest {
|
||||
expectedParseInfo.getDimensions().add(DataUtils.getSchemaElement("用户名"));
|
||||
|
||||
expectedParseInfo.setDateInfo(DataUtils.getDateConf(3, DateConf.DateMode.RECENT, "DAY"));
|
||||
expectedParseInfo.setNativeQuery(false);
|
||||
expectedParseInfo.setQueryType(QueryType.METRIC);
|
||||
|
||||
assertQueryResult(expectedResult, actualResult);
|
||||
}
|
||||
@@ -172,7 +173,7 @@ public class MetricQueryTest extends BaseQueryTest {
|
||||
expectedParseInfo.getDimensions().add(DataUtils.getSchemaElement("部门"));
|
||||
|
||||
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay));
|
||||
expectedParseInfo.setNativeQuery(false);
|
||||
expectedParseInfo.setQueryType(QueryType.METRIC);
|
||||
|
||||
assertQueryResult(expectedResult, actualResult);
|
||||
}
|
||||
@@ -198,7 +199,7 @@ public class MetricQueryTest extends BaseQueryTest {
|
||||
FilterOperatorEnum.EQUALS, "alice", "用户名", 2L));
|
||||
|
||||
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.BETWEEN, 1, period, startDay, startDay));
|
||||
expectedParseInfo.setNativeQuery(false);
|
||||
expectedParseInfo.setQueryType(QueryType.METRIC);
|
||||
|
||||
assertQueryResult(expectedResult, actualResult);
|
||||
}
|
||||
@@ -225,7 +226,7 @@ public class MetricQueryTest extends BaseQueryTest {
|
||||
expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("访问次数"));
|
||||
|
||||
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay));
|
||||
expectedParseInfo.setNativeQuery(false);
|
||||
expectedParseInfo.setQueryType(QueryType.METRIC);
|
||||
|
||||
assertQueryResult(expectedResult, actualResult);
|
||||
|
||||
|
||||
@@ -1,19 +1,19 @@
|
||||
package com.tencent.supersonic.integration;
|
||||
|
||||
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE;
|
||||
|
||||
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
|
||||
import com.tencent.supersonic.chat.query.rule.metric.MetricFilterQuery;
|
||||
import com.tencent.supersonic.chat.query.rule.metric.MetricGroupByQuery;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.pojo.QueryType;
|
||||
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
|
||||
import com.tencent.supersonic.util.DataUtils;
|
||||
import org.junit.Test;
|
||||
import org.junit.jupiter.api.Order;
|
||||
|
||||
import java.text.DateFormat;
|
||||
import java.text.SimpleDateFormat;
|
||||
|
||||
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE;
|
||||
import org.junit.Test;
|
||||
import org.junit.jupiter.api.Order;
|
||||
|
||||
public class MultiTurnsTest extends BaseQueryTest {
|
||||
|
||||
@@ -35,7 +35,7 @@ public class MultiTurnsTest extends BaseQueryTest {
|
||||
FilterOperatorEnum.EQUALS, "alice", "用户名", 2L));
|
||||
|
||||
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay));
|
||||
expectedParseInfo.setNativeQuery(false);
|
||||
expectedParseInfo.setQueryType(QueryType.METRIC);
|
||||
|
||||
assertQueryResult(expectedResult, actualResult);
|
||||
}
|
||||
@@ -58,7 +58,7 @@ public class MultiTurnsTest extends BaseQueryTest {
|
||||
FilterOperatorEnum.EQUALS, "alice", "用户名", 2L));
|
||||
|
||||
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay));
|
||||
expectedParseInfo.setNativeQuery(false);
|
||||
expectedParseInfo.setQueryType(QueryType.METRIC);
|
||||
|
||||
assertQueryResult(expectedResult, actualResult);
|
||||
}
|
||||
@@ -81,7 +81,7 @@ public class MultiTurnsTest extends BaseQueryTest {
|
||||
FilterOperatorEnum.EQUALS, "lucy", "用户名", 2L));
|
||||
|
||||
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay));
|
||||
expectedParseInfo.setNativeQuery(false);
|
||||
expectedParseInfo.setQueryType(QueryType.METRIC);
|
||||
|
||||
assertQueryResult(expectedResult, actualResult);
|
||||
}
|
||||
@@ -102,7 +102,7 @@ public class MultiTurnsTest extends BaseQueryTest {
|
||||
expectedParseInfo.getDimensions().add(DataUtils.getSchemaElement("部门"));
|
||||
|
||||
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay));
|
||||
expectedParseInfo.setNativeQuery(false);
|
||||
expectedParseInfo.setQueryType(QueryType.METRIC);
|
||||
|
||||
assertQueryResult(expectedResult, actualResult);
|
||||
}
|
||||
@@ -125,7 +125,7 @@ public class MultiTurnsTest extends BaseQueryTest {
|
||||
expectedParseInfo.getDimensions().add(DataUtils.getSchemaElement("部门"));
|
||||
|
||||
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.BETWEEN, 1, period, startDay, startDay));
|
||||
expectedParseInfo.setNativeQuery(false);
|
||||
expectedParseInfo.setQueryType(QueryType.METRIC);
|
||||
|
||||
assertQueryResult(expectedResult, actualResult);
|
||||
}
|
||||
@@ -146,7 +146,7 @@ public class MultiTurnsTest extends BaseQueryTest {
|
||||
expectedParseInfo.getDimensions().add(DataUtils.getSchemaElement("部门"));
|
||||
|
||||
expectedParseInfo.setDateInfo(DataUtils.getDateConf(30, DateConf.DateMode.RECENT, "DAY"));
|
||||
expectedParseInfo.setNativeQuery(false);
|
||||
expectedParseInfo.setQueryType(QueryType.METRIC);
|
||||
|
||||
assertQueryResult(expectedResult, actualResult);
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
|
||||
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
|
||||
import com.tencent.supersonic.chat.query.rule.metric.MetricEntityQuery;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.pojo.QueryType;
|
||||
import com.tencent.supersonic.integration.BaseQueryTest;
|
||||
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
|
||||
import com.tencent.supersonic.util.DataUtils;
|
||||
@@ -38,7 +39,7 @@ public class MapperTest extends BaseQueryTest {
|
||||
expectedParseInfo.getMetrics().add(metric);
|
||||
|
||||
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, 7, period, startDay, endDay));
|
||||
expectedParseInfo.setNativeQuery(false);
|
||||
expectedParseInfo.setQueryType(QueryType.METRIC);
|
||||
|
||||
assertQueryResult(expectedResult, actualResult);
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import com.tencent.supersonic.common.pojo.Constants;
|
||||
import com.tencent.supersonic.common.pojo.DateConf;
|
||||
import com.tencent.supersonic.common.pojo.Filter;
|
||||
import com.tencent.supersonic.common.pojo.Order;
|
||||
import com.tencent.supersonic.common.pojo.QueryType;
|
||||
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
|
||||
import com.tencent.supersonic.common.util.ContextUtils;
|
||||
import com.tencent.supersonic.common.util.DateModeUtils;
|
||||
@@ -57,7 +58,7 @@ public class QueryStructReq {
|
||||
private List<Param> params = new ArrayList<>();
|
||||
private DateConf dateInfo;
|
||||
private Long limit = 2000L;
|
||||
private Boolean nativeQuery = false;
|
||||
private QueryType queryType = QueryType.OTHER;
|
||||
private Cache cacheInfo;
|
||||
|
||||
/**
|
||||
@@ -121,8 +122,6 @@ public class QueryStructReq {
|
||||
.append(params);
|
||||
stringBuilder.append(",\"limit\":")
|
||||
.append(limit);
|
||||
stringBuilder.append(",\"nativeQuery\":")
|
||||
.append(nativeQuery);
|
||||
stringBuilder.append('}');
|
||||
return stringBuilder.toString();
|
||||
}
|
||||
@@ -157,8 +156,6 @@ public class QueryStructReq {
|
||||
.append(dateInfo);
|
||||
sb.append(",\"limit\":")
|
||||
.append(limit);
|
||||
sb.append(",\"nativeQuery\":")
|
||||
.append(nativeQuery);
|
||||
sb.append(",\"cacheInfo\":")
|
||||
.append(cacheInfo);
|
||||
sb.append('}');
|
||||
@@ -202,7 +199,7 @@ public class QueryStructReq {
|
||||
if (!CollectionUtils.isEmpty(aggregators)) {
|
||||
for (Aggregator aggregator : aggregators) {
|
||||
String columnName = aggregator.getColumn();
|
||||
if (queryStructReq.getNativeQuery()) {
|
||||
if (queryStructReq.getQueryType().isNativeAggQuery()) {
|
||||
selectItems.add(new SelectExpressionItem(new Column(columnName)));
|
||||
} else {
|
||||
Function sumFunction = new Function();
|
||||
@@ -247,7 +244,7 @@ public class QueryStructReq {
|
||||
}
|
||||
|
||||
//4.Set the group by clause
|
||||
if (!CollectionUtils.isEmpty(groups) && !queryStructReq.getNativeQuery()) {
|
||||
if (!CollectionUtils.isEmpty(groups) && !queryStructReq.getQueryType().isNativeAggQuery()) {
|
||||
GroupByElement groupByElement = new GroupByElement();
|
||||
for (String group : groups) {
|
||||
groupByElement.addGroupByExpression(new Column(group));
|
||||
|
||||
@@ -32,7 +32,7 @@ public class DetailQuery implements QueryOptimizer {
|
||||
}
|
||||
|
||||
public boolean isDetailQuery(QueryStructReq queryStructCmd) {
|
||||
return Objects.nonNull(queryStructCmd) && queryStructCmd.getNativeQuery() && CollectionUtils.isEmpty(
|
||||
queryStructCmd.getMetrics());
|
||||
return Objects.nonNull(queryStructCmd) && queryStructCmd.getQueryType().isNativeAggQuery()
|
||||
&& CollectionUtils.isEmpty(queryStructCmd.getMetrics());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -266,7 +266,8 @@ public class MaterializationQuery implements QueryOptimizer {
|
||||
getTimeRanges(queryStructReq, queryStatement);
|
||||
removeDefaultMetric(queryStructReq, queryStatement.getMetricReq());
|
||||
MaterializationPlanner materializationPlanner = new MaterializationPlanner(schema);
|
||||
materializationPlanner.explain(queryStatement, AggOption.getAggregation(queryStructReq.getNativeQuery()));
|
||||
materializationPlanner.explain(queryStatement,
|
||||
AggOption.getAggregation(queryStructReq.getQueryType().isNativeAggQuery()));
|
||||
log.info("optimize {}", materializationPlanner.findBest().getDatasource());
|
||||
SemanticSchema semanticSchema = materializationPlanner.findBest();
|
||||
if (!CollectionUtils.isEmpty(semanticSchema.getDatasource())) {
|
||||
@@ -278,7 +279,8 @@ public class MaterializationQuery implements QueryOptimizer {
|
||||
doSingleZipperSource(queryStructReq, queryStatement);
|
||||
}
|
||||
AggPlanner aggBuilder = new AggPlanner(semanticSchema);
|
||||
aggBuilder.explain(queryStatement, AggOption.getAggregation(queryStructReq.getNativeQuery()));
|
||||
aggBuilder.explain(queryStatement,
|
||||
AggOption.getAggregation(queryStructReq.getQueryType().isNativeAggQuery()));
|
||||
log.debug("optimize before {} sql {}", queryStatement.getSourceId(), queryStatement.getSql());
|
||||
log.debug("optimize after {} sql {}", aggBuilder.getSourceId(), aggBuilder.getSql());
|
||||
queryStatement.setSourceId(aggBuilder.getSourceId());
|
||||
|
||||
@@ -44,7 +44,8 @@ public class QueryParser {
|
||||
if (!parseSqlReq.getSql().isEmpty()) {
|
||||
return parser(parseSqlReq);
|
||||
}
|
||||
metricReq.setNativeQuery(queryStructReq.getNativeQuery());
|
||||
|
||||
metricReq.setNativeQuery(queryStructReq.getQueryType().isNativeAggQuery());
|
||||
return parser(metricReq);
|
||||
}
|
||||
|
||||
|
||||
@@ -95,7 +95,7 @@ public class CalculateAggConverter implements SemanticConverter {
|
||||
|
||||
@Override
|
||||
public boolean accept(QueryStructReq queryStructCmd) {
|
||||
if (queryStructCmd.getNativeQuery()) {
|
||||
if (queryStructCmd.getQueryType().isNativeAggQuery()) {
|
||||
return false;
|
||||
}
|
||||
if (CollectionUtils.isEmpty(queryStructCmd.getAggregators())) {
|
||||
|
||||
@@ -24,7 +24,7 @@ public class MetricCheckConverter implements SemanticConverter {
|
||||
|
||||
@Override
|
||||
public boolean accept(QueryStructReq queryStructCmd) {
|
||||
if (queryStructCmd.getNativeQuery()) {
|
||||
if (queryStructCmd.getQueryType().isNativeAggQuery()) {
|
||||
return false;
|
||||
}
|
||||
return !CollectionUtils.isEmpty(queryStructCmd.getAggregators());
|
||||
|
||||
@@ -62,7 +62,7 @@ public class ParserDefaultConverter implements SemanticConverter {
|
||||
|
||||
// todo tmp delete
|
||||
// support detail query
|
||||
if (queryStructCmd.getNativeQuery() && CollectionUtils.isEmpty(sqlCommend.getMetrics())) {
|
||||
if (queryStructCmd.getQueryType().isNativeAggQuery() && CollectionUtils.isEmpty(sqlCommend.getMetrics())) {
|
||||
String internalMetricName = queryStructUtils.generateInternalMetricName(
|
||||
queryStructCmd.getModelId(), queryStructCmd.getGroups());
|
||||
sqlCommend.getMetrics().add(internalMetricName);
|
||||
|
||||
@@ -3,6 +3,7 @@ package com.tencent.supersonic.semantic.query.parser.convert;
|
||||
|
||||
import com.tencent.supersonic.common.pojo.Aggregator;
|
||||
import com.tencent.supersonic.common.pojo.Constants;
|
||||
import com.tencent.supersonic.common.pojo.QueryType;
|
||||
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
|
||||
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
|
||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
|
||||
@@ -112,7 +113,7 @@ public class QueryReqConverter {
|
||||
//5.physicalSql by ParseSqlReq
|
||||
queryStructCmd.setDateInfo(queryStructUtils.getDateConfBySql(databaseReq.getSql()));
|
||||
queryStructCmd.setModelId(databaseReq.getModelId());
|
||||
queryStructCmd.setNativeQuery(!AggOption.isAgg(aggOption));
|
||||
queryStructCmd.setQueryType(getQueryType(aggOption));
|
||||
log.info("QueryReqConverter queryStructCmd[{}]", queryStructCmd);
|
||||
QueryStatement queryStatement = parserService.physicalSql(queryStructCmd, result);
|
||||
queryStatement.setSql(String.format(SqlExecuteReq.LIMIT_WRAPPER, queryStatement.getSql()));
|
||||
@@ -146,18 +147,16 @@ public class QueryReqConverter {
|
||||
Map<String, String> internalLowerToNameMap = QueryStructUtils.internalCols.stream()
|
||||
.collect(Collectors.toMap(a -> a.toLowerCase(), a -> a));
|
||||
dimensionLowerToNameMap.putAll(internalLowerToNameMap);
|
||||
Set<String> collect = allFields.stream()
|
||||
return allFields.stream()
|
||||
.filter(entry -> dimensionLowerToNameMap.containsKey(entry.toLowerCase()))
|
||||
.map(entry -> dimensionLowerToNameMap.get(entry.toLowerCase())).collect(Collectors.toSet());
|
||||
return collect;
|
||||
}
|
||||
|
||||
private List<String> getMetrics(ModelSchemaResp modelSchemaResp, List<String> allFields) {
|
||||
Map<String, String> metricLowerToNameMap = modelSchemaResp.getMetrics().stream()
|
||||
.collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(), entry -> entry.getBizName()));
|
||||
List<String> metrics = allFields.stream().filter(entry -> metricLowerToNameMap.containsKey(entry.toLowerCase()))
|
||||
return allFields.stream().filter(entry -> metricLowerToNameMap.containsKey(entry.toLowerCase()))
|
||||
.map(entry -> metricLowerToNameMap.get(entry.toLowerCase())).collect(Collectors.toList());
|
||||
return metrics;
|
||||
}
|
||||
|
||||
private void functionNameCorrector(QueryS2SQLReq databaseReq) {
|
||||
@@ -175,7 +174,6 @@ public class QueryReqConverter {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
protected Map<String, String> getFieldNameToBizNameMap(ModelSchemaResp modelSchemaResp) {
|
||||
// support fieldName and field alias to bizName
|
||||
Map<String, String> dimensionResults = modelSchemaResp.getDimensions().stream()
|
||||
@@ -216,4 +214,13 @@ public class QueryReqConverter {
|
||||
databaseReq.setSql(sql);
|
||||
}
|
||||
|
||||
private QueryType getQueryType(AggOption aggOption) {
|
||||
boolean isAgg = AggOption.isAgg(aggOption);
|
||||
QueryType queryType = QueryType.ENTITY;
|
||||
if (isAgg) {
|
||||
queryType = QueryType.METRIC;
|
||||
}
|
||||
return queryType;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -144,7 +144,7 @@ public class StatUtils {
|
||||
.setQueryStructCmd(queryStructCmd.toString())
|
||||
.setQueryStructCmdMd5(DigestUtils.md5Hex(queryStructCmd.toString()))
|
||||
.setStartTime(System.currentTimeMillis())
|
||||
.setNativeQuery(queryStructCmd.getNativeQuery())
|
||||
.setNativeQuery(queryStructCmd.getQueryType().isNativeAggQuery())
|
||||
.setGroupByCols(objectMapper.writeValueAsString(queryStructCmd.getGroups()))
|
||||
.setAggCols(objectMapper.writeValueAsString(queryStructCmd.getAggregators()))
|
||||
.setOrderByCols(objectMapper.writeValueAsString(queryStructCmd.getOrders()))
|
||||
|
||||
Reference in New Issue
Block a user