[improvement](chat) remove nativeQuery config in chat (#394)

This commit is contained in:
lexluo09
2023-11-16 21:51:08 +08:00
committed by GitHub
parent 11cdcb29fa
commit 05b1a7ec3b
25 changed files with 99 additions and 117 deletions

View File

@@ -6,6 +6,7 @@ import com.tencent.supersonic.chat.api.pojo.response.EntityInfo;
import com.tencent.supersonic.chat.api.pojo.response.SqlInfo;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.QueryType;
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
import java.util.ArrayList;
import java.util.Comparator;
@@ -34,7 +35,6 @@ public class SemanticParseInfo {
private Set<Order> orders = new LinkedHashSet();
private DateConf dateInfo;
private Long limit;
private Boolean nativeQuery = false;
private double score;
private List<SchemaElementMatch> elementMatches = new ArrayList<>();
private Map<String, Object> properties = new HashMap<>();

View File

@@ -5,7 +5,7 @@ import com.tencent.supersonic.chat.api.component.SemanticParser;
import com.tencent.supersonic.chat.api.component.SemanticQuery;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.chat.api.pojo.QueryType;
import com.tencent.supersonic.common.pojo.QueryType;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
@@ -26,7 +26,7 @@ import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
/**
* Query type parser, determine if the query is a metric query, a entity query,
* Query type parser, determine if the query is a metric query, an entity query,
* or another type of query.
*/
@Slf4j

View File

@@ -17,6 +17,7 @@ import com.tencent.supersonic.chat.utils.ComponentFactory;
import com.tencent.supersonic.chat.utils.QueryReqBuilder;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.QueryType;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.semantic.api.model.response.QueryResultWithSchemaResp;
@@ -89,7 +90,7 @@ public class MetricInterpretQuery extends LLMSemanticQuery {
protected QueryStructReq convertQueryStruct() {
QueryStructReq queryStructReq = QueryReqBuilder.buildStructReq(parseInfo);
fillAggregator(queryStructReq, parseInfo.getMetrics());
queryStructReq.setNativeQuery(true);
queryStructReq.setQueryType(QueryType.ENTITY);
return queryStructReq;
}

View File

@@ -1,7 +1,12 @@
package com.tencent.supersonic.chat.query.rule.entity;
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ENTITY;
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.REQUIRED;
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
import com.tencent.supersonic.chat.api.pojo.ChatContext;
import com.tencent.supersonic.chat.api.pojo.QueryContext;
import com.tencent.supersonic.common.pojo.QueryType;
import com.tencent.supersonic.chat.api.pojo.SchemaElementMatch;
import com.tencent.supersonic.chat.api.pojo.SchemaElementType;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
@@ -11,17 +16,12 @@ import com.tencent.supersonic.chat.query.rule.RuleSemanticQuery;
import com.tencent.supersonic.chat.service.ConfigService;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.util.ContextUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import static com.tencent.supersonic.chat.api.pojo.SchemaElementType.ENTITY;
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.RequireNumberType.AT_LEAST;
import static com.tencent.supersonic.chat.query.rule.QueryMatchOption.OptionType.REQUIRED;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
@Slf4j
public abstract class EntitySemanticQuery extends RuleSemanticQuery {
@@ -81,7 +81,7 @@ public abstract class EntitySemanticQuery extends RuleSemanticQuery {
public void fillParseInfo(Long modelId, QueryContext queryContext, ChatContext chatContext) {
super.fillParseInfo(modelId, queryContext, chatContext);
parseInfo.setNativeQuery(true);
parseInfo.setQueryType(QueryType.ENTITY);
parseInfo.setLimit(ENTITY_MAX_RESULTS);
if (parseInfo.getDateInfo() == null) {
ConfigService configService = ContextUtils.getBean(ConfigService.class);

View File

@@ -9,12 +9,6 @@ import com.tencent.supersonic.chat.query.QueryManager;
import com.tencent.supersonic.chat.query.llm.interpret.MetricInterpretQuery;
import com.tencent.supersonic.chat.service.SemanticService;
import com.tencent.supersonic.common.util.ContextUtils;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
public class EntityInfoExecuteResponder implements ExecuteResponder {
@@ -33,29 +27,6 @@ public class EntityInfoExecuteResponder implements ExecuteResponder {
EntityInfo entityInfo = semanticService.getEntityInfo(semanticParseInfo, user);
queryResult.setEntityInfo(entityInfo);
String primaryEntityBizName = semanticService.getPrimaryEntityBizName(entityInfo);
if (StringUtils.isEmpty(primaryEntityBizName)
|| CollectionUtils.isEmpty(queryResult.getQueryColumns())) {
return;
}
boolean existPrimaryEntityName = queryResult.getQueryColumns().stream()
.anyMatch(queryColumn -> primaryEntityBizName.equals(queryColumn.getNameEn()));
semanticParseInfo.setNativeQuery(existPrimaryEntityName);
if (!existPrimaryEntityName) {
return;
}
List<Map<String, Object>> queryResults = queryResult.getQueryResults();
List<String> entities = queryResults.stream()
.map(entry -> entry.get(primaryEntityBizName))
.filter(Objects::nonNull)
.map(String::valueOf)
.collect(Collectors.toList());
if (CollectionUtils.isEmpty(entities)) {
return;
}
}
}

View File

@@ -11,7 +11,6 @@ import com.tencent.supersonic.chat.query.llm.interpret.MetricInterpretQuery;
import com.tencent.supersonic.chat.service.SemanticService;
import com.tencent.supersonic.common.util.ContextUtils;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
public class EntityInfoParseResponder implements ParseResponder {
@@ -37,14 +36,6 @@ public class EntityInfoParseResponder implements ParseResponder {
|| QueryManager.isMetricQuery(queryMode)) {
parseInfo.setEntityInfo(entityInfo);
}
//2. set native value
String primaryEntityBizName = semanticService.getPrimaryEntityBizName(entityInfo);
if (StringUtils.isNotEmpty(primaryEntityBizName)) {
//if exist primaryEntityBizName in parseInfo's dimensions, set nativeQuery to true
boolean existPrimaryEntityBizName = parseInfo.getDimensions().stream()
.anyMatch(schemaElement -> primaryEntityBizName.equalsIgnoreCase(schemaElement.getBizName()));
parseInfo.setNativeQuery(existPrimaryEntityBizName);
}
});
}
}

View File

@@ -14,6 +14,7 @@ import static com.tencent.supersonic.common.pojo.Constants.WEEK;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.component.SemanticInterpreter;
import com.tencent.supersonic.chat.api.pojo.ModelSchema;
import com.tencent.supersonic.common.pojo.QueryType;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.request.ChatAggConfigReq;
@@ -233,7 +234,7 @@ public class SemanticService {
modelInfo.setEntityId(entities.get(0));
SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
semanticParseInfo.setModel(modelSchema.getModel());
semanticParseInfo.setNativeQuery(true);
semanticParseInfo.setQueryType(QueryType.ENTITY);
semanticParseInfo.setMetrics(getMetrics(modelInfo));
semanticParseInfo.setDimensions(getDimensions(modelInfo));
DateConf dateInfo = new DateConf();

View File

@@ -2,6 +2,7 @@
package com.tencent.supersonic.chat.service.impl;
import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.QueryType;
import com.tencent.supersonic.chat.api.pojo.SchemaElement;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.SemanticSchema;
@@ -79,13 +80,13 @@ public class ParserInfoServiceImpl implements ParseInfoService {
parseInfo.setMetrics(metrics);
if (SqlParserSelectFunctionHelper.hasAggregateFunction(sqlInfo.getCorrectS2SQL())) {
parseInfo.setNativeQuery(false);
parseInfo.setQueryType(QueryType.METRIC);
List<String> groupByFields = SqlParserSelectHelper.getGroupByFields(sqlInfo.getCorrectS2SQL());
List<String> groupByDimensions = getFieldsExceptDate(groupByFields);
parseInfo.setDimensions(
getElements(parseInfo.getModelId(), groupByDimensions, semanticSchema.getDimensions()));
} else {
parseInfo.setNativeQuery(true);
parseInfo.setQueryType(QueryType.ENTITY);
List<String> selectFields = SqlParserSelectHelper.getSelectFields(sqlInfo.getCorrectS2SQL());
List<String> selectDimensions = getFieldsExceptDate(selectFields);
parseInfo.setDimensions(

View File

@@ -42,6 +42,7 @@ import com.tencent.supersonic.chat.utils.ComponentFactory;
import com.tencent.supersonic.chat.utils.SolvedQueryManager;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.QueryColumn;
import com.tencent.supersonic.common.pojo.QueryType;
import com.tencent.supersonic.common.pojo.enums.DictWordType;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
@@ -609,7 +610,7 @@ public class QueryServiceImpl implements QueryService {
queryStructReq.setDateInfo(dateConf);
queryStructReq.setLimit(20L);
queryStructReq.setModelId(dimensionValueReq.getModelId());
queryStructReq.setNativeQuery(false);
queryStructReq.setQueryType(QueryType.OTHER);
List<String> groups = new ArrayList<>();
groups.add(dimensionValueReq.getBizName());
queryStructReq.setGroups(groups);

View File

@@ -9,6 +9,7 @@ import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.QueryType;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
@@ -36,7 +37,7 @@ public class QueryReqBuilder {
public static QueryStructReq buildStructReq(SemanticParseInfo parseInfo) {
QueryStructReq queryStructCmd = new QueryStructReq();
queryStructCmd.setModelId(parseInfo.getModelId());
queryStructCmd.setNativeQuery(parseInfo.getNativeQuery());
queryStructCmd.setQueryType(parseInfo.getQueryType());
queryStructCmd.setDateInfo(rewrite2Between(parseInfo.getDateInfo()));
List<Filter> dimensionFilters = parseInfo.getDimensionFilters().stream()
@@ -231,7 +232,7 @@ public class QueryReqBuilder {
public static QueryStructReq buildStructRatioReq(SemanticParseInfo parseInfo, SchemaElement metric,
AggOperatorEnum aggOperatorEnum) {
QueryStructReq queryStructCmd = buildStructReq(parseInfo);
queryStructCmd.setNativeQuery(false);
queryStructCmd.setQueryType(QueryType.METRIC);
queryStructCmd.setOrders(new ArrayList<>());
List<Aggregator> aggregators = new ArrayList<>();
Aggregator ratioRoll = new Aggregator(metric.getBizName(), aggOperatorEnum);

View File

@@ -5,6 +5,7 @@ import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.DateConf.DateMode;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.QueryType;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateModeUtils;
@@ -29,7 +30,7 @@ class QueryReqBuilderTest {
init();
QueryStructReq queryStructReq = new QueryStructReq();
queryStructReq.setModelId(1L);
queryStructReq.setNativeQuery(false);
queryStructReq.setQueryType(QueryType.METRIC);
queryStructReq.setModelName("内容库");
Aggregator aggregator = new Aggregator();
@@ -55,7 +56,7 @@ class QueryReqBuilderTest {
"SELECT department, SUM(pv) FROM 内容库 WHERE (sys_imp_date IN ('2023-08-01')) "
+ "GROUP BY department ORDER BY uv LIMIT 2000", queryS2SQLReq.getSql());
queryStructReq.setNativeQuery(true);
queryStructReq.setQueryType(QueryType.ENTITY);
queryS2SQLReq = queryStructReq.convert(queryStructReq);
Assert.assertEquals(
"SELECT department, pv FROM 内容库 WHERE (sys_imp_date IN ('2023-08-01')) "

View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.chat.api.pojo;
package com.tencent.supersonic.common.pojo;
/***
* Query Type
@@ -15,5 +15,9 @@ public enum QueryType {
/**
* the other queries
*/
OTHER
OTHER;
public boolean isNativeAggQuery() {
return ENTITY.equals(this);
}
}

View File

@@ -10,6 +10,7 @@ import com.tencent.supersonic.chat.query.rule.entity.EntityFilterQuery;
import com.tencent.supersonic.chat.query.rule.metric.MetricEntityQuery;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.DateConf.DateMode;
import com.tencent.supersonic.common.pojo.QueryType;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.util.DataUtils;
import java.util.ArrayList;
@@ -36,7 +37,7 @@ public class EntityQueryTest extends BaseQueryTest {
expectedParseInfo.getMetrics().add(metric);
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateMode.BETWEEN, 1, period, startDay, endDay));
expectedParseInfo.setNativeQuery(false);
expectedParseInfo.setQueryType(QueryType.METRIC);
assertQueryResult(expectedResult, actualResult);
}
@@ -71,7 +72,7 @@ public class EntityQueryTest extends BaseQueryTest {
expectedParseInfo.getDimensions().add(dim4);
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.BETWEEN, startDay, startDay));
expectedParseInfo.setNativeQuery(true);
expectedParseInfo.setQueryType(QueryType.ENTITY);
assertQueryResult(expectedResult, actualResult);
}

View File

@@ -1,33 +1,34 @@
package com.tencent.supersonic.integration;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.SUM;
import com.tencent.supersonic.auth.api.authentication.pojo.User;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq;
import com.tencent.supersonic.chat.api.pojo.request.ItemVisibility;
import com.tencent.supersonic.chat.api.pojo.request.QueryFilter;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
import com.tencent.supersonic.chat.api.pojo.response.ParseResp;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.api.pojo.request.ChatConfigEditReqReq;
import com.tencent.supersonic.chat.api.pojo.response.ChatConfigResp;
import com.tencent.supersonic.chat.api.pojo.request.ItemVisibility;
import com.tencent.supersonic.chat.query.rule.metric.MetricModelQuery;
import com.tencent.supersonic.chat.query.rule.metric.MetricFilterQuery;
import com.tencent.supersonic.chat.query.rule.metric.MetricGroupByQuery;
import com.tencent.supersonic.chat.query.rule.metric.MetricModelQuery;
import com.tencent.supersonic.chat.query.rule.metric.MetricTopNQuery;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.QueryType;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.util.DataUtils;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
import org.junit.Assert;
import org.junit.Test;
import org.springframework.beans.BeanUtils;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.List;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.stream.Collectors;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.SUM;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE;
public class MetricQueryTest extends BaseQueryTest {
@@ -48,7 +49,7 @@ public class MetricQueryTest extends BaseQueryTest {
FilterOperatorEnum.EQUALS, "alice", "用户名", 2L));
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay));
expectedParseInfo.setNativeQuery(false);
expectedParseInfo.setQueryType(QueryType.METRIC);
assertQueryResult(expectedResult, actualResult);
}
@@ -78,7 +79,7 @@ public class MetricQueryTest extends BaseQueryTest {
expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("访问次数"));
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay));
expectedParseInfo.setNativeQuery(false);
expectedParseInfo.setQueryType(QueryType.METRIC);
assertQueryResult(expectedResult, actualResult);
}
@@ -108,7 +109,7 @@ public class MetricQueryTest extends BaseQueryTest {
expectedParseInfo.getDimensions().add(DataUtils.getSchemaElement("部门"));
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay));
expectedParseInfo.setNativeQuery(false);
expectedParseInfo.setQueryType(QueryType.METRIC);
assertQueryResult(expectedResult, actualResult);
}
@@ -133,7 +134,7 @@ public class MetricQueryTest extends BaseQueryTest {
expectedParseInfo.getDimensionFilters().add(dimensionFilter);
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay));
expectedParseInfo.setNativeQuery(false);
expectedParseInfo.setQueryType(QueryType.METRIC);
assertQueryResult(expectedResult, actualResult);
}
@@ -153,7 +154,7 @@ public class MetricQueryTest extends BaseQueryTest {
expectedParseInfo.getDimensions().add(DataUtils.getSchemaElement("用户名"));
expectedParseInfo.setDateInfo(DataUtils.getDateConf(3, DateConf.DateMode.RECENT, "DAY"));
expectedParseInfo.setNativeQuery(false);
expectedParseInfo.setQueryType(QueryType.METRIC);
assertQueryResult(expectedResult, actualResult);
}
@@ -172,7 +173,7 @@ public class MetricQueryTest extends BaseQueryTest {
expectedParseInfo.getDimensions().add(DataUtils.getSchemaElement("部门"));
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay));
expectedParseInfo.setNativeQuery(false);
expectedParseInfo.setQueryType(QueryType.METRIC);
assertQueryResult(expectedResult, actualResult);
}
@@ -198,7 +199,7 @@ public class MetricQueryTest extends BaseQueryTest {
FilterOperatorEnum.EQUALS, "alice", "用户名", 2L));
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.BETWEEN, 1, period, startDay, startDay));
expectedParseInfo.setNativeQuery(false);
expectedParseInfo.setQueryType(QueryType.METRIC);
assertQueryResult(expectedResult, actualResult);
}
@@ -225,7 +226,7 @@ public class MetricQueryTest extends BaseQueryTest {
expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("访问次数"));
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay));
expectedParseInfo.setNativeQuery(false);
expectedParseInfo.setQueryType(QueryType.METRIC);
assertQueryResult(expectedResult, actualResult);

View File

@@ -1,19 +1,19 @@
package com.tencent.supersonic.integration;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE;
import com.tencent.supersonic.chat.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.query.rule.metric.MetricFilterQuery;
import com.tencent.supersonic.chat.query.rule.metric.MetricGroupByQuery;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.QueryType;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.util.DataUtils;
import org.junit.Test;
import org.junit.jupiter.api.Order;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.NONE;
import org.junit.Test;
import org.junit.jupiter.api.Order;
public class MultiTurnsTest extends BaseQueryTest {
@@ -35,7 +35,7 @@ public class MultiTurnsTest extends BaseQueryTest {
FilterOperatorEnum.EQUALS, "alice", "用户名", 2L));
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay));
expectedParseInfo.setNativeQuery(false);
expectedParseInfo.setQueryType(QueryType.METRIC);
assertQueryResult(expectedResult, actualResult);
}
@@ -58,7 +58,7 @@ public class MultiTurnsTest extends BaseQueryTest {
FilterOperatorEnum.EQUALS, "alice", "用户名", 2L));
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay));
expectedParseInfo.setNativeQuery(false);
expectedParseInfo.setQueryType(QueryType.METRIC);
assertQueryResult(expectedResult, actualResult);
}
@@ -81,7 +81,7 @@ public class MultiTurnsTest extends BaseQueryTest {
FilterOperatorEnum.EQUALS, "lucy", "用户名", 2L));
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay));
expectedParseInfo.setNativeQuery(false);
expectedParseInfo.setQueryType(QueryType.METRIC);
assertQueryResult(expectedResult, actualResult);
}
@@ -102,7 +102,7 @@ public class MultiTurnsTest extends BaseQueryTest {
expectedParseInfo.getDimensions().add(DataUtils.getSchemaElement("部门"));
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, unit, period, startDay, endDay));
expectedParseInfo.setNativeQuery(false);
expectedParseInfo.setQueryType(QueryType.METRIC);
assertQueryResult(expectedResult, actualResult);
}
@@ -125,7 +125,7 @@ public class MultiTurnsTest extends BaseQueryTest {
expectedParseInfo.getDimensions().add(DataUtils.getSchemaElement("部门"));
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.BETWEEN, 1, period, startDay, startDay));
expectedParseInfo.setNativeQuery(false);
expectedParseInfo.setQueryType(QueryType.METRIC);
assertQueryResult(expectedResult, actualResult);
}
@@ -146,7 +146,7 @@ public class MultiTurnsTest extends BaseQueryTest {
expectedParseInfo.getDimensions().add(DataUtils.getSchemaElement("部门"));
expectedParseInfo.setDateInfo(DataUtils.getDateConf(30, DateConf.DateMode.RECENT, "DAY"));
expectedParseInfo.setNativeQuery(false);
expectedParseInfo.setQueryType(QueryType.METRIC);
assertQueryResult(expectedResult, actualResult);
}

View File

@@ -9,6 +9,7 @@ import com.tencent.supersonic.chat.api.pojo.request.QueryReq;
import com.tencent.supersonic.chat.api.pojo.response.QueryResult;
import com.tencent.supersonic.chat.query.rule.metric.MetricEntityQuery;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.QueryType;
import com.tencent.supersonic.integration.BaseQueryTest;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.util.DataUtils;
@@ -38,7 +39,7 @@ public class MapperTest extends BaseQueryTest {
expectedParseInfo.getMetrics().add(metric);
expectedParseInfo.setDateInfo(DataUtils.getDateConf(DateConf.DateMode.RECENT, 7, period, startDay, endDay));
expectedParseInfo.setNativeQuery(false);
expectedParseInfo.setQueryType(QueryType.METRIC);
assertQueryResult(expectedResult, actualResult);
}

View File

@@ -6,6 +6,7 @@ import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.QueryType;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateModeUtils;
@@ -57,7 +58,7 @@ public class QueryStructReq {
private List<Param> params = new ArrayList<>();
private DateConf dateInfo;
private Long limit = 2000L;
private Boolean nativeQuery = false;
private QueryType queryType = QueryType.OTHER;
private Cache cacheInfo;
/**
@@ -121,8 +122,6 @@ public class QueryStructReq {
.append(params);
stringBuilder.append(",\"limit\":")
.append(limit);
stringBuilder.append(",\"nativeQuery\":")
.append(nativeQuery);
stringBuilder.append('}');
return stringBuilder.toString();
}
@@ -157,8 +156,6 @@ public class QueryStructReq {
.append(dateInfo);
sb.append(",\"limit\":")
.append(limit);
sb.append(",\"nativeQuery\":")
.append(nativeQuery);
sb.append(",\"cacheInfo\":")
.append(cacheInfo);
sb.append('}');
@@ -202,7 +199,7 @@ public class QueryStructReq {
if (!CollectionUtils.isEmpty(aggregators)) {
for (Aggregator aggregator : aggregators) {
String columnName = aggregator.getColumn();
if (queryStructReq.getNativeQuery()) {
if (queryStructReq.getQueryType().isNativeAggQuery()) {
selectItems.add(new SelectExpressionItem(new Column(columnName)));
} else {
Function sumFunction = new Function();
@@ -247,7 +244,7 @@ public class QueryStructReq {
}
//4.Set the group by clause
if (!CollectionUtils.isEmpty(groups) && !queryStructReq.getNativeQuery()) {
if (!CollectionUtils.isEmpty(groups) && !queryStructReq.getQueryType().isNativeAggQuery()) {
GroupByElement groupByElement = new GroupByElement();
for (String group : groups) {
groupByElement.addGroupByExpression(new Column(group));

View File

@@ -32,7 +32,7 @@ public class DetailQuery implements QueryOptimizer {
}
public boolean isDetailQuery(QueryStructReq queryStructCmd) {
return Objects.nonNull(queryStructCmd) && queryStructCmd.getNativeQuery() && CollectionUtils.isEmpty(
queryStructCmd.getMetrics());
return Objects.nonNull(queryStructCmd) && queryStructCmd.getQueryType().isNativeAggQuery()
&& CollectionUtils.isEmpty(queryStructCmd.getMetrics());
}
}

View File

@@ -266,7 +266,8 @@ public class MaterializationQuery implements QueryOptimizer {
getTimeRanges(queryStructReq, queryStatement);
removeDefaultMetric(queryStructReq, queryStatement.getMetricReq());
MaterializationPlanner materializationPlanner = new MaterializationPlanner(schema);
materializationPlanner.explain(queryStatement, AggOption.getAggregation(queryStructReq.getNativeQuery()));
materializationPlanner.explain(queryStatement,
AggOption.getAggregation(queryStructReq.getQueryType().isNativeAggQuery()));
log.info("optimize {}", materializationPlanner.findBest().getDatasource());
SemanticSchema semanticSchema = materializationPlanner.findBest();
if (!CollectionUtils.isEmpty(semanticSchema.getDatasource())) {
@@ -278,7 +279,8 @@ public class MaterializationQuery implements QueryOptimizer {
doSingleZipperSource(queryStructReq, queryStatement);
}
AggPlanner aggBuilder = new AggPlanner(semanticSchema);
aggBuilder.explain(queryStatement, AggOption.getAggregation(queryStructReq.getNativeQuery()));
aggBuilder.explain(queryStatement,
AggOption.getAggregation(queryStructReq.getQueryType().isNativeAggQuery()));
log.debug("optimize before {} sql {}", queryStatement.getSourceId(), queryStatement.getSql());
log.debug("optimize after {} sql {}", aggBuilder.getSourceId(), aggBuilder.getSql());
queryStatement.setSourceId(aggBuilder.getSourceId());

View File

@@ -44,7 +44,8 @@ public class QueryParser {
if (!parseSqlReq.getSql().isEmpty()) {
return parser(parseSqlReq);
}
metricReq.setNativeQuery(queryStructReq.getNativeQuery());
metricReq.setNativeQuery(queryStructReq.getQueryType().isNativeAggQuery());
return parser(metricReq);
}

View File

@@ -95,7 +95,7 @@ public class CalculateAggConverter implements SemanticConverter {
@Override
public boolean accept(QueryStructReq queryStructCmd) {
if (queryStructCmd.getNativeQuery()) {
if (queryStructCmd.getQueryType().isNativeAggQuery()) {
return false;
}
if (CollectionUtils.isEmpty(queryStructCmd.getAggregators())) {

View File

@@ -24,7 +24,7 @@ public class MetricCheckConverter implements SemanticConverter {
@Override
public boolean accept(QueryStructReq queryStructCmd) {
if (queryStructCmd.getNativeQuery()) {
if (queryStructCmd.getQueryType().isNativeAggQuery()) {
return false;
}
return !CollectionUtils.isEmpty(queryStructCmd.getAggregators());

View File

@@ -62,7 +62,7 @@ public class ParserDefaultConverter implements SemanticConverter {
// todo tmp delete
// support detail query
if (queryStructCmd.getNativeQuery() && CollectionUtils.isEmpty(sqlCommend.getMetrics())) {
if (queryStructCmd.getQueryType().isNativeAggQuery() && CollectionUtils.isEmpty(sqlCommend.getMetrics())) {
String internalMetricName = queryStructUtils.generateInternalMetricName(
queryStructCmd.getModelId(), queryStructCmd.getGroups());
sqlCommend.getMetrics().add(internalMetricName);

View File

@@ -3,6 +3,7 @@ package com.tencent.supersonic.semantic.query.parser.convert;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.QueryType;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
@@ -112,7 +113,7 @@ public class QueryReqConverter {
//5.physicalSql by ParseSqlReq
queryStructCmd.setDateInfo(queryStructUtils.getDateConfBySql(databaseReq.getSql()));
queryStructCmd.setModelId(databaseReq.getModelId());
queryStructCmd.setNativeQuery(!AggOption.isAgg(aggOption));
queryStructCmd.setQueryType(getQueryType(aggOption));
log.info("QueryReqConverter queryStructCmd[{}]", queryStructCmd);
QueryStatement queryStatement = parserService.physicalSql(queryStructCmd, result);
queryStatement.setSql(String.format(SqlExecuteReq.LIMIT_WRAPPER, queryStatement.getSql()));
@@ -146,18 +147,16 @@ public class QueryReqConverter {
Map<String, String> internalLowerToNameMap = QueryStructUtils.internalCols.stream()
.collect(Collectors.toMap(a -> a.toLowerCase(), a -> a));
dimensionLowerToNameMap.putAll(internalLowerToNameMap);
Set<String> collect = allFields.stream()
return allFields.stream()
.filter(entry -> dimensionLowerToNameMap.containsKey(entry.toLowerCase()))
.map(entry -> dimensionLowerToNameMap.get(entry.toLowerCase())).collect(Collectors.toSet());
return collect;
}
private List<String> getMetrics(ModelSchemaResp modelSchemaResp, List<String> allFields) {
Map<String, String> metricLowerToNameMap = modelSchemaResp.getMetrics().stream()
.collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(), entry -> entry.getBizName()));
List<String> metrics = allFields.stream().filter(entry -> metricLowerToNameMap.containsKey(entry.toLowerCase()))
return allFields.stream().filter(entry -> metricLowerToNameMap.containsKey(entry.toLowerCase()))
.map(entry -> metricLowerToNameMap.get(entry.toLowerCase())).collect(Collectors.toList());
return metrics;
}
private void functionNameCorrector(QueryS2SQLReq databaseReq) {
@@ -175,7 +174,6 @@ public class QueryReqConverter {
}
}
protected Map<String, String> getFieldNameToBizNameMap(ModelSchemaResp modelSchemaResp) {
// support fieldName and field alias to bizName
Map<String, String> dimensionResults = modelSchemaResp.getDimensions().stream()
@@ -216,4 +214,13 @@ public class QueryReqConverter {
databaseReq.setSql(sql);
}
private QueryType getQueryType(AggOption aggOption) {
boolean isAgg = AggOption.isAgg(aggOption);
QueryType queryType = QueryType.ENTITY;
if (isAgg) {
queryType = QueryType.METRIC;
}
return queryType;
}
}

View File

@@ -144,7 +144,7 @@ public class StatUtils {
.setQueryStructCmd(queryStructCmd.toString())
.setQueryStructCmdMd5(DigestUtils.md5Hex(queryStructCmd.toString()))
.setStartTime(System.currentTimeMillis())
.setNativeQuery(queryStructCmd.getNativeQuery())
.setNativeQuery(queryStructCmd.getQueryType().isNativeAggQuery())
.setGroupByCols(objectMapper.writeValueAsString(queryStructCmd.getGroups()))
.setAggCols(objectMapper.writeValueAsString(queryStructCmd.getAggregators()))
.setOrderByCols(objectMapper.writeValueAsString(queryStructCmd.getOrders()))