topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-11 03:58:14 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement](chat) remove nativeQuery config in chat (#394)

Browse Source
This commit is contained in:
lexluo09
2023-11-16 21:51:08 +08:00
committed by GitHub
parent 11cdcb29fa
commit 05b1a7ec3b
25 changed files with 99 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
semantic/api/src/main/java/com/tencent/supersonic/semantic/api/query/request/QueryStructReq.java
View File

@@ -6,6 +6,7 @@ import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.DateConf;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.Order;
import com.tencent.supersonic.common.pojo.QueryType;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.util.DateModeUtils;
@@ -57,7 +58,7 @@ public class QueryStructReq {
private List<Param> params = new ArrayList<>();
private DateConf dateInfo;
private Long limit = 2000L;
private Boolean nativeQuery = false;
private QueryType queryType = QueryType.OTHER;
private Cache cacheInfo;
/**
@@ -121,8 +122,6 @@ public class QueryStructReq {
.append(params);
stringBuilder.append(",\"limit\":")
.append(limit);
stringBuilder.append(",\"nativeQuery\":")
.append(nativeQuery);
stringBuilder.append('}');
return stringBuilder.toString();
}
@@ -157,8 +156,6 @@ public class QueryStructReq {
.append(dateInfo);
sb.append(",\"limit\":")
.append(limit);
sb.append(",\"nativeQuery\":")
.append(nativeQuery);
sb.append(",\"cacheInfo\":")
.append(cacheInfo);
sb.append('}');
@@ -202,7 +199,7 @@ public class QueryStructReq {
if (!CollectionUtils.isEmpty(aggregators)) {
for (Aggregator aggregator : aggregators) {
String columnName = aggregator.getColumn();
if (queryStructReq.getNativeQuery()) {
if (queryStructReq.getQueryType().isNativeAggQuery()) {
selectItems.add(new SelectExpressionItem(new Column(columnName)));
} else {
Function sumFunction = new Function();
@@ -247,7 +244,7 @@ public class QueryStructReq {
}
//4.Set the group by clause
if (!CollectionUtils.isEmpty(groups) && !queryStructReq.getNativeQuery()) {
if (!CollectionUtils.isEmpty(groups) && !queryStructReq.getQueryType().isNativeAggQuery()) {
GroupByElement groupByElement = new GroupByElement();
for (String group : groups) {
groupByElement.addGroupByExpression(new Column(group));

4
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/optimizer/DetailQuery.java
View File

@@ -32,7 +32,7 @@ public class DetailQuery implements QueryOptimizer {
}
public boolean isDetailQuery(QueryStructReq queryStructCmd) {
return Objects.nonNull(queryStructCmd) && queryStructCmd.getNativeQuery() && CollectionUtils.isEmpty(
queryStructCmd.getMetrics());
return Objects.nonNull(queryStructCmd) && queryStructCmd.getQueryType().isNativeAggQuery()
&& CollectionUtils.isEmpty(queryStructCmd.getMetrics());
}
}

6
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/optimizer/MaterializationQuery.java
View File

@@ -266,7 +266,8 @@ public class MaterializationQuery implements QueryOptimizer {
getTimeRanges(queryStructReq, queryStatement);
removeDefaultMetric(queryStructReq, queryStatement.getMetricReq());
MaterializationPlanner materializationPlanner = new MaterializationPlanner(schema);
materializationPlanner.explain(queryStatement, AggOption.getAggregation(queryStructReq.getNativeQuery()));
materializationPlanner.explain(queryStatement,
AggOption.getAggregation(queryStructReq.getQueryType().isNativeAggQuery()));
log.info("optimize {}", materializationPlanner.findBest().getDatasource());
SemanticSchema semanticSchema = materializationPlanner.findBest();
if (!CollectionUtils.isEmpty(semanticSchema.getDatasource())) {
@@ -278,7 +279,8 @@ public class MaterializationQuery implements QueryOptimizer {
doSingleZipperSource(queryStructReq, queryStatement);
}
AggPlanner aggBuilder = new AggPlanner(semanticSchema);
aggBuilder.explain(queryStatement, AggOption.getAggregation(queryStructReq.getNativeQuery()));
aggBuilder.explain(queryStatement,
AggOption.getAggregation(queryStructReq.getQueryType().isNativeAggQuery()));
log.debug("optimize before {} sql {}", queryStatement.getSourceId(), queryStatement.getSql());
log.debug("optimize after {} sql {}", aggBuilder.getSourceId(), aggBuilder.getSql());
queryStatement.setSourceId(aggBuilder.getSourceId());

3
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/QueryParser.java
View File

@@ -44,7 +44,8 @@ public class QueryParser {
if (!parseSqlReq.getSql().isEmpty()) {
return parser(parseSqlReq);
}
metricReq.setNativeQuery(queryStructReq.getNativeQuery());
metricReq.setNativeQuery(queryStructReq.getQueryType().isNativeAggQuery());
return parser(metricReq);
}

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/CalculateAggConverter.java
View File

@@ -95,7 +95,7 @@ public class CalculateAggConverter implements SemanticConverter {
@Override
public boolean accept(QueryStructReq queryStructCmd) {
if (queryStructCmd.getNativeQuery()) {
if (queryStructCmd.getQueryType().isNativeAggQuery()) {
return false;
}
if (CollectionUtils.isEmpty(queryStructCmd.getAggregators())) {

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/MetricCheckConverter.java
View File

@@ -24,7 +24,7 @@ public class MetricCheckConverter implements SemanticConverter {
@Override
public boolean accept(QueryStructReq queryStructCmd) {
if (queryStructCmd.getNativeQuery()) {
if (queryStructCmd.getQueryType().isNativeAggQuery()) {
return false;
}
return !CollectionUtils.isEmpty(queryStructCmd.getAggregators());

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/ParserDefaultConverter.java
View File

@@ -62,7 +62,7 @@ public class ParserDefaultConverter implements SemanticConverter {
// todo tmp delete
// support detail query
if (queryStructCmd.getNativeQuery() && CollectionUtils.isEmpty(sqlCommend.getMetrics())) {
if (queryStructCmd.getQueryType().isNativeAggQuery() && CollectionUtils.isEmpty(sqlCommend.getMetrics())) {
String internalMetricName = queryStructUtils.generateInternalMetricName(
queryStructCmd.getModelId(), queryStructCmd.getGroups());
sqlCommend.getMetrics().add(internalMetricName);

19
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/parser/convert/QueryReqConverter.java
View File

@@ -3,6 +3,7 @@ package com.tencent.supersonic.semantic.query.parser.convert;
import com.tencent.supersonic.common.pojo.Aggregator;
import com.tencent.supersonic.common.pojo.Constants;
import com.tencent.supersonic.common.pojo.QueryType;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
@@ -112,7 +113,7 @@ public class QueryReqConverter {
//5.physicalSql by ParseSqlReq
queryStructCmd.setDateInfo(queryStructUtils.getDateConfBySql(databaseReq.getSql()));
queryStructCmd.setModelId(databaseReq.getModelId());
queryStructCmd.setNativeQuery(!AggOption.isAgg(aggOption));
queryStructCmd.setQueryType(getQueryType(aggOption));
log.info("QueryReqConverter queryStructCmd[{}]", queryStructCmd);
QueryStatement queryStatement = parserService.physicalSql(queryStructCmd, result);
queryStatement.setSql(String.format(SqlExecuteReq.LIMIT_WRAPPER, queryStatement.getSql()));
@@ -146,18 +147,16 @@ public class QueryReqConverter {
Map<String, String> internalLowerToNameMap = QueryStructUtils.internalCols.stream()
.collect(Collectors.toMap(a -> a.toLowerCase(), a -> a));
dimensionLowerToNameMap.putAll(internalLowerToNameMap);
Set<String> collect = allFields.stream()
return allFields.stream()
.filter(entry -> dimensionLowerToNameMap.containsKey(entry.toLowerCase()))
.map(entry -> dimensionLowerToNameMap.get(entry.toLowerCase())).collect(Collectors.toSet());
return collect;
}
private List<String> getMetrics(ModelSchemaResp modelSchemaResp, List<String> allFields) {
Map<String, String> metricLowerToNameMap = modelSchemaResp.getMetrics().stream()
.collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(), entry -> entry.getBizName()));
List<String> metrics = allFields.stream().filter(entry -> metricLowerToNameMap.containsKey(entry.toLowerCase()))
return allFields.stream().filter(entry -> metricLowerToNameMap.containsKey(entry.toLowerCase()))
.map(entry -> metricLowerToNameMap.get(entry.toLowerCase())).collect(Collectors.toList());
return metrics;
}
private void functionNameCorrector(QueryS2SQLReq databaseReq) {
@@ -175,7 +174,6 @@ public class QueryReqConverter {
}
}
protected Map<String, String> getFieldNameToBizNameMap(ModelSchemaResp modelSchemaResp) {
// support fieldName and field alias to bizName
Map<String, String> dimensionResults = modelSchemaResp.getDimensions().stream()
@@ -216,4 +214,13 @@ public class QueryReqConverter {
databaseReq.setSql(sql);
}
private QueryType getQueryType(AggOption aggOption) {
boolean isAgg = AggOption.isAgg(aggOption);
QueryType queryType = QueryType.ENTITY;
if (isAgg) {
queryType = QueryType.METRIC;
}
return queryType;
}
}

2
semantic/query/src/main/java/com/tencent/supersonic/semantic/query/utils/StatUtils.java
View File

@@ -144,7 +144,7 @@ public class StatUtils {
.setQueryStructCmd(queryStructCmd.toString())
.setQueryStructCmdMd5(DigestUtils.md5Hex(queryStructCmd.toString()))
.setStartTime(System.currentTimeMillis())
.setNativeQuery(queryStructCmd.getNativeQuery())
.setNativeQuery(queryStructCmd.getQueryType().isNativeAggQuery())
.setGroupByCols(objectMapper.writeValueAsString(queryStructCmd.getGroups()))
.setAggCols(objectMapper.writeValueAsString(queryStructCmd.getAggregators()))
.setOrderByCols(objectMapper.writeValueAsString(queryStructCmd.getOrders()))