topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 11:07:06 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(headless) performParsed skip translation (#1222)

Browse Source
This commit is contained in:
jipeli
2024-06-25 20:40:41 +08:00
committed by GitHub
parent 5ce81b3724
commit db9a3fa056
14 changed files with 66 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SqlInfo.java
View File

@@ -8,4 +8,5 @@ public class SqlInfo {
private String s2SQL;
private String correctS2SQL;
private String querySQL;
private String sourceId;
}

3
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/AggOption.java
View File

@@ -8,6 +8,7 @@ package com.tencent.supersonic.headless.api.pojo.enums;
public enum AggOption {
NATIVE,
AGGREGATION,
OUTER,
DEFAULT;
public static AggOption getAggregation(boolean isNativeQuery) {
@@ -15,6 +16,6 @@ public enum AggOption {
}
public static boolean isAgg(AggOption aggOption) {
return NATIVE.equals(aggOption) ? false : true;
return NATIVE.equals(aggOption) || OUTER.equals(aggOption) ? false : true;
}
}

1
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java
View File

@@ -164,6 +164,7 @@ public class QueryStructReq extends SemanticQueryReq {
result.setDataSetId(this.getDataSetId());
result.setModelIds(this.getModelIdSet());
result.setParams(new ArrayList<>());
result.setSqlInfo(this.getSqlInfo());
return result;
}

3
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SemanticQueryReq.java
View File

@@ -3,6 +3,7 @@ package com.tencent.supersonic.headless.api.pojo.request;
import com.google.common.collect.Lists;
import com.tencent.supersonic.headless.api.pojo.Cache;
import com.tencent.supersonic.headless.api.pojo.Param;
import com.tencent.supersonic.headless.api.pojo.SqlInfo;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.digest.DigestUtils;
@@ -31,6 +32,8 @@ public abstract class SemanticQueryReq {
protected Cache cacheInfo = new Cache();
protected SqlInfo sqlInfo = new SqlInfo();
public void addModelId(Long modelId) {
modelIds.add(modelId);
}

3
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ExplainResp.java
View File

@@ -17,4 +17,7 @@ public class ExplainResp implements Serializable {
private String sql;
private String sourceId;
}

4
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/llm/s2sql/LLMSqlQuery.java
View File

@@ -27,9 +27,7 @@ public class LLMSqlQuery extends LLMSemanticQuery {
@Override
public SemanticQueryReq buildSemanticQueryReq() {
String querySql = parseInfo.getSqlInfo().getCorrectS2SQL();
return QueryReqBuilder.buildS2SQLReq(querySql, parseInfo.getDataSetId());
return QueryReqBuilder.buildS2SQLReq(parseInfo.getSqlInfo(), parseInfo.getDataSetId());
}
@Override

12
headless/chat/src/main/java/com/tencent/supersonic/headless/chat/utils/QueryReqBuilder.java
View File

@@ -12,6 +12,7 @@ import com.tencent.supersonic.common.pojo.enums.QueryType;
import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.SqlInfo;
import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
@@ -127,6 +128,7 @@ public class QueryReqBuilder {
BeanUtils.copyProperties(queryStructReq, req);
req.setDataSetId(parseInfo.getDataSetId());
req.setDimensionFilters(Lists.newArrayList(dimensionFilter));
req.setSqlInfo(parseInfo.getSqlInfo());
queryStructReqs.add(req);
}
queryMultiStructReq.setQueryStructReqs(queryStructReqs);
@@ -149,6 +151,16 @@ public class QueryReqBuilder {
return querySQLReq;
}
public static QuerySqlReq buildS2SQLReq(SqlInfo sqlInfo, Long dataSetId) {
QuerySqlReq querySQLReq = new QuerySqlReq();
if (Objects.nonNull(sqlInfo.getCorrectS2SQL())) {
querySQLReq.setSql(sqlInfo.getCorrectS2SQL());
}
querySQLReq.setSqlInfo(sqlInfo);
querySQLReq.setDataSetId(dataSetId);
return querySQLReq;
}
private static List<Aggregator> getAggregatorByMetric(AggregateTypeEnum aggregateType, SchemaElement metric) {
List<Aggregator> aggregators = new ArrayList<>();
if (metric != null) {

6
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java
View File

@@ -11,6 +11,7 @@ import java.util.List;
@Data
public class QueryStatement {
private Long dataSetId;
private List<Long> modelIds;
private String sql = "";
@@ -34,12 +35,17 @@ public class QueryStatement {
private SemanticSchemaResp semanticSchemaResp;
private Integer limit = 1000;
private Boolean isTranslated = false;
public boolean isOk() {
this.ok = "".equals(errMsg) && !"".equals(sql);
return ok;
}
public boolean isTranslated() {
return isTranslated != null && isTranslated && isOk();
}
public QueryStatement error(String msg) {
this.setErrMsg(msg);
return this;

11
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/Configuration.java
View File

@@ -61,6 +61,9 @@ public class Configuration {
static {
configProperties.put(CalciteConnectionProperty.CASE_SENSITIVE.camelName(), Boolean.TRUE.toString());
configProperties.put(CalciteConnectionProperty.UNQUOTED_CASING.camelName(), Casing.UNCHANGED.toString());
configProperties.put(CalciteConnectionProperty.QUOTED_CASING.camelName(), Casing.TO_LOWER.toString());
}
public static SqlParser.Config getParserConfig(EngineType engineType) {
@@ -77,10 +80,14 @@ public class Configuration {
.setIdentifierMaxLength(Integer.MAX_VALUE)
.setQuoting(Quoting.BACK_TICK)
.setQuoting(Quoting.SINGLE_QUOTE)
.setQuotedCasing(Casing.TO_UPPER)
.setUnquotedCasing(Casing.TO_UPPER)
.setConformance(sqlDialect.getConformance())
.setLex(Lex.BIG_QUERY);
parserConfig = parserConfig.setQuotedCasing(Casing.TO_LOWER);
parserConfig = parserConfig.setUnquotedCasing(Casing.TO_LOWER);
if (!EngineType.CLICKHOUSE.equals(engineType)) {
parserConfig = parserConfig.setQuotedCasing(Casing.TO_LOWER);
parserConfig = parserConfig.setUnquotedCasing(Casing.TO_LOWER);
}
return parserConfig.build();
}

4
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java
View File

@@ -327,12 +327,12 @@ public class SqlGenerateUtils {
public String getExpr(Measure measure, AggOption aggOption) {
if (AggOperatorEnum.COUNT_DISTINCT.getOperator().equalsIgnoreCase(measure.getAgg())) {
return aggOption.equals(AggOption.NATIVE) ? measure.getBizName()
return AggOption.NATIVE.equals(aggOption) ? measure.getBizName()
: AggOperatorEnum.COUNT.getOperator() + " ( " + AggOperatorEnum.DISTINCT + " "
+ measure.getBizName()
+ " ) ";
}
return aggOption.equals(AggOption.NATIVE) ? measure.getBizName()
return AggOption.NATIVE.equals(aggOption) ? measure.getBizName()
: measure.getAgg() + " ( " + measure.getBizName() + " ) ";
}

1
headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/ChatQueryServiceImpl.java
View File

@@ -240,6 +240,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
ExplainResp explain = semanticLayerService.explain(explainSqlReq, user);
if (StringUtils.isNotBlank(explain.getSql())) {
parseInfo.getSqlInfo().setQuerySQL(explain.getSql());
parseInfo.getSqlInfo().setSourceId(explain.getSourceId());
}
} else {
log.info("rule begin replace metrics and revise filters!");

1
headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/SqlInfoProcessor.java
View File

@@ -79,6 +79,7 @@ public class SqlInfoProcessor implements ResultProcessor {
sqlInfo.getS2SQL(), sqlInfo.getCorrectS2SQL(), explainSql);
}
sqlInfo.setQuerySQL(explainSql);
sqlInfo.setSourceId(explain.getSourceId());
}
}

6
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java
View File

@@ -151,19 +151,19 @@ public class QueryReqConverter {
if (!SqlSelectFunctionHelper.hasAggregateFunction(sql)
|| SqlSelectFunctionHelper.hasFunction(sql, "count")
|| SqlSelectFunctionHelper.hasFunction(sql, "count_distinct")) {
return AggOption.NATIVE;
return AggOption.OUTER;
}
if (databaseReq.isInnerLayerNative()) {
return AggOption.NATIVE;
}
if (SqlSelectHelper.hasSubSelect(sql) || SqlSelectHelper.hasWith(sql) || SqlSelectHelper.hasGroupBy(sql)) {
return AggOption.NATIVE;
return AggOption.OUTER;
}
long defaultAggNullCnt = metricSchemas.stream()
.filter(m -> Objects.isNull(m.getDefaultAgg()) || Strings.isBlank(m.getDefaultAgg())).count();
if (defaultAggNullCnt > 0) {
log.info("getAggOption find null defaultAgg metric set to NATIVE");
return AggOption.NATIVE;
return AggOption.OUTER;
}
return AggOption.DEFAULT;
}

24
headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/impl/S2SemanticLayerService.java
View File

@@ -18,6 +18,7 @@ import com.tencent.supersonic.headless.api.pojo.SchemaElement;
import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
import com.tencent.supersonic.headless.api.pojo.SqlInfo;
import com.tencent.supersonic.headless.api.pojo.TagTypeDefaultConfig;
import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig;
import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq;
@@ -172,6 +173,10 @@ public class S2SemanticLayerService implements SemanticLayerService {
}
private QueryStatement buildQueryStatement(SemanticQueryReq semanticQueryReq, User user) throws Exception {
if (Objects.nonNull(semanticQueryReq.getSqlInfo()) && StringUtils.isNotBlank(
semanticQueryReq.getSqlInfo().getQuerySQL())) {
return buildSqlInfoStatement(semanticQueryReq.getSqlInfo(), semanticQueryReq.getDataSetId());
}
if (semanticQueryReq instanceof QuerySqlReq) {
return buildSqlQueryStatement((QuerySqlReq) semanticQueryReq, user);
}
@@ -215,6 +220,15 @@ public class S2SemanticLayerService implements SemanticLayerService {
return queryUtils.sqlParserUnion(queryMultiStructReq, sqlParsers);
}
private QueryStatement buildSqlInfoStatement(SqlInfo sqlInfo, Long dataSetId) {
QueryStatement queryStatement = new QueryStatement();
queryStatement.setSql(sqlInfo.getQuerySQL());
queryStatement.setSourceId(sqlInfo.getSourceId());
queryStatement.setDataSetId(dataSetId);
queryStatement.setIsTranslated(true);
return queryStatement;
}
private SchemaFilterReq buildSchemaFilterReq(SemanticQueryReq semanticQueryReq) {
SchemaFilterReq schemaFilterReq = new SchemaFilterReq();
schemaFilterReq.setDataSetId(semanticQueryReq.getDataSetId());
@@ -242,10 +256,12 @@ public class S2SemanticLayerService implements SemanticLayerService {
semanticTranslator.translate(queryStatement);
String sql = "";
String sorceId = "";
if (Objects.nonNull(queryStatement)) {
sql = queryStatement.getSql();
sorceId = queryStatement.getSourceId();
}
return ExplainResp.builder().sql(sql).build();
return ExplainResp.builder().sql(sql).sourceId(sorceId).build();
}
private QuerySqlReq buildQuerySqlReq(QueryDimValueReq queryDimValueReq) {
@@ -270,7 +286,9 @@ public class S2SemanticLayerService implements SemanticLayerService {
SemanticQueryResp semanticQueryResp = null;
try {
//1 translate
semanticTranslator.translate(queryStatement);
if (!queryStatement.isTranslated()) {
semanticTranslator.translate(queryStatement);
}
//2 execute
for (QueryExecutor queryExecutor : ComponentFactory.getQueryExecutors()) {
@@ -381,7 +399,7 @@ public class S2SemanticLayerService implements SemanticLayerService {
}
private SemanticQueryResp getQueryResultWithSchemaResp(EntityInfo entityInfo,
DataSetSchema dataSetSchema, User user) {
DataSetSchema dataSetSchema, User user) {
SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
semanticParseInfo.setDataSet(dataSetSchema.getDataSet());
semanticParseInfo.setQueryType(QueryType.DETAIL);