diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SqlInfo.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SqlInfo.java index 6d7b332c0..f3c1bbea2 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SqlInfo.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SqlInfo.java @@ -8,4 +8,5 @@ public class SqlInfo { private String s2SQL; private String correctS2SQL; private String querySQL; + private String sourceId; } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/AggOption.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/AggOption.java index 3fdfa574c..1c46d8d6b 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/AggOption.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/AggOption.java @@ -8,6 +8,7 @@ package com.tencent.supersonic.headless.api.pojo.enums; public enum AggOption { NATIVE, AGGREGATION, + OUTER, DEFAULT; public static AggOption getAggregation(boolean isNativeQuery) { @@ -15,6 +16,6 @@ public enum AggOption { } public static boolean isAgg(AggOption aggOption) { - return NATIVE.equals(aggOption) ? false : true; + return NATIVE.equals(aggOption) || OUTER.equals(aggOption) ? false : true; } } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java index 12bd3187d..5854270fb 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java @@ -164,6 +164,7 @@ public class QueryStructReq extends SemanticQueryReq { result.setDataSetId(this.getDataSetId()); result.setModelIds(this.getModelIdSet()); result.setParams(new ArrayList<>()); + result.setSqlInfo(this.getSqlInfo()); return result; } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SemanticQueryReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SemanticQueryReq.java index 6273e8f03..ebaaf3a40 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SemanticQueryReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SemanticQueryReq.java @@ -3,6 +3,7 @@ package com.tencent.supersonic.headless.api.pojo.request; import com.google.common.collect.Lists; import com.tencent.supersonic.headless.api.pojo.Cache; import com.tencent.supersonic.headless.api.pojo.Param; +import com.tencent.supersonic.headless.api.pojo.SqlInfo; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.apache.commons.codec.digest.DigestUtils; @@ -31,6 +32,8 @@ public abstract class SemanticQueryReq { protected Cache cacheInfo = new Cache(); + protected SqlInfo sqlInfo = new SqlInfo(); + public void addModelId(Long modelId) { modelIds.add(modelId); } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ExplainResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ExplainResp.java index 4f2730506..92d888423 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ExplainResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ExplainResp.java @@ -17,4 +17,7 @@ public class ExplainResp implements Serializable { private String sql; + private String sourceId; + + } diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/llm/s2sql/LLMSqlQuery.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/llm/s2sql/LLMSqlQuery.java index 75466c71c..f2add7ff2 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/llm/s2sql/LLMSqlQuery.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/llm/s2sql/LLMSqlQuery.java @@ -27,9 +27,7 @@ public class LLMSqlQuery extends LLMSemanticQuery { @Override public SemanticQueryReq buildSemanticQueryReq() { - - String querySql = parseInfo.getSqlInfo().getCorrectS2SQL(); - return QueryReqBuilder.buildS2SQLReq(querySql, parseInfo.getDataSetId()); + return QueryReqBuilder.buildS2SQLReq(parseInfo.getSqlInfo(), parseInfo.getDataSetId()); } @Override diff --git a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/utils/QueryReqBuilder.java b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/utils/QueryReqBuilder.java index 6a224f378..69b619e32 100644 --- a/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/utils/QueryReqBuilder.java +++ b/headless/chat/src/main/java/com/tencent/supersonic/headless/chat/utils/QueryReqBuilder.java @@ -12,6 +12,7 @@ import com.tencent.supersonic.common.pojo.enums.QueryType; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; +import com.tencent.supersonic.headless.api.pojo.SqlInfo; import com.tencent.supersonic.headless.api.pojo.request.QueryFilter; import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; @@ -127,6 +128,7 @@ public class QueryReqBuilder { BeanUtils.copyProperties(queryStructReq, req); req.setDataSetId(parseInfo.getDataSetId()); req.setDimensionFilters(Lists.newArrayList(dimensionFilter)); + req.setSqlInfo(parseInfo.getSqlInfo()); queryStructReqs.add(req); } queryMultiStructReq.setQueryStructReqs(queryStructReqs); @@ -149,6 +151,16 @@ public class QueryReqBuilder { return querySQLReq; } + public static QuerySqlReq buildS2SQLReq(SqlInfo sqlInfo, Long dataSetId) { + QuerySqlReq querySQLReq = new QuerySqlReq(); + if (Objects.nonNull(sqlInfo.getCorrectS2SQL())) { + querySQLReq.setSql(sqlInfo.getCorrectS2SQL()); + } + querySQLReq.setSqlInfo(sqlInfo); + querySQLReq.setDataSetId(dataSetId); + return querySQLReq; + } + private static List getAggregatorByMetric(AggregateTypeEnum aggregateType, SchemaElement metric) { List aggregators = new ArrayList<>(); if (metric != null) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java index 913e09e10..d8b737882 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java @@ -11,6 +11,7 @@ import java.util.List; @Data public class QueryStatement { + private Long dataSetId; private List modelIds; private String sql = ""; @@ -34,12 +35,17 @@ public class QueryStatement { private SemanticSchemaResp semanticSchemaResp; private Integer limit = 1000; + private Boolean isTranslated = false; public boolean isOk() { this.ok = "".equals(errMsg) && !"".equals(sql); return ok; } + public boolean isTranslated() { + return isTranslated != null && isTranslated && isOk(); + } + public QueryStatement error(String msg) { this.setErrMsg(msg); return this; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/Configuration.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/Configuration.java index 84d7ef998..caaa4f26b 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/Configuration.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/Configuration.java @@ -61,6 +61,9 @@ public class Configuration { static { configProperties.put(CalciteConnectionProperty.CASE_SENSITIVE.camelName(), Boolean.TRUE.toString()); + configProperties.put(CalciteConnectionProperty.UNQUOTED_CASING.camelName(), Casing.UNCHANGED.toString()); + configProperties.put(CalciteConnectionProperty.QUOTED_CASING.camelName(), Casing.TO_LOWER.toString()); + } public static SqlParser.Config getParserConfig(EngineType engineType) { @@ -77,10 +80,14 @@ public class Configuration { .setIdentifierMaxLength(Integer.MAX_VALUE) .setQuoting(Quoting.BACK_TICK) .setQuoting(Quoting.SINGLE_QUOTE) + .setQuotedCasing(Casing.TO_UPPER) + .setUnquotedCasing(Casing.TO_UPPER) .setConformance(sqlDialect.getConformance()) .setLex(Lex.BIG_QUERY); - parserConfig = parserConfig.setQuotedCasing(Casing.TO_LOWER); - parserConfig = parserConfig.setUnquotedCasing(Casing.TO_LOWER); + if (!EngineType.CLICKHOUSE.equals(engineType)) { + parserConfig = parserConfig.setQuotedCasing(Casing.TO_LOWER); + parserConfig = parserConfig.setUnquotedCasing(Casing.TO_LOWER); + } return parserConfig.build(); } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java index 52fd87fb2..eac9952b7 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java @@ -327,12 +327,12 @@ public class SqlGenerateUtils { public String getExpr(Measure measure, AggOption aggOption) { if (AggOperatorEnum.COUNT_DISTINCT.getOperator().equalsIgnoreCase(measure.getAgg())) { - return aggOption.equals(AggOption.NATIVE) ? measure.getBizName() + return AggOption.NATIVE.equals(aggOption) ? measure.getBizName() : AggOperatorEnum.COUNT.getOperator() + " ( " + AggOperatorEnum.DISTINCT + " " + measure.getBizName() + " ) "; } - return aggOption.equals(AggOption.NATIVE) ? measure.getBizName() + return AggOption.NATIVE.equals(aggOption) ? measure.getBizName() : measure.getAgg() + " ( " + measure.getBizName() + " ) "; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/ChatQueryServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/ChatQueryServiceImpl.java index 07825891a..41fb2cdd5 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/ChatQueryServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/ChatQueryServiceImpl.java @@ -240,6 +240,7 @@ public class ChatQueryServiceImpl implements ChatQueryService { ExplainResp explain = semanticLayerService.explain(explainSqlReq, user); if (StringUtils.isNotBlank(explain.getSql())) { parseInfo.getSqlInfo().setQuerySQL(explain.getSql()); + parseInfo.getSqlInfo().setSourceId(explain.getSourceId()); } } else { log.info("rule begin replace metrics and revise filters!"); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/SqlInfoProcessor.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/SqlInfoProcessor.java index 316f1633c..541a0d89f 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/SqlInfoProcessor.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/SqlInfoProcessor.java @@ -79,6 +79,7 @@ public class SqlInfoProcessor implements ResultProcessor { sqlInfo.getS2SQL(), sqlInfo.getCorrectS2SQL(), explainSql); } sqlInfo.setQuerySQL(explainSql); + sqlInfo.setSourceId(explain.getSourceId()); } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java index 860ce4450..a950a777b 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java @@ -151,19 +151,19 @@ public class QueryReqConverter { if (!SqlSelectFunctionHelper.hasAggregateFunction(sql) || SqlSelectFunctionHelper.hasFunction(sql, "count") || SqlSelectFunctionHelper.hasFunction(sql, "count_distinct")) { - return AggOption.NATIVE; + return AggOption.OUTER; } if (databaseReq.isInnerLayerNative()) { return AggOption.NATIVE; } if (SqlSelectHelper.hasSubSelect(sql) || SqlSelectHelper.hasWith(sql) || SqlSelectHelper.hasGroupBy(sql)) { - return AggOption.NATIVE; + return AggOption.OUTER; } long defaultAggNullCnt = metricSchemas.stream() .filter(m -> Objects.isNull(m.getDefaultAgg()) || Strings.isBlank(m.getDefaultAgg())).count(); if (defaultAggNullCnt > 0) { log.info("getAggOption find null defaultAgg metric set to NATIVE"); - return AggOption.NATIVE; + return AggOption.OUTER; } return AggOption.DEFAULT; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/impl/S2SemanticLayerService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/impl/S2SemanticLayerService.java index c1d5f6796..230abb97f 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/impl/S2SemanticLayerService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/impl/S2SemanticLayerService.java @@ -18,6 +18,7 @@ import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SchemaElementType; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.SemanticSchema; +import com.tencent.supersonic.headless.api.pojo.SqlInfo; import com.tencent.supersonic.headless.api.pojo.TagTypeDefaultConfig; import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq; @@ -172,6 +173,10 @@ public class S2SemanticLayerService implements SemanticLayerService { } private QueryStatement buildQueryStatement(SemanticQueryReq semanticQueryReq, User user) throws Exception { + if (Objects.nonNull(semanticQueryReq.getSqlInfo()) && StringUtils.isNotBlank( + semanticQueryReq.getSqlInfo().getQuerySQL())) { + return buildSqlInfoStatement(semanticQueryReq.getSqlInfo(), semanticQueryReq.getDataSetId()); + } if (semanticQueryReq instanceof QuerySqlReq) { return buildSqlQueryStatement((QuerySqlReq) semanticQueryReq, user); } @@ -215,6 +220,15 @@ public class S2SemanticLayerService implements SemanticLayerService { return queryUtils.sqlParserUnion(queryMultiStructReq, sqlParsers); } + private QueryStatement buildSqlInfoStatement(SqlInfo sqlInfo, Long dataSetId) { + QueryStatement queryStatement = new QueryStatement(); + queryStatement.setSql(sqlInfo.getQuerySQL()); + queryStatement.setSourceId(sqlInfo.getSourceId()); + queryStatement.setDataSetId(dataSetId); + queryStatement.setIsTranslated(true); + return queryStatement; + } + private SchemaFilterReq buildSchemaFilterReq(SemanticQueryReq semanticQueryReq) { SchemaFilterReq schemaFilterReq = new SchemaFilterReq(); schemaFilterReq.setDataSetId(semanticQueryReq.getDataSetId()); @@ -242,10 +256,12 @@ public class S2SemanticLayerService implements SemanticLayerService { semanticTranslator.translate(queryStatement); String sql = ""; + String sorceId = ""; if (Objects.nonNull(queryStatement)) { sql = queryStatement.getSql(); + sorceId = queryStatement.getSourceId(); } - return ExplainResp.builder().sql(sql).build(); + return ExplainResp.builder().sql(sql).sourceId(sorceId).build(); } private QuerySqlReq buildQuerySqlReq(QueryDimValueReq queryDimValueReq) { @@ -270,7 +286,9 @@ public class S2SemanticLayerService implements SemanticLayerService { SemanticQueryResp semanticQueryResp = null; try { //1 translate - semanticTranslator.translate(queryStatement); + if (!queryStatement.isTranslated()) { + semanticTranslator.translate(queryStatement); + } //2 execute for (QueryExecutor queryExecutor : ComponentFactory.getQueryExecutors()) { @@ -381,7 +399,7 @@ public class S2SemanticLayerService implements SemanticLayerService { } private SemanticQueryResp getQueryResultWithSchemaResp(EntityInfo entityInfo, - DataSetSchema dataSetSchema, User user) { + DataSetSchema dataSetSchema, User user) { SemanticParseInfo semanticParseInfo = new SemanticParseInfo(); semanticParseInfo.setDataSet(dataSetSchema.getDataSet()); semanticParseInfo.setQueryType(QueryType.DETAIL);