(improvement)(headless) performParsed skip translation (#1222)

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SqlInfo.java

@@ -8,4 +8,5 @@ public class SqlInfo {
     private String s2SQL;
     private String correctS2SQL;
     private String querySQL;
+    private String sourceId;
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/AggOption.java

@@ -8,6 +8,7 @@ package com.tencent.supersonic.headless.api.pojo.enums;
 public enum AggOption {
     NATIVE,
     AGGREGATION,
+    OUTER,
     DEFAULT;
 
     public static AggOption getAggregation(boolean isNativeQuery) {
@@ -15,6 +16,6 @@ public enum AggOption {
     }
 
     public static boolean isAgg(AggOption aggOption) {
-        return NATIVE.equals(aggOption) ? false : true;
+        return NATIVE.equals(aggOption) || OUTER.equals(aggOption) ? false : true;
     }
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java

@@ -164,6 +164,7 @@ public class QueryStructReq extends SemanticQueryReq {
         result.setDataSetId(this.getDataSetId());
         result.setModelIds(this.getModelIdSet());
         result.setParams(new ArrayList<>());
+        result.setSqlInfo(this.getSqlInfo());
         return result;
     }
 

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/SemanticQueryReq.java

@@ -3,6 +3,7 @@ package com.tencent.supersonic.headless.api.pojo.request;
 import com.google.common.collect.Lists;
 import com.tencent.supersonic.headless.api.pojo.Cache;
 import com.tencent.supersonic.headless.api.pojo.Param;
+import com.tencent.supersonic.headless.api.pojo.SqlInfo;
 import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.codec.digest.DigestUtils;
@@ -31,6 +32,8 @@ public abstract class SemanticQueryReq {
 
     protected Cache cacheInfo = new Cache();
 
+    protected SqlInfo sqlInfo = new SqlInfo();
+
     public void addModelId(Long modelId) {
         modelIds.add(modelId);
     }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ExplainResp.java

@@ -17,4 +17,7 @@ public class ExplainResp implements Serializable {
 
     private String sql;
 
+    private String sourceId;
+
+
 }

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/query/llm/s2sql/LLMSqlQuery.java

@@ -27,9 +27,7 @@ public class LLMSqlQuery extends LLMSemanticQuery {
 
     @Override
     public SemanticQueryReq buildSemanticQueryReq() {
-
+        return QueryReqBuilder.buildS2SQLReq(parseInfo.getSqlInfo(), parseInfo.getDataSetId());
-        String querySql = parseInfo.getSqlInfo().getCorrectS2SQL();
-        return QueryReqBuilder.buildS2SQLReq(querySql, parseInfo.getDataSetId());
     }
 
     @Override

headless/chat/src/main/java/com/tencent/supersonic/headless/chat/utils/QueryReqBuilder.java

@@ -12,6 +12,7 @@ import com.tencent.supersonic.common.pojo.enums.QueryType;
 import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
 import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
+import com.tencent.supersonic.headless.api.pojo.SqlInfo;
 import com.tencent.supersonic.headless.api.pojo.request.QueryFilter;
 import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq;
 import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
@@ -127,6 +128,7 @@ public class QueryReqBuilder {
             BeanUtils.copyProperties(queryStructReq, req);
             req.setDataSetId(parseInfo.getDataSetId());
             req.setDimensionFilters(Lists.newArrayList(dimensionFilter));
+            req.setSqlInfo(parseInfo.getSqlInfo());
             queryStructReqs.add(req);
         }
         queryMultiStructReq.setQueryStructReqs(queryStructReqs);
@@ -149,6 +151,16 @@ public class QueryReqBuilder {
         return querySQLReq;
     }
 
+    public static QuerySqlReq buildS2SQLReq(SqlInfo sqlInfo, Long dataSetId) {
+        QuerySqlReq querySQLReq = new QuerySqlReq();
+        if (Objects.nonNull(sqlInfo.getCorrectS2SQL())) {
+            querySQLReq.setSql(sqlInfo.getCorrectS2SQL());
+        }
+        querySQLReq.setSqlInfo(sqlInfo);
+        querySQLReq.setDataSetId(dataSetId);
+        return querySQLReq;
+    }
+
     private static List<Aggregator> getAggregatorByMetric(AggregateTypeEnum aggregateType, SchemaElement metric) {
         List<Aggregator> aggregators = new ArrayList<>();
         if (metric != null) {

headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java

@@ -11,6 +11,7 @@ import java.util.List;
 
 @Data
 public class QueryStatement {
+
     private Long dataSetId;
     private List<Long> modelIds;
     private String sql = "";
@@ -34,12 +35,17 @@ public class QueryStatement {
 
     private SemanticSchemaResp semanticSchemaResp;
     private Integer limit = 1000;
+    private Boolean isTranslated = false;
 
     public boolean isOk() {
         this.ok = "".equals(errMsg) && !"".equals(sql);
         return ok;
     }
 
+    public boolean isTranslated() {
+        return isTranslated != null && isTranslated && isOk();
+    }
+
     public QueryStatement error(String msg) {
         this.setErrMsg(msg);
         return this;

headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/Configuration.java

@@ -61,6 +61,9 @@ public class Configuration {
 
     static {
         configProperties.put(CalciteConnectionProperty.CASE_SENSITIVE.camelName(), Boolean.TRUE.toString());
+        configProperties.put(CalciteConnectionProperty.UNQUOTED_CASING.camelName(), Casing.UNCHANGED.toString());
+        configProperties.put(CalciteConnectionProperty.QUOTED_CASING.camelName(), Casing.TO_LOWER.toString());
+
     }
 
     public static SqlParser.Config getParserConfig(EngineType engineType) {
@@ -77,10 +80,14 @@ public class Configuration {
                 .setIdentifierMaxLength(Integer.MAX_VALUE)
                 .setQuoting(Quoting.BACK_TICK)
                 .setQuoting(Quoting.SINGLE_QUOTE)
+                .setQuotedCasing(Casing.TO_UPPER)
+                .setUnquotedCasing(Casing.TO_UPPER)
                 .setConformance(sqlDialect.getConformance())
                 .setLex(Lex.BIG_QUERY);
-        parserConfig = parserConfig.setQuotedCasing(Casing.TO_LOWER);
+        if (!EngineType.CLICKHOUSE.equals(engineType)) {
-        parserConfig = parserConfig.setUnquotedCasing(Casing.TO_LOWER);
+            parserConfig = parserConfig.setQuotedCasing(Casing.TO_LOWER);
+            parserConfig = parserConfig.setUnquotedCasing(Casing.TO_LOWER);
+        }
         return parserConfig.build();
     }
 

headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java

@@ -327,12 +327,12 @@ public class SqlGenerateUtils {
 
     public String getExpr(Measure measure, AggOption aggOption) {
         if (AggOperatorEnum.COUNT_DISTINCT.getOperator().equalsIgnoreCase(measure.getAgg())) {
-            return aggOption.equals(AggOption.NATIVE) ? measure.getBizName()
+            return AggOption.NATIVE.equals(aggOption) ? measure.getBizName()
                     : AggOperatorEnum.COUNT.getOperator() + " ( " + AggOperatorEnum.DISTINCT + " "
                             + measure.getBizName()
                             + " ) ";
         }
-        return aggOption.equals(AggOption.NATIVE) ? measure.getBizName()
+        return AggOption.NATIVE.equals(aggOption) ? measure.getBizName()
                 : measure.getAgg() + " ( " + measure.getBizName() + " ) ";
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/ChatQueryServiceImpl.java

@@ -240,6 +240,7 @@ public class ChatQueryServiceImpl implements ChatQueryService {
             ExplainResp explain = semanticLayerService.explain(explainSqlReq, user);
             if (StringUtils.isNotBlank(explain.getSql())) {
                 parseInfo.getSqlInfo().setQuerySQL(explain.getSql());
+                parseInfo.getSqlInfo().setSourceId(explain.getSourceId());
             }
         } else {
             log.info("rule begin replace metrics and revise filters!");

headless/server/src/main/java/com/tencent/supersonic/headless/server/processor/SqlInfoProcessor.java

@@ -79,6 +79,7 @@ public class SqlInfoProcessor implements ResultProcessor {
                     sqlInfo.getS2SQL(), sqlInfo.getCorrectS2SQL(), explainSql);
         }
         sqlInfo.setQuerySQL(explainSql);
+        sqlInfo.setSourceId(explain.getSourceId());
     }
 
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java

@@ -151,19 +151,19 @@ public class QueryReqConverter {
         if (!SqlSelectFunctionHelper.hasAggregateFunction(sql)
                 || SqlSelectFunctionHelper.hasFunction(sql, "count")
                 || SqlSelectFunctionHelper.hasFunction(sql, "count_distinct")) {
-            return AggOption.NATIVE;
+            return AggOption.OUTER;
         }
         if (databaseReq.isInnerLayerNative()) {
             return AggOption.NATIVE;
         }
         if (SqlSelectHelper.hasSubSelect(sql) || SqlSelectHelper.hasWith(sql) || SqlSelectHelper.hasGroupBy(sql)) {
-            return AggOption.NATIVE;
+            return AggOption.OUTER;
         }
         long defaultAggNullCnt = metricSchemas.stream()
                 .filter(m -> Objects.isNull(m.getDefaultAgg()) || Strings.isBlank(m.getDefaultAgg())).count();
         if (defaultAggNullCnt > 0) {
             log.info("getAggOption find null defaultAgg metric set to NATIVE");
-            return AggOption.NATIVE;
+            return AggOption.OUTER;
         }
         return AggOption.DEFAULT;
     }

headless/server/src/main/java/com/tencent/supersonic/headless/server/web/service/impl/S2SemanticLayerService.java

@@ -18,6 +18,7 @@ import com.tencent.supersonic.headless.api.pojo.SchemaElement;
 import com.tencent.supersonic.headless.api.pojo.SchemaElementType;
 import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo;
 import com.tencent.supersonic.headless.api.pojo.SemanticSchema;
+import com.tencent.supersonic.headless.api.pojo.SqlInfo;
 import com.tencent.supersonic.headless.api.pojo.TagTypeDefaultConfig;
 import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig;
 import com.tencent.supersonic.headless.api.pojo.request.ExplainSqlReq;
@@ -172,6 +173,10 @@ public class S2SemanticLayerService implements SemanticLayerService {
     }
 
     private QueryStatement buildQueryStatement(SemanticQueryReq semanticQueryReq, User user) throws Exception {
+        if (Objects.nonNull(semanticQueryReq.getSqlInfo()) && StringUtils.isNotBlank(
+                semanticQueryReq.getSqlInfo().getQuerySQL())) {
+            return buildSqlInfoStatement(semanticQueryReq.getSqlInfo(), semanticQueryReq.getDataSetId());
+        }
         if (semanticQueryReq instanceof QuerySqlReq) {
             return buildSqlQueryStatement((QuerySqlReq) semanticQueryReq, user);
         }
@@ -215,6 +220,15 @@ public class S2SemanticLayerService implements SemanticLayerService {
         return queryUtils.sqlParserUnion(queryMultiStructReq, sqlParsers);
     }
 
+    private QueryStatement buildSqlInfoStatement(SqlInfo sqlInfo, Long dataSetId) {
+        QueryStatement queryStatement = new QueryStatement();
+        queryStatement.setSql(sqlInfo.getQuerySQL());
+        queryStatement.setSourceId(sqlInfo.getSourceId());
+        queryStatement.setDataSetId(dataSetId);
+        queryStatement.setIsTranslated(true);
+        return queryStatement;
+    }
+
     private SchemaFilterReq buildSchemaFilterReq(SemanticQueryReq semanticQueryReq) {
         SchemaFilterReq schemaFilterReq = new SchemaFilterReq();
         schemaFilterReq.setDataSetId(semanticQueryReq.getDataSetId());
@@ -242,10 +256,12 @@ public class S2SemanticLayerService implements SemanticLayerService {
         semanticTranslator.translate(queryStatement);
 
         String sql = "";
+        String sorceId = "";
         if (Objects.nonNull(queryStatement)) {
             sql = queryStatement.getSql();
+            sorceId = queryStatement.getSourceId();
         }
-        return ExplainResp.builder().sql(sql).build();
+        return ExplainResp.builder().sql(sql).sourceId(sorceId).build();
     }
 
     private QuerySqlReq buildQuerySqlReq(QueryDimValueReq queryDimValueReq) {
@@ -270,7 +286,9 @@ public class S2SemanticLayerService implements SemanticLayerService {
         SemanticQueryResp semanticQueryResp = null;
         try {
             //1 translate
-            semanticTranslator.translate(queryStatement);
+            if (!queryStatement.isTranslated()) {
+                semanticTranslator.translate(queryStatement);
+            }
 
             //2 execute
             for (QueryExecutor queryExecutor : ComponentFactory.getQueryExecutors()) {
@@ -381,7 +399,7 @@ public class S2SemanticLayerService implements SemanticLayerService {
     }
 
     private SemanticQueryResp getQueryResultWithSchemaResp(EntityInfo entityInfo,
-                                                           DataSetSchema dataSetSchema, User user) {
+            DataSetSchema dataSetSchema, User user) {
         SemanticParseInfo semanticParseInfo = new SemanticParseInfo();
         semanticParseInfo.setDataSet(dataSetSchema.getDataSet());
         semanticParseInfo.setQueryType(QueryType.DETAIL);