diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelper.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelper.java index 6604f2487..d5443c328 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelper.java @@ -528,7 +528,7 @@ public class SqlReplaceHelper { } } - private static Select replaceAggAliasOrderItem(Select selectStatement) { + private static Select replaceAggAliasOrderbyField(Select selectStatement) { if (selectStatement instanceof PlainSelect) { PlainSelect plainSelect = (PlainSelect) selectStatement; if (Objects.nonNull(plainSelect.getOrderByElements())) { @@ -564,15 +564,15 @@ public class SqlReplaceHelper { if (plainSelect.getFromItem() instanceof ParenthesedSelect) { ParenthesedSelect parenthesedSelect = (ParenthesedSelect) plainSelect.getFromItem(); parenthesedSelect - .setSelect(replaceAggAliasOrderItem(parenthesedSelect.getSelect())); + .setSelect(replaceAggAliasOrderbyField(parenthesedSelect.getSelect())); } return selectStatement; } return selectStatement; } - public static String replaceAggAliasOrderItem(String sql) { - Select selectStatement = replaceAggAliasOrderItem(SqlSelectHelper.getSelect(sql)); + public static String replaceAggAliasOrderbyField(String sql) { + Select selectStatement = replaceAggAliasOrderbyField(SqlSelectHelper.getSelect(sql)); return selectStatement.toString(); } diff --git a/common/src/main/java/com/tencent/supersonic/common/util/DateModeUtils.java b/common/src/main/java/com/tencent/supersonic/common/util/DateModeUtils.java index a5753c085..75ee5a597 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/DateModeUtils.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/DateModeUtils.java @@ -4,6 +4,7 @@ import com.tencent.supersonic.common.pojo.Constants; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.ItemDateResp; import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum; +import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -32,14 +33,9 @@ import static com.tencent.supersonic.common.pojo.Constants.MONTH_FORMAT; @Data public class DateModeUtils { - @Value("${s2.query.parameter.sys.date:sys_imp_date}") - private String sysDateCol; - - @Value("${s2.query.parameter.sys.month:sys_imp_month}") - private String sysDateMonthCol; - - @Value("${s2.query.parameter.sys.month:sys_imp_week}") - private String sysDateWeekCol; + private final String sysDateCol = TimeDimensionEnum.DAY.getName(); + private final String sysDateMonthCol = TimeDimensionEnum.MONTH.getName(); + private final String sysDateWeekCol = TimeDimensionEnum.WEEK.getName(); @Value("${s2.query.parameter.sys.zipper.begin:start_}") private String sysZipperDateColBegin; diff --git a/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelperTest.java b/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelperTest.java index a641b2539..084978319 100644 --- a/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelperTest.java +++ b/common/src/test/java/com/tencent/supersonic/common/jsqlparser/SqlReplaceHelperTest.java @@ -325,10 +325,10 @@ class SqlReplaceHelperTest { } @Test - void testReplaceAggAliasOrderItem() { + void testReplaceAggAliasOrderbyField() { String sql = "SELECT SUM(访问次数) AS top10总播放量 FROM (SELECT 部门, SUM(访问次数) AS 访问次数 FROM 超音数 " + "GROUP BY 部门 ORDER BY SUM(访问次数) DESC LIMIT 10) AS top10"; - String replaceSql = SqlReplaceHelper.replaceAggAliasOrderItem(sql); + String replaceSql = SqlReplaceHelper.replaceAggAliasOrderbyField(sql); Assert.assertEquals( "SELECT SUM(访问次数) AS top10总播放量 FROM (SELECT 部门, SUM(访问次数) AS 访问次数 FROM 超音数 " + "GROUP BY 部门 ORDER BY 2 DESC LIMIT 10) AS top10", diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ParseSqlReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ParseSqlReq.java deleted file mode 100644 index ec4519030..000000000 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/ParseSqlReq.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.tencent.supersonic.headless.api.pojo.request; - -import com.tencent.supersonic.headless.api.pojo.MetricTable; -import lombok.Data; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -@Data -public class ParseSqlReq { - private Map variables; - private String sql = ""; - private List tables; - private boolean supportWith = true; - private boolean withAlias = true; - - public Map getVariables() { - if (variables == null) { - variables = new HashMap<>(); - } - return variables; - } -} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryFilter.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryFilter.java index b06814aef..446377a98 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryFilter.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryFilter.java @@ -34,12 +34,11 @@ public class QueryFilter implements Serializable { QueryFilter that = (QueryFilter) o; return Objects.equal(bizName, that.bizName) && Objects.equal(name, that.name) && operator == that.operator && Objects.equal(value, that.value) - && Objects.equal(elementID, that.elementID) && Objects.equal(function, that.function); } @Override public int hashCode() { - return Objects.hashCode(bizName, name, operator, value, elementID, function); + return Objects.hashCode(bizName, name, operator, value, function); } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/DataSetQueryParam.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/DataSetQueryParam.java deleted file mode 100644 index e6f613651..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/DataSetQueryParam.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.tencent.supersonic.headless.core.pojo; - -import com.tencent.supersonic.headless.api.pojo.MetricTable; -import lombok.Data; - -import java.util.List; - -@Data -public class DataSetQueryParam { - private String sql = ""; - private List tables; - private boolean supportWith = true; - private boolean withAlias = true; -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/MetricQueryParam.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/MetricQueryParam.java deleted file mode 100644 index 2a3bc8b8b..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/MetricQueryParam.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.tencent.supersonic.headless.core.pojo; - -import com.tencent.supersonic.common.pojo.ColumnOrder; -import lombok.Data; - -import java.util.List; - -@Data -public class MetricQueryParam { - - private List metrics; - private List dimensions; - private String where; - private Long limit; - private List order; - private boolean nativeQuery = false; -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java index 33266cc7a..2a6ffd63e 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/QueryStatement.java @@ -1,34 +1,25 @@ package com.tencent.supersonic.headless.core.pojo; -import com.tencent.supersonic.headless.api.pojo.QueryParam; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Ontology; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam; import lombok.Data; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Triple; -import java.util.List; - @Data public class QueryStatement { private Long dataSetId; - private List modelIds; private String sql; private String errMsg; - private QueryParam queryParam; - private MetricQueryParam metricQueryParam; - private DataSetQueryParam dataSetQueryParam; + private StructQueryParam structQueryParam; + private SqlQueryParam sqlQueryParam; + private OntologyQueryParam ontologyQueryParam; private Integer status = 0; private Boolean isS2SQL = false; - private List> timeRanges; private Boolean enableOptimize = true; private Triple minMaxTime; - private String dataSetSql; - private String dataSetAlias; - private String dataSetSimplifySql; - private Boolean enableLimitWrapper = false; private Ontology ontology; private SemanticSchemaResp semanticSchemaResp; private Integer limit = 1000; @@ -41,9 +32,4 @@ public class QueryStatement { public boolean isTranslated() { return isTranslated != null && isTranslated && isOk(); } - - public QueryStatement error(String msg) { - this.setErrMsg(msg); - return this; - } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/SqlQueryParam.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/SqlQueryParam.java new file mode 100644 index 000000000..086397e88 --- /dev/null +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/SqlQueryParam.java @@ -0,0 +1,12 @@ +package com.tencent.supersonic.headless.core.pojo; + +import lombok.Data; + +@Data +public class SqlQueryParam { + private String sql; + private String table; + private boolean supportWith = true; + private boolean withAlias = true; + private String simplifiedSql; +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryParam.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/StructQueryParam.java similarity index 57% rename from headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryParam.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/StructQueryParam.java index 401d18204..8fd129885 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/QueryParam.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/StructQueryParam.java @@ -1,21 +1,18 @@ -package com.tencent.supersonic.headless.api.pojo; +package com.tencent.supersonic.headless.core.pojo; import com.tencent.supersonic.common.pojo.Aggregator; -import com.tencent.supersonic.common.pojo.ColumnOrder; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.Filter; import com.tencent.supersonic.common.pojo.Order; import com.tencent.supersonic.common.pojo.enums.QueryType; +import com.tencent.supersonic.headless.api.pojo.Param; import lombok.Data; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; @Data -public class QueryParam { - // struct +public class StructQueryParam { private List groups = new ArrayList(); private List aggregators = new ArrayList(); private List orders = new ArrayList(); @@ -24,17 +21,5 @@ public class QueryParam { private DateConf dateInfo; private Long limit = 2000L; private QueryType queryType; - private String s2SQL; - private String correctS2SQL; - private Long dataSetId; - private String dataSetName; - private Set modelIds = new HashSet<>(); private List params = new ArrayList<>(); - - // metric - private List metrics = new ArrayList(); - private List dimensions; - private String where; - private List order; - private boolean nativeQuery = false; } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java index a1f7a5f48..79bfb478c 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java @@ -1,507 +1,97 @@ package com.tencent.supersonic.headless.core.translator; import com.tencent.supersonic.common.calcite.SqlMergeWithUtils; -import com.tencent.supersonic.common.jsqlparser.SqlRemoveHelper; -import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper; -import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper; import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; -import com.tencent.supersonic.common.pojo.Aggregator; -import com.tencent.supersonic.common.pojo.Constants; -import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.pojo.enums.EngineType; -import com.tencent.supersonic.common.pojo.enums.QueryType; -import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; -import com.tencent.supersonic.common.util.StringUtil; -import com.tencent.supersonic.headless.api.pojo.Measure; -import com.tencent.supersonic.headless.api.pojo.MetricTable; -import com.tencent.supersonic.headless.api.pojo.QueryParam; -import com.tencent.supersonic.headless.api.pojo.SchemaItem; -import com.tencent.supersonic.headless.api.pojo.enums.AggOption; -import com.tencent.supersonic.headless.api.pojo.enums.MetricType; -import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; -import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; -import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; -import com.tencent.supersonic.headless.api.pojo.response.MetricResp; -import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelResp; -import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; -import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor; -import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory; -import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam; -import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Ontology; +import com.tencent.supersonic.headless.core.pojo.SqlQueryParam; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam; import com.tencent.supersonic.headless.core.translator.converter.QueryConverter; import com.tencent.supersonic.headless.core.utils.ComponentFactory; -import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; -import org.springframework.beans.BeanUtils; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; import java.util.stream.Collectors; -import java.util.stream.Stream; @Component @Slf4j public class DefaultSemanticTranslator implements SemanticTranslator { - @Autowired - private SqlGenerateUtils sqlGenerateUtils; - public void translate(QueryStatement queryStatement) { if (queryStatement.isTranslated()) { return; } - try { - preprocess(queryStatement); - parse(queryStatement); - optimize(queryStatement); + for (QueryConverter converter : ComponentFactory.getQueryConverters()) { + if (converter.accept(queryStatement)) { + log.debug("QueryConverter accept [{}]", converter.getClass().getName()); + converter.convert(queryStatement); + } + } + doOntologyParse(queryStatement); + + if (StringUtils.isNotBlank(queryStatement.getSqlQueryParam().getSimplifiedSql())) { + queryStatement.setSql(queryStatement.getSqlQueryParam().getSimplifiedSql()); + } + if (StringUtils.isBlank(queryStatement.getSql())) { + throw new RuntimeException("parse exception: " + queryStatement.getErrMsg()); + } + if (!SqlSelectHelper.hasLimit(queryStatement.getSql())) { + queryStatement + .setSql(queryStatement.getSql() + " limit " + queryStatement.getLimit()); + } + + for (QueryOptimizer queryOptimizer : ComponentFactory.getQueryOptimizers()) { + queryOptimizer.rewrite(queryStatement); + } } catch (Exception e) { queryStatement.setErrMsg(e.getMessage()); + log.error("Failed to translate semantic query [{}]", e); } } - private void parse(QueryStatement queryStatement) throws Exception { - QueryParam queryParam = queryStatement.getQueryParam(); - if (Objects.isNull(queryStatement.getDataSetQueryParam())) { - queryStatement.setDataSetQueryParam(new DataSetQueryParam()); - } - if (Objects.isNull(queryStatement.getMetricQueryParam())) { - queryStatement.setMetricQueryParam(new MetricQueryParam()); + private void doOntologyParse(QueryStatement queryStatement) throws Exception { + OntologyQueryParam ontologyQueryParam = queryStatement.getOntologyQueryParam(); + SqlQueryParam sqlQueryParam = queryStatement.getSqlQueryParam(); + log.info("parse with ontology: [{}]", ontologyQueryParam); + ComponentFactory.getQueryParser().parse(queryStatement); + + if (!queryStatement.isOk()) { + throw new Exception(String.format("parse table [%s] error [%s]", + sqlQueryParam.getTable(), queryStatement.getErrMsg())); } - log.debug("SemanticConverter before [{}]", queryParam); - for (QueryConverter headlessConverter : ComponentFactory.getQueryConverters()) { - if (headlessConverter.accept(queryStatement)) { - log.debug("SemanticConverter accept [{}]", headlessConverter.getClass().getName()); - headlessConverter.convert(queryStatement); + List> tables = new ArrayList<>(); + tables.add(Pair.of(sqlQueryParam.getTable(), queryStatement.getSql())); + if (sqlQueryParam.isSupportWith()) { + EngineType engineType = + EngineType.fromString(queryStatement.getOntology().getDatabase().getType()); + if (!SqlMergeWithUtils.hasWith(engineType, sqlQueryParam.getSql())) { + String withSql = "with " + tables.stream() + .map(t -> String.format("%s as (%s)", t.getLeft(), t.getRight())) + .collect(Collectors.joining(",")) + "\n" + sqlQueryParam.getSql(); + queryStatement.setSql(withSql); + } else { + List parentTableList = + tables.stream().map(Pair::getLeft).collect(Collectors.toList()); + List parentSqlList = + tables.stream().map(Pair::getRight).collect(Collectors.toList()); + String mergeSql = SqlMergeWithUtils.mergeWith(engineType, sqlQueryParam.getSql(), + parentSqlList, parentTableList); + queryStatement.setSql(mergeSql); } - } - log.debug("SemanticConverter after {} {} {}", queryParam, - queryStatement.getDataSetQueryParam(), queryStatement.getMetricQueryParam()); - - if (!queryStatement.getDataSetQueryParam().getSql().isEmpty()) { - doParse(queryStatement.getDataSetQueryParam(), queryStatement); } else { - queryStatement.getMetricQueryParam() - .setNativeQuery(queryParam.getQueryType().isNativeAggQuery()); - doParse(queryStatement, - AggOption.getAggregation(queryStatement.getMetricQueryParam().isNativeQuery())); - } - - if (StringUtils.isEmpty(queryStatement.getSql())) { - throw new RuntimeException("parse Exception: " + queryStatement.getErrMsg()); - } - if (StringUtils.isNotBlank(queryStatement.getSql()) - && !SqlSelectHelper.hasLimit(queryStatement.getSql())) { - String querySql = - queryStatement.getSql() + " limit " + queryStatement.getLimit().toString(); - queryStatement.setSql(querySql); - } - } - - private QueryStatement doParse(DataSetQueryParam dataSetQueryParam, - QueryStatement queryStatement) { - log.info("parse dataSetQuery [{}] ", dataSetQueryParam); - Ontology ontology = queryStatement.getOntology(); - EngineType engineType = EngineType.fromString(ontology.getDatabase().getType()); - try { - if (!CollectionUtils.isEmpty(dataSetQueryParam.getTables())) { - List tables = new ArrayList<>(); - boolean isSingleTable = dataSetQueryParam.getTables().size() == 1; - for (MetricTable metricTable : dataSetQueryParam.getTables()) { - QueryStatement tableSql = parserSql(metricTable, isSingleTable, - dataSetQueryParam, queryStatement); - if (isSingleTable && StringUtils.isNotBlank(tableSql.getDataSetSimplifySql())) { - queryStatement.setSql(tableSql.getDataSetSimplifySql()); - queryStatement.setDataSetQueryParam(dataSetQueryParam); - return queryStatement; - } - tables.add(new String[] {metricTable.getAlias(), tableSql.getSql()}); - } - if (!tables.isEmpty()) { - String sql; - if (dataSetQueryParam.isSupportWith()) { - if (!SqlMergeWithUtils.hasWith(engineType, dataSetQueryParam.getSql())) { - sql = "with " - + tables.stream() - .map(t -> String.format("%s as (%s)", t[0], t[1])) - .collect(Collectors.joining(",")) - + "\n" + dataSetQueryParam.getSql(); - } else { - List parentWithNameList = tables.stream().map(table -> table[0]) - .collect(Collectors.toList()); - List parentSqlList = tables.stream().map(table -> table[1]) - .collect(Collectors.toList()); - sql = SqlMergeWithUtils.mergeWith(engineType, - dataSetQueryParam.getSql(), parentSqlList, parentWithNameList); - } - } else { - sql = dataSetQueryParam.getSql(); - for (String[] tb : tables) { - sql = StringUtils.replace(sql, tb[0], "(" + tb[1] + ") " - + (dataSetQueryParam.isWithAlias() ? "" : tb[0]), -1); - } - } - queryStatement.setSql(sql); - queryStatement.setDataSetQueryParam(dataSetQueryParam); - return queryStatement; - } + String dsSql = sqlQueryParam.getSql(); + for (Pair tb : tables) { + dsSql = StringUtils.replace(dsSql, tb.getLeft(), "(" + tb.getRight() + ") " + + (sqlQueryParam.isWithAlias() ? "" : tb.getLeft()), -1); } - } catch (Exception e) { - log.error("physicalSql error {}", e); - queryStatement.setErrMsg(e.getMessage()); - } - return queryStatement; - } - - private QueryStatement doParse(QueryStatement queryStatement, AggOption isAgg) { - MetricQueryParam metricQueryParam = queryStatement.getMetricQueryParam(); - log.info("parse metricQuery [{}] isAgg [{}]", metricQueryParam, isAgg); - try { - ComponentFactory.getQueryParser().parse(queryStatement, isAgg); - } catch (Exception e) { - queryStatement.setErrMsg(e.getMessage()); - log.error("parser error metricQueryReq[{}] error [{}]", metricQueryParam, e); - } - return queryStatement; - } - - private QueryStatement parserSql(MetricTable metricTable, Boolean isSingleMetricTable, - DataSetQueryParam dataSetQueryParam, QueryStatement queryStatement) throws Exception { - MetricQueryParam metricQueryParam = new MetricQueryParam(); - metricQueryParam.setMetrics(metricTable.getMetrics()); - metricQueryParam.setDimensions(metricTable.getDimensions()); - metricQueryParam.setWhere(StringUtil.formatSqlQuota(metricTable.getWhere())); - metricQueryParam.setNativeQuery(!AggOption.isAgg(metricTable.getAggOption())); - - QueryStatement tableSql = new QueryStatement(); - tableSql.setIsS2SQL(false); - tableSql.setMetricQueryParam(metricQueryParam); - tableSql.setMinMaxTime(queryStatement.getMinMaxTime()); - tableSql.setEnableOptimize(queryStatement.getEnableOptimize()); - tableSql.setDataSetId(queryStatement.getDataSetId()); - tableSql.setOntology(queryStatement.getOntology()); - if (isSingleMetricTable) { - tableSql.setDataSetSql(dataSetQueryParam.getSql()); - tableSql.setDataSetAlias(metricTable.getAlias()); - } - tableSql = doParse(tableSql, metricTable.getAggOption()); - if (!tableSql.isOk()) { - throw new Exception(String.format("parser table [%s] error [%s]", - metricTable.getAlias(), tableSql.getErrMsg())); - } - return tableSql; - } - - private void optimize(QueryStatement queryStatement) { - for (QueryOptimizer queryOptimizer : ComponentFactory.getQueryOptimizers()) { - queryOptimizer.rewrite(queryStatement); + queryStatement.setSql(dsSql); } } - private void preprocess(QueryStatement queryStatement) { - if (StringUtils.isBlank(queryStatement.getSql())) { - return; - } - SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp(); - - convertNameToBizName(queryStatement); - rewriteFunction(queryStatement); - queryStatement.setSql(SqlRemoveHelper.removeUnderscores(queryStatement.getSql())); - - String tableName = SqlSelectHelper.getTableName(queryStatement.getSql()); - if (StringUtils.isEmpty(tableName)) { - return; - } - // correct order item is same as agg alias - String reqSql = queryStatement.getSql(); - queryStatement.setSql(SqlReplaceHelper.replaceAggAliasOrderItem(queryStatement.getSql())); - log.debug("replaceOrderAggSameAlias {} -> {}", reqSql, queryStatement.getSql()); - // 5.build MetricTables - List allFields = SqlSelectHelper.getAllSelectFields(queryStatement.getSql()); - List metricSchemas = getMetrics(semanticSchemaResp, allFields); - List metrics = - metricSchemas.stream().map(SchemaItem::getBizName).collect(Collectors.toList()); - Set dimensions = getDimensions(semanticSchemaResp, allFields); - QueryStructReq queryStructReq = new QueryStructReq(); - - MetricTable metricTable = new MetricTable(); - metricTable.getMetrics().addAll(metrics); - metricTable.getDimensions().addAll(dimensions); - metricTable.setAlias(tableName.toLowerCase()); - // if metric empty , fill model default - if (CollectionUtils.isEmpty(metricTable.getMetrics())) { - metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName( - getDefaultModel(semanticSchemaResp, metricTable.getDimensions()))); - } else { - queryStructReq.getAggregators() - .addAll(metricTable.getMetrics().stream() - .map(m -> new Aggregator(m, AggOperatorEnum.UNKNOWN)) - .collect(Collectors.toList())); - } - AggOption aggOption = getAggOption(queryStatement, metricSchemas); - metricTable.setAggOption(aggOption); - List tables = new ArrayList<>(); - tables.add(metricTable); - - // 6.build ParseSqlReq - DataSetQueryParam datasetQueryParam = new DataSetQueryParam(); - datasetQueryParam.setTables(tables); - datasetQueryParam.setSql(queryStatement.getSql()); - DatabaseResp database = semanticSchemaResp.getDatabaseResp(); - if (!sqlGenerateUtils.isSupportWith(EngineType.fromString(database.getType().toUpperCase()), - database.getVersion())) { - datasetQueryParam.setSupportWith(false); - datasetQueryParam.setWithAlias(false); - } - - // 7. do deriveMetric - generateDerivedMetric(semanticSchemaResp, aggOption, datasetQueryParam); - - // 8.physicalSql by ParseSqlReq - // queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(queryStatement.getSql())); - queryStructReq.setDataSetId(queryStatement.getDataSetId()); - queryStructReq.setQueryType(getQueryType(aggOption)); - log.debug("QueryReqConverter queryStructReq[{}]", queryStructReq); - QueryParam queryParam = new QueryParam(); - BeanUtils.copyProperties(queryStructReq, queryParam); - queryStatement.setQueryParam(queryParam); - queryStatement.setDataSetQueryParam(datasetQueryParam); - // queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq)); - } - - private AggOption getAggOption(QueryStatement queryStatement, - List metricSchemas) { - String sql = queryStatement.getSql(); - if (!SqlSelectFunctionHelper.hasAggregateFunction(sql) && !SqlSelectHelper.hasGroupBy(sql) - && !SqlSelectHelper.hasWith(sql) && !SqlSelectHelper.hasSubSelect(sql)) { - log.debug("getAggOption simple sql set to DEFAULT"); - return AggOption.DEFAULT; - } - // if there is no group by in S2SQL,set MetricTable's aggOption to "NATIVE" - // if there is count() in S2SQL,set MetricTable's aggOption to "NATIVE" - if (!SqlSelectFunctionHelper.hasAggregateFunction(sql) - || SqlSelectFunctionHelper.hasFunction(sql, "count") - || SqlSelectFunctionHelper.hasFunction(sql, "count_distinct")) { - return AggOption.OUTER; - } - // if (queryStatement.isInnerLayerNative()) { - // return AggOption.NATIVE; - // } - if (SqlSelectHelper.hasSubSelect(sql) || SqlSelectHelper.hasWith(sql) - || SqlSelectHelper.hasGroupBy(sql)) { - return AggOption.OUTER; - } - long defaultAggNullCnt = metricSchemas.stream().filter( - m -> Objects.isNull(m.getDefaultAgg()) || StringUtils.isBlank(m.getDefaultAgg())) - .count(); - if (defaultAggNullCnt > 0) { - log.debug("getAggOption find null defaultAgg metric set to NATIVE"); - return AggOption.OUTER; - } - return AggOption.DEFAULT; - } - - private void convertNameToBizName(QueryStatement queryStatement) { - SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp(); - Map fieldNameToBizNameMap = getFieldNameToBizNameMap(semanticSchemaResp); - String sql = queryStatement.getSql(); - log.debug("dataSetId:{},convert name to bizName before:{}", queryStatement.getDataSetId(), - sql); - sql = SqlReplaceHelper.replaceSqlByPositions(sql); - log.debug("replaceSqlByPositions:{}", sql); - sql = SqlReplaceHelper.replaceFields(sql, fieldNameToBizNameMap, true); - log.debug("dataSetId:{},convert name to bizName after:{}", queryStatement.getDataSetId(), - sql); - sql = SqlReplaceHelper.replaceTable(sql, - Constants.TABLE_PREFIX + queryStatement.getDataSetId()); - log.debug("replaceTableName after:{}", sql); - queryStatement.setSql(sql); - } - - private Set getDimensions(SemanticSchemaResp semanticSchemaResp, - List allFields) { - Map dimensionLowerToNameMap = semanticSchemaResp.getDimensions().stream() - .collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(), - SchemaItem::getBizName, (k1, k2) -> k1)); - dimensionLowerToNameMap.put(TimeDimensionEnum.DAY.getName(), - TimeDimensionEnum.DAY.getName()); - return allFields.stream() - .filter(entry -> dimensionLowerToNameMap.containsKey(entry.toLowerCase())) - .map(entry -> dimensionLowerToNameMap.get(entry.toLowerCase())) - .collect(Collectors.toSet()); - } - - private List getMetrics(SemanticSchemaResp semanticSchemaResp, - List allFields) { - Map metricLowerToNameMap = - semanticSchemaResp.getMetrics().stream().collect(Collectors - .toMap(entry -> entry.getBizName().toLowerCase(), entry -> entry)); - return allFields.stream() - .filter(entry -> metricLowerToNameMap.containsKey(entry.toLowerCase())) - .map(entry -> metricLowerToNameMap.get(entry.toLowerCase())) - .collect(Collectors.toList()); - } - - private void rewriteFunction(QueryStatement queryStatement) { - SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp(); - DatabaseResp database = semanticSchemaResp.getDatabaseResp(); - if (Objects.isNull(database) || Objects.isNull(database.getType())) { - return; - } - String type = database.getType(); - DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(type.toLowerCase()); - if (Objects.nonNull(engineAdaptor)) { - String functionNameCorrector = - engineAdaptor.functionNameCorrector(queryStatement.getSql()); - queryStatement.setSql(functionNameCorrector); - } - } - - protected Map getFieldNameToBizNameMap(SemanticSchemaResp semanticSchemaResp) { - // support fieldName and field alias to bizName - Map dimensionResults = semanticSchemaResp.getDimensions().stream().flatMap( - entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName())) - .collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1)); - - Map metricResults = semanticSchemaResp.getMetrics().stream().flatMap( - entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName())) - .collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1)); - - dimensionResults.putAll(TimeDimensionEnum.getChNameToNameMap()); - dimensionResults.putAll(TimeDimensionEnum.getNameToNameMap()); - dimensionResults.putAll(metricResults); - return dimensionResults; - } - - private Stream> getPairStream(String aliasStr, String name, - String bizName) { - Set> elements = new HashSet<>(); - elements.add(Pair.of(name, bizName)); - if (StringUtils.isNotBlank(aliasStr)) { - List aliasList = SchemaItem.getAliasList(aliasStr); - for (String alias : aliasList) { - elements.add(Pair.of(alias, bizName)); - } - } - return elements.stream(); - } - - private QueryType getQueryType(AggOption aggOption) { - boolean isAgg = AggOption.isAgg(aggOption); - QueryType queryType = QueryType.DETAIL; - if (isAgg) { - queryType = QueryType.AGGREGATE; - } - return queryType; - } - - private void generateDerivedMetric(SemanticSchemaResp semanticSchemaResp, AggOption aggOption, - DataSetQueryParam viewQueryParam) { - String sql = viewQueryParam.getSql(); - for (MetricTable metricTable : viewQueryParam.getTables()) { - Set measures = new HashSet<>(); - Map replaces = generateDerivedMetric(semanticSchemaResp, aggOption, - metricTable.getMetrics(), metricTable.getDimensions(), measures); - - if (!CollectionUtils.isEmpty(replaces)) { - // metricTable sql use measures replace metric - sql = SqlReplaceHelper.replaceSqlByExpression(sql, replaces); - metricTable.setAggOption(AggOption.NATIVE); - // metricTable use measures replace metric - if (!CollectionUtils.isEmpty(measures)) { - metricTable.setMetrics(new ArrayList<>(measures)); - } else { - // empty measure , fill default - metricTable.setMetrics(new ArrayList<>()); - metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName( - getDefaultModel(semanticSchemaResp, metricTable.getDimensions()))); - } - } - } - viewQueryParam.setSql(sql); - } - - private Map generateDerivedMetric(SemanticSchemaResp semanticSchemaResp, - AggOption aggOption, List metrics, List dimensions, - Set measures) { - Map result = new HashMap<>(); - List metricResps = semanticSchemaResp.getMetrics(); - List dimensionResps = semanticSchemaResp.getDimensions(); - - // Check if any metric is derived - boolean hasDerivedMetrics = - metricResps.stream().anyMatch(m -> metrics.contains(m.getBizName()) && MetricType - .isDerived(m.getMetricDefineType(), m.getMetricDefineByMeasureParams())); - if (!hasDerivedMetrics) { - return result; - } - - log.debug("begin to generateDerivedMetric {} [{}]", aggOption, metrics); - - Set allFields = new HashSet<>(); - Map allMeasures = new HashMap<>(); - semanticSchemaResp.getModelResps().forEach(modelResp -> { - allFields.addAll(modelResp.getFieldList()); - if (modelResp.getModelDetail().getMeasures() != null) { - modelResp.getModelDetail().getMeasures() - .forEach(measure -> allMeasures.put(measure.getBizName(), measure)); - } - }); - - Set derivedDimensions = new HashSet<>(); - Set derivedMetrics = new HashSet<>(); - Map visitedMetrics = new HashMap<>(); - - for (MetricResp metricResp : metricResps) { - if (metrics.contains(metricResp.getBizName())) { - boolean isDerived = MetricType.isDerived(metricResp.getMetricDefineType(), - metricResp.getMetricDefineByMeasureParams()); - if (isDerived) { - String expr = sqlGenerateUtils.generateDerivedMetric(metricResps, allFields, - allMeasures, dimensionResps, sqlGenerateUtils.getExpr(metricResp), - metricResp.getMetricDefineType(), aggOption, visitedMetrics, - derivedMetrics, derivedDimensions); - result.put(metricResp.getBizName(), expr); - log.debug("derived metric {}->{}", metricResp.getBizName(), expr); - } else { - measures.add(metricResp.getBizName()); - } - } - } - - measures.addAll(derivedMetrics); - derivedDimensions.stream().filter(dimension -> !dimensions.contains(dimension)) - .forEach(dimensions::add); - - return result; - } - - private String getDefaultModel(SemanticSchemaResp semanticSchemaResp, List dimensions) { - if (!CollectionUtils.isEmpty(dimensions)) { - Map modelMatchCnt = new HashMap<>(); - for (ModelResp modelResp : semanticSchemaResp.getModelResps()) { - modelMatchCnt.put(modelResp.getBizName(), modelResp.getModelDetail().getDimensions() - .stream().filter(d -> dimensions.contains(d.getBizName())).count()); - } - return modelMatchCnt.entrySet().stream() - .sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())) - .map(m -> m.getKey()).findFirst().orElse(""); - } - return semanticSchemaResp.getModelResps().get(0).getBizName(); - } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DetailQueryOptimizer.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DetailQueryOptimizer.java index bf84cbb2b..3a30a91f9 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DetailQueryOptimizer.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DetailQueryOptimizer.java @@ -1,14 +1,12 @@ package com.tencent.supersonic.headless.core.translator; -import com.tencent.supersonic.headless.api.pojo.QueryParam; import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import com.tencent.supersonic.headless.core.pojo.StructQueryParam; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; import java.util.Objects; -import java.util.stream.Collectors; /** Remove the default metric added by the system when the query only has dimensions */ @Slf4j @@ -17,26 +15,26 @@ public class DetailQueryOptimizer implements QueryOptimizer { @Override public void rewrite(QueryStatement queryStatement) { - QueryParam queryParam = queryStatement.getQueryParam(); + StructQueryParam structQueryParam = queryStatement.getStructQueryParam(); String sqlRaw = queryStatement.getSql().trim(); if (StringUtils.isEmpty(sqlRaw)) { throw new RuntimeException("sql is empty or null"); } log.debug("before handleNoMetric, sql:{}", sqlRaw); - if (isDetailQuery(queryParam)) { - if (queryParam.getMetrics().size() == 0 - && !CollectionUtils.isEmpty(queryParam.getGroups())) { - String sqlForm = "select %s from ( %s ) src_no_metric"; - String sql = String.format(sqlForm, - queryParam.getGroups().stream().collect(Collectors.joining(",")), sqlRaw); - queryStatement.setSql(sql); - } - } + // if (isDetailQuery(structQueryParam)) { + // if (!CollectionUtils.isEmpty(structQueryParam.getGroups())) { + // String sqlForm = "select %s from ( %s ) src_no_metric"; + // String sql = String.format(sqlForm, + // structQueryParam.getGroups().stream().collect(Collectors.joining(",")), + // sqlRaw); + // queryStatement.setSql(sql); + // } + // } log.debug("after handleNoMetric, sql:{}", queryStatement.getSql()); } - public boolean isDetailQuery(QueryParam queryParam) { - return Objects.nonNull(queryParam) && queryParam.getQueryType().isNativeAggQuery() - && CollectionUtils.isEmpty(queryParam.getMetrics()); + public boolean isDetailQuery(StructQueryParam structQueryParam) { + return Objects.nonNull(structQueryParam) + && structQueryParam.getQueryType().isNativeAggQuery(); } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/QueryParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/QueryParser.java index b15212c65..344335af0 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/QueryParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/QueryParser.java @@ -1,9 +1,8 @@ package com.tencent.supersonic.headless.core.translator; -import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.core.pojo.QueryStatement; /** A query parser generates physical SQL for the QueryStatement. */ public interface QueryParser { - void parse(QueryStatement queryStatement, AggOption aggOption) throws Exception; + void parse(QueryStatement queryStatement) throws Exception; } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/CalciteQueryParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/CalciteQueryParser.java index ce0d27a11..8a787b92c 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/CalciteQueryParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/CalciteQueryParser.java @@ -1,6 +1,5 @@ package com.tencent.supersonic.headless.core.translator.calcite; -import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.translator.QueryParser; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Ontology; @@ -16,7 +15,7 @@ import org.springframework.stereotype.Component; public class CalciteQueryParser implements QueryParser { @Override - public void parse(QueryStatement queryStatement, AggOption isAgg) throws Exception { + public void parse(QueryStatement queryStatement) throws Exception { Ontology ontology = queryStatement.getOntology(); if (ontology == null) { queryStatement.setErrMsg("No ontology could be found"); @@ -29,7 +28,7 @@ public class CalciteQueryParser implements QueryParser { .enableOptimize(queryStatement.getEnableOptimize()).build()) .build(); SqlBuilder sqlBuilder = new SqlBuilder(semanticSchema); - sqlBuilder.build(queryStatement, isAgg); + sqlBuilder.build(queryStatement); } } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricTable.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/OntologyQueryParam.java similarity index 57% rename from headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricTable.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/OntologyQueryParam.java index 9740891de..b399eb486 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/MetricTable.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/s2sql/OntologyQueryParam.java @@ -1,17 +1,19 @@ -package com.tencent.supersonic.headless.api.pojo; +package com.tencent.supersonic.headless.core.translator.calcite.s2sql; import com.google.common.collect.Lists; +import com.tencent.supersonic.common.pojo.ColumnOrder; import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import lombok.Data; import java.util.List; @Data -public class MetricTable { - - private String alias; +public class OntologyQueryParam { private List metrics = Lists.newArrayList(); private List dimensions = Lists.newArrayList(); private String where; + private Long limit; + private List order; + private boolean nativeQuery = false; private AggOption aggOption = AggOption.DEFAULT; } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/SqlBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/SqlBuilder.java index 971886ed4..6477245f9 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/SqlBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/SqlBuilder.java @@ -1,14 +1,13 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql; import com.tencent.supersonic.common.calcite.Configuration; -import com.tencent.supersonic.common.calcite.SqlMergeWithUtils; import com.tencent.supersonic.common.pojo.enums.EngineType; import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.core.pojo.Database; -import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.DataModelNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.SemanticNode; import com.tencent.supersonic.headless.core.translator.calcite.sql.render.FilterRender; @@ -17,23 +16,16 @@ import com.tencent.supersonic.headless.core.translator.calcite.sql.render.Render import com.tencent.supersonic.headless.core.translator.calcite.sql.render.SourceRender; import lombok.extern.slf4j.Slf4j; import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParser; import org.apache.calcite.sql.validate.SqlValidatorScope; -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; -import java.util.ListIterator; -import java.util.Objects; +import java.util.*; -/** parsing from query dimensions and metrics */ @Slf4j public class SqlBuilder { private final S2CalciteSchema schema; - private MetricQueryParam metricQueryParam; + private OntologyQueryParam ontologyQueryParam; private SqlValidatorScope scope; private SqlNode parserNode; private boolean isAgg = false; @@ -43,45 +35,32 @@ public class SqlBuilder { this.schema = schema; } - public void build(QueryStatement queryStatement, AggOption aggOption) throws Exception { - this.metricQueryParam = queryStatement.getMetricQueryParam(); - if (metricQueryParam.getMetrics() == null) { - metricQueryParam.setMetrics(new ArrayList<>()); + public void build(QueryStatement queryStatement) throws Exception { + this.ontologyQueryParam = queryStatement.getOntologyQueryParam(); + if (ontologyQueryParam.getMetrics() == null) { + ontologyQueryParam.setMetrics(new ArrayList<>()); } - if (metricQueryParam.getDimensions() == null) { - metricQueryParam.setDimensions(new ArrayList<>()); + if (ontologyQueryParam.getDimensions() == null) { + ontologyQueryParam.setDimensions(new ArrayList<>()); } - if (metricQueryParam.getLimit() == null) { - metricQueryParam.setLimit(0L); + if (ontologyQueryParam.getLimit() == null) { + ontologyQueryParam.setLimit(0L); } - this.aggOption = aggOption; + this.aggOption = ontologyQueryParam.getAggOption(); buildParseNode(); Database database = queryStatement.getOntology().getDatabase(); EngineType engineType = EngineType.fromString(database.getType()); optimizeParseNode(engineType); String sql = getSql(engineType); - queryStatement.setSql(sql); - if (Objects.nonNull(queryStatement.getEnableOptimize()) - && queryStatement.getEnableOptimize() - && Objects.nonNull(queryStatement.getDataSetAlias()) - && !queryStatement.getDataSetAlias().isEmpty()) { - // simplify model sql with query sql - String simplifySql = rewrite(getSqlByDataSet(engineType, sql, - queryStatement.getDataSetSql(), queryStatement.getDataSetAlias()), engineType); - if (Objects.nonNull(simplifySql) && !simplifySql.isEmpty()) { - log.debug("simplifySql [{}]", simplifySql); - queryStatement.setDataSetSimplifySql(simplifySql); - } - } } private void buildParseNode() throws Exception { // find the match Datasource scope = SchemaBuilder.getScope(schema); List dataModels = - DataModelNode.getRelatedDataModels(scope, schema, metricQueryParam); + DataModelNode.getRelatedDataModels(scope, schema, ontologyQueryParam); if (dataModels == null || dataModels.isEmpty()) { throw new Exception("data model not found"); } @@ -98,14 +77,14 @@ public class SqlBuilder { while (it.hasNext()) { Renderer renderer = it.next(); if (previous != null) { - previous.render(metricQueryParam, dataModels, scope, schema, !isAgg); + previous.render(ontologyQueryParam, dataModels, scope, schema, !isAgg); renderer.setTable(previous .builderAs(DataModelNode.getNames(dataModels) + "_" + String.valueOf(i))); i++; } previous = renderer; } - builders.getLast().render(metricQueryParam, dataModels, scope, schema, !isAgg); + builders.getLast().render(ontologyQueryParam, dataModels, scope, schema, !isAgg); parserNode = builders.getLast().builder(); } @@ -116,7 +95,7 @@ public class SqlBuilder { // default by dataModel time aggregation if (Objects.nonNull(dataModel.getAggTime()) && !dataModel.getAggTime() .equalsIgnoreCase(Constants.DIMENSION_TYPE_TIME_GRANULARITY_NONE)) { - if (!metricQueryParam.isNativeQuery()) { + if (!ontologyQueryParam.isNativeQuery()) { return true; } } @@ -164,13 +143,4 @@ public class SqlBuilder { } } - private String getSqlByDataSet(EngineType engineType, String parentSql, String dataSetSql, - String parentAlias) throws SqlParseException { - if (!SqlMergeWithUtils.hasWith(engineType, dataSetSql)) { - return String.format("with %s as (%s) %s", parentAlias, parentSql, dataSetSql); - } - return SqlMergeWithUtils.mergeWith(engineType, dataSetSql, - Collections.singletonList(parentSql), Collections.singletonList(parentAlias)); - } - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/DataModelNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/DataModelNode.java index 73c5c422e..a648b4122 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/DataModelNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/node/DataModelNode.java @@ -4,13 +4,13 @@ import com.google.common.collect.Lists; import com.tencent.supersonic.common.calcite.Configuration; import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.common.pojo.enums.EngineType; -import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.JoinRelation; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Measure; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam; import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema; import com.tencent.supersonic.headless.core.translator.calcite.sql.SchemaBuilder; import lombok.extern.slf4j.Slf4j; @@ -150,7 +150,7 @@ public class DataModelNode extends SemanticNode { } public static void getQueryDimensionMeasure(S2CalciteSchema schema, - MetricQueryParam metricCommand, Set queryDimension, List measures) { + OntologyQueryParam metricCommand, Set queryDimension, List measures) { queryDimension.addAll(metricCommand.getDimensions().stream() .map(d -> d.contains(Constants.DIMENSION_IDENTIFY) ? d.split(Constants.DIMENSION_IDENTIFY)[1] @@ -166,7 +166,7 @@ public class DataModelNode extends SemanticNode { } public static void mergeQueryFilterDimensionMeasure(S2CalciteSchema schema, - MetricQueryParam metricCommand, Set queryDimension, List measures, + OntologyQueryParam metricCommand, Set queryDimension, List measures, SqlValidatorScope scope) throws Exception { EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType()); if (Objects.nonNull(metricCommand.getWhere()) && !metricCommand.getWhere().isEmpty()) { @@ -192,7 +192,7 @@ public class DataModelNode extends SemanticNode { } public static List getRelatedDataModels(SqlValidatorScope scope, - S2CalciteSchema schema, MetricQueryParam metricCommand) throws Exception { + S2CalciteSchema schema, OntologyQueryParam metricCommand) throws Exception { List dataModels = new ArrayList<>(); // check by metric @@ -208,7 +208,7 @@ public class DataModelNode extends SemanticNode { sourceMeasure.retainAll(measures); dataSourceMeasures.put(entry.getKey(), sourceMeasure.size()); } - log.info("dataSourceMeasures [{}]", dataSourceMeasures); + log.info("metrics: [{}]", dataSourceMeasures); Optional> base = dataSourceMeasures.entrySet().stream() .sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())).findFirst(); if (base.isPresent()) { @@ -267,7 +267,7 @@ public class DataModelNode extends SemanticNode { } private static boolean checkMatch(Set sourceMeasure, Set queryDimension, - List measures, Set dimension, MetricQueryParam metricCommand, + List measures, Set dimension, OntologyQueryParam metricCommand, SqlValidatorScope scope, EngineType engineType) throws Exception { boolean isAllMatch = true; sourceMeasure.retainAll(measures); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/FilterRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/FilterRender.java index fd058739a..6ad65aa1b 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/FilterRender.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/FilterRender.java @@ -1,10 +1,10 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql.render; import com.tencent.supersonic.common.pojo.enums.EngineType; -import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam; import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema; import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.FilterNode; @@ -26,7 +26,7 @@ import java.util.stream.Collectors; public class FilterRender extends Renderer { @Override - public void render(MetricQueryParam metricCommand, List dataModels, + public void render(OntologyQueryParam metricCommand, List dataModels, SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception { TableView tableView = super.tableView; SqlNode filterNode = null; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java index a74b85934..5d04eef8f 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/JoinRender.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql.render; import com.tencent.supersonic.common.pojo.enums.EngineType; -import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension; @@ -9,6 +8,7 @@ import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.JoinRelation; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Materialization; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam; import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema; import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.AggFunctionNode; @@ -47,7 +47,7 @@ import java.util.stream.Collectors; public class JoinRender extends Renderer { @Override - public void render(MetricQueryParam metricCommand, List dataModels, + public void render(OntologyQueryParam metricCommand, List dataModels, SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception { String queryWhere = metricCommand.getWhere(); EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType()); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/OutputRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/OutputRender.java index ef20426f9..9a516cefc 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/OutputRender.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/OutputRender.java @@ -2,8 +2,8 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql.render; import com.tencent.supersonic.common.pojo.ColumnOrder; import com.tencent.supersonic.common.pojo.enums.EngineType; -import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam; import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema; import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.MetricNode; @@ -22,7 +22,7 @@ import java.util.List; public class OutputRender extends Renderer { @Override - public void render(MetricQueryParam metricCommand, List dataModels, + public void render(OntologyQueryParam metricCommand, List dataModels, SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception { TableView selectDataSet = super.tableView; EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType()); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/Renderer.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/Renderer.java index d4322a411..c3466b540 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/Renderer.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/Renderer.java @@ -1,12 +1,12 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql.render; import com.tencent.supersonic.common.pojo.enums.EngineType; -import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Measure; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam; import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema; import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.MeasureNode; @@ -114,6 +114,6 @@ public abstract class Renderer { return SemanticNode.buildAs(alias, tableView.build()); } - public abstract void render(MetricQueryParam metricCommand, List dataModels, + public abstract void render(OntologyQueryParam metricCommand, List dataModels, SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception; } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/SourceRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/SourceRender.java index c4eb0d24c..19fea0587 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/SourceRender.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/calcite/sql/render/SourceRender.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.headless.core.translator.calcite.sql.render; import com.tencent.supersonic.common.pojo.enums.EngineType; -import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension; @@ -9,6 +8,7 @@ import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Identify; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Materialization; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Measure; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam; import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema; import com.tencent.supersonic.headless.core.translator.calcite.sql.TableView; import com.tencent.supersonic.headless.core.translator.calcite.sql.node.DataModelNode; @@ -336,9 +336,9 @@ public class SourceRender extends Renderer { } } - public void render(MetricQueryParam metricQueryParam, List dataModels, + public void render(OntologyQueryParam ontologyQueryParam, List dataModels, SqlValidatorScope scope, S2CalciteSchema schema, boolean nonAgg) throws Exception { - String queryWhere = metricQueryParam.getWhere(); + String queryWhere = ontologyQueryParam.getWhere(); Set whereFields = new HashSet<>(); List fieldWhere = new ArrayList<>(); EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType()); @@ -349,13 +349,13 @@ public class SourceRender extends Renderer { } if (dataModels.size() == 1) { DataModel dataModel = dataModels.get(0); - super.tableView = renderOne("", fieldWhere, metricQueryParam.getMetrics(), - metricQueryParam.getDimensions(), metricQueryParam.getWhere(), dataModel, scope, - schema, nonAgg); + super.tableView = renderOne("", fieldWhere, ontologyQueryParam.getMetrics(), + ontologyQueryParam.getDimensions(), ontologyQueryParam.getWhere(), dataModel, + scope, schema, nonAgg); return; } JoinRender joinRender = new JoinRender(); - joinRender.render(metricQueryParam, dataModels, scope, schema, nonAgg); + joinRender.render(ontologyQueryParam, dataModels, scope, schema, nonAgg); super.tableView = joinRender.getTableView(); } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/DefaultDimValueConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/DefaultDimValueConverter.java index 54b24dd84..8d44f7750 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/DefaultDimValueConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/DefaultDimValueConverter.java @@ -4,7 +4,6 @@ import com.google.common.collect.Lists; import com.tencent.supersonic.common.jsqlparser.SqlAddHelper; import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; -import com.tencent.supersonic.headless.api.pojo.MetricTable; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension; import lombok.extern.slf4j.Slf4j; @@ -28,8 +27,8 @@ public class DefaultDimValueConverter implements QueryConverter { @Override public boolean accept(QueryStatement queryStatement) { - return !Objects.isNull(queryStatement.getDataSetQueryParam()) - && !StringUtils.isBlank(queryStatement.getDataSetQueryParam().getSql()); + return Objects.nonNull(queryStatement.getSqlQueryParam()) + && StringUtils.isNotBlank(queryStatement.getSqlQueryParam().getSql()); } @Override @@ -40,15 +39,13 @@ public class DefaultDimValueConverter implements QueryConverter { if (CollectionUtils.isEmpty(dimensions)) { return; } - String sql = queryStatement.getDataSetQueryParam().getSql(); + String sql = queryStatement.getSqlQueryParam().getSql(); List whereFields = SqlSelectHelper.getWhereFields(sql).stream() .filter(field -> !TimeDimensionEnum.containsTimeDimension(field)) .collect(Collectors.toList()); if (!CollectionUtils.isEmpty(whereFields)) { return; } - MetricTable metricTable = - queryStatement.getDataSetQueryParam().getTables().stream().findFirst().orElse(null); List expressions = Lists.newArrayList(); for (Dimension dimension : dimensions) { ExpressionList expressionList = new ExpressionList(); @@ -59,11 +56,11 @@ public class DefaultDimValueConverter implements QueryConverter { inExpression.setLeftExpression(new Column(dimension.getBizName())); inExpression.setRightExpression(expressionList); expressions.add(inExpression); - if (metricTable != null) { - metricTable.getDimensions().add(dimension.getBizName()); + if (Objects.nonNull(queryStatement.getSqlQueryParam().getTable())) { + queryStatement.getOntologyQueryParam().getDimensions().add(dimension.getBizName()); } } sql = SqlAddHelper.addWhere(sql, expressions); - queryStatement.getDataSetQueryParam().setSql(sql); + queryStatement.getSqlQueryParam().setSql(sql); } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/CalculateAggConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/MetricRatioConverter.java similarity index 50% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/CalculateAggConverter.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/MetricRatioConverter.java index e0287e427..285ea655b 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/CalculateAggConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/MetricRatioConverter.java @@ -6,82 +6,46 @@ import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum; import com.tencent.supersonic.common.pojo.enums.EngineType; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.common.util.DateModeUtils; -import com.tencent.supersonic.headless.api.pojo.MetricTable; -import com.tencent.supersonic.headless.api.pojo.QueryParam; import com.tencent.supersonic.headless.api.pojo.enums.AggOption; -import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam; import com.tencent.supersonic.headless.core.pojo.Database; import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import com.tencent.supersonic.headless.core.pojo.SqlQueryParam; +import com.tencent.supersonic.headless.core.pojo.StructQueryParam; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam; import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; -/** supplement the QueryStatement when query with custom aggregation method */ @Component("CalculateAggConverter") @Slf4j -public class CalculateAggConverter implements QueryConverter { +public class MetricRatioConverter implements QueryConverter { public interface EngineSql { - String sql(QueryParam queryParam, boolean isOver, boolean asWith, String metricSql); - } - - public DataSetQueryParam generateSqlCommend(QueryStatement queryStatement, - EngineType engineTypeEnum, String version) throws Exception { - SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class); - QueryParam queryParam = queryStatement.getQueryParam(); - // 同环比 - if (isRatioAccept(queryParam)) { - return generateRatioSqlCommand(queryStatement, engineTypeEnum, version); - } - DataSetQueryParam sqlCommand = new DataSetQueryParam(); - String metricTableName = "v_metric_tb_tmp"; - MetricTable metricTable = new MetricTable(); - metricTable.setAlias(metricTableName); - metricTable.setMetrics(queryParam.getMetrics()); - metricTable.setDimensions(queryParam.getGroups()); - String where = sqlGenerateUtils.generateWhere(queryParam, null); - log.info("in generateSqlCommand, complete where:{}", where); - metricTable.setWhere(where); - metricTable.setAggOption(AggOption.AGGREGATION); - sqlCommand.setTables(new ArrayList<>(Collections.singletonList(metricTable))); - String sql = String.format("select %s from %s %s %s %s", - sqlGenerateUtils.getSelect(queryParam), metricTableName, - sqlGenerateUtils.getGroupBy(queryParam), sqlGenerateUtils.getOrderBy(queryParam), - sqlGenerateUtils.getLimit(queryParam)); - if (!sqlGenerateUtils.isSupportWith(engineTypeEnum, version)) { - sqlCommand.setSupportWith(false); - sql = String.format("select %s from %s t0 %s %s %s", - sqlGenerateUtils.getSelect(queryParam), metricTableName, - sqlGenerateUtils.getGroupBy(queryParam), - sqlGenerateUtils.getOrderBy(queryParam), sqlGenerateUtils.getLimit(queryParam)); - } - sqlCommand.setSql(sql); - return sqlCommand; + String sql(StructQueryParam structQueryParam, boolean isOver, boolean asWith, + String metricSql); } @Override public boolean accept(QueryStatement queryStatement) { - if (Objects.isNull(queryStatement.getQueryParam()) || queryStatement.getIsS2SQL()) { + if (Objects.isNull(queryStatement.getStructQueryParam()) || queryStatement.getIsS2SQL() + || !isRatioAccept(queryStatement.getStructQueryParam())) { return false; } - QueryParam queryParam = queryStatement.getQueryParam(); - if (queryParam.getQueryType().isNativeAggQuery()) { - return false; - } - if (CollectionUtils.isEmpty(queryParam.getAggregators())) { + StructQueryParam structQueryParam = queryStatement.getStructQueryParam(); + if (structQueryParam.getQueryType().isNativeAggQuery() + || CollectionUtils.isEmpty(structQueryParam.getAggregators())) { return false; } int nonSumFunction = 0; - for (Aggregator agg : queryParam.getAggregators()) { + for (Aggregator agg : structQueryParam.getAggregators()) { if (agg.getFunc() == null || "".equals(agg.getFunc())) { return false; } @@ -98,14 +62,13 @@ public class CalculateAggConverter implements QueryConverter { @Override public void convert(QueryStatement queryStatement) throws Exception { Database database = queryStatement.getOntology().getDatabase(); - DataSetQueryParam dataSetQueryParam = generateSqlCommend(queryStatement, - EngineType.fromString(database.getType().toUpperCase()), database.getVersion()); - queryStatement.setDataSetQueryParam(dataSetQueryParam); + generateRatioSql(queryStatement, EngineType.fromString(database.getType().toUpperCase()), + database.getVersion()); } /** Ratio */ - public boolean isRatioAccept(QueryParam queryParam) { - Long ratioFuncNum = queryParam.getAggregators().stream() + public boolean isRatioAccept(StructQueryParam structQueryParam) { + Long ratioFuncNum = structQueryParam.getAggregators().stream() .filter(f -> (f.getFunc().equals(AggOperatorEnum.RATIO_ROLL) || f.getFunc().equals(AggOperatorEnum.RATIO_OVER))) .count(); @@ -115,53 +78,47 @@ public class CalculateAggConverter implements QueryConverter { return false; } - public DataSetQueryParam generateRatioSqlCommand(QueryStatement queryStatement, - EngineType engineTypeEnum, String version) throws Exception { + public void generateRatioSql(QueryStatement queryStatement, EngineType engineTypeEnum, + String version) throws Exception { SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class); - QueryParam queryParam = queryStatement.getQueryParam(); - check(queryParam); + StructQueryParam structQueryParam = queryStatement.getStructQueryParam(); + check(structQueryParam); queryStatement.setEnableOptimize(false); - DataSetQueryParam sqlCommand = new DataSetQueryParam(); + OntologyQueryParam ontologyQueryParam = queryStatement.getOntologyQueryParam(); + ontologyQueryParam.setAggOption(AggOption.AGGREGATION); String metricTableName = "v_metric_tb_tmp"; - MetricTable metricTable = new MetricTable(); - metricTable.setAlias(metricTableName); - metricTable.setMetrics(queryParam.getMetrics()); - metricTable.setDimensions(queryParam.getGroups()); - String where = sqlGenerateUtils.generateWhere(queryParam, null); - log.info("in generateSqlCommend, complete where:{}", where); - metricTable.setWhere(where); - metricTable.setAggOption(AggOption.AGGREGATION); - sqlCommand.setTables(new ArrayList<>(Collections.singletonList(metricTable))); - boolean isOver = isOverRatio(queryParam); + boolean isOver = isOverRatio(structQueryParam); String sql = ""; + + SqlQueryParam dsParam = queryStatement.getSqlQueryParam(); + dsParam.setTable(metricTableName); switch (engineTypeEnum) { case H2: - sql = new H2EngineSql().sql(queryParam, isOver, true, metricTableName); + sql = new H2EngineSql().sql(structQueryParam, isOver, true, metricTableName); break; case MYSQL: case DORIS: case CLICKHOUSE: if (!sqlGenerateUtils.isSupportWith(engineTypeEnum, version)) { - sqlCommand.setSupportWith(false); + dsParam.setSupportWith(false); } if (!engineTypeEnum.equals(engineTypeEnum.CLICKHOUSE)) { - sql = new MysqlEngineSql().sql(queryParam, isOver, sqlCommand.isSupportWith(), - metricTableName); + sql = new MysqlEngineSql().sql(structQueryParam, isOver, + dsParam.isSupportWith(), metricTableName); } else { - sql = new CkEngineSql().sql(queryParam, isOver, sqlCommand.isSupportWith(), + sql = new CkEngineSql().sql(structQueryParam, isOver, dsParam.isSupportWith(), metricTableName); } break; default: } - sqlCommand.setSql(sql); - return sqlCommand; + dsParam.setSql(sql); } public class H2EngineSql implements EngineSql { - public String getOverSelect(QueryParam queryParam, boolean isOver) { - String aggStr = queryParam.getAggregators().stream().map(f -> { + public String getOverSelect(StructQueryParam structQueryParam, boolean isOver) { + String aggStr = structQueryParam.getAggregators().stream().map(f -> { if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) { return String.format("( (%s-%s_roll)/cast(%s_roll as DOUBLE) ) as %s_%s,%s", @@ -171,43 +128,44 @@ public class CalculateAggConverter implements QueryConverter { return f.getColumn(); } }).collect(Collectors.joining(",")); - return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr - : String.join(",", queryParam.getGroups()) + "," + aggStr; + return CollectionUtils.isEmpty(structQueryParam.getGroups()) ? aggStr + : String.join(",", structQueryParam.getGroups()) + "," + aggStr; } - public String getTimeSpan(QueryParam queryParam, boolean isOver, boolean isAdd) { - if (Objects.nonNull(queryParam.getDateInfo())) { + public String getTimeSpan(StructQueryParam structQueryParam, boolean isOver, + boolean isAdd) { + if (Objects.nonNull(structQueryParam.getDateInfo())) { String addStr = isAdd ? "" : "-"; - if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.DAY)) { + if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.DAY)) { return "day," + (isOver ? addStr + "7" : addStr + "1"); } - if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) { + if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) { return isOver ? "month," + addStr + "1" : "day," + addStr + "7"; } - if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH.MONTH)) { + if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH.MONTH)) { return isOver ? "year," + addStr + "1" : "month," + addStr + "1"; } } return ""; } - public String getJoinOn(QueryParam queryParam, boolean isOver, String aliasLeft, + public String getJoinOn(StructQueryParam structQueryParam, boolean isOver, String aliasLeft, String aliasRight) { - String timeDim = getTimeDim(queryParam); - String timeSpan = getTimeSpan(queryParam, isOver, true); - String aggStr = queryParam.getAggregators().stream().map(f -> { + String timeDim = getTimeDim(structQueryParam); + String timeSpan = getTimeSpan(structQueryParam, isOver, true); + String aggStr = structQueryParam.getAggregators().stream().map(f -> { if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) { - if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) { + if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) { return String.format( "%s is not null and %s = FORMATDATETIME(DATEADD(%s,CONCAT(%s,'-01')),'yyyy-MM') ", aliasRight + timeDim, aliasLeft + timeDim, timeSpan, aliasRight + timeDim); } - if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK) + if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK) && isOver) { return String.format(" DATE_TRUNC('week',DATEADD(%s,%s) ) = %s ", - getTimeSpan(queryParam, isOver, false), aliasLeft + timeDim, + getTimeSpan(structQueryParam, isOver, false), aliasLeft + timeDim, aliasRight + timeDim); } return String.format("%s = TIMESTAMPADD(%s,%s) ", aliasLeft + timeDim, timeSpan, @@ -217,7 +175,7 @@ public class CalculateAggConverter implements QueryConverter { } }).collect(Collectors.joining(" and ")); List groups = new ArrayList<>(); - for (String group : queryParam.getGroups()) { + for (String group : structQueryParam.getGroups()) { if (group.equalsIgnoreCase(timeDim)) { continue; } @@ -228,35 +186,36 @@ public class CalculateAggConverter implements QueryConverter { } @Override - public String sql(QueryParam queryParam, boolean isOver, boolean asWith, String metricSql) { + public String sql(StructQueryParam structQueryParam, boolean isOver, boolean asWith, + String metricSql) { String sql = String.format( "select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ", - getOverSelect(queryParam, isOver), getAllSelect(queryParam, "t0."), - getAllJoinSelect(queryParam, "t1."), metricSql, metricSql, - getJoinOn(queryParam, isOver, "t0.", "t1."), getOrderBy(queryParam), - getLimit(queryParam)); + getOverSelect(structQueryParam, isOver), getAllSelect(structQueryParam, "t0."), + getAllJoinSelect(structQueryParam, "t1."), metricSql, metricSql, + getJoinOn(structQueryParam, isOver, "t0.", "t1."), getOrderBy(structQueryParam), + getLimit(structQueryParam)); return sql; } } public class CkEngineSql extends MysqlEngineSql { - public String getJoinOn(QueryParam queryParam, boolean isOver, String aliasLeft, + public String getJoinOn(StructQueryParam structQueryParam, boolean isOver, String aliasLeft, String aliasRight) { - String timeDim = getTimeDim(queryParam); - String timeSpan = "INTERVAL " + getTimeSpan(queryParam, isOver, true); - String aggStr = queryParam.getAggregators().stream().map(f -> { + String timeDim = getTimeDim(structQueryParam); + String timeSpan = "INTERVAL " + getTimeSpan(structQueryParam, isOver, true); + String aggStr = structQueryParam.getAggregators().stream().map(f -> { if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) { - if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) { + if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) { return String.format( "toDate(CONCAT(%s,'-01')) = date_add(toDate(CONCAT(%s,'-01')),%s) ", aliasLeft + timeDim, aliasRight + timeDim, timeSpan); } - if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK) + if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK) && isOver) { return String.format("toMonday(date_add(%s ,INTERVAL %s) ) = %s", - aliasLeft + timeDim, getTimeSpan(queryParam, isOver, false), + aliasLeft + timeDim, getTimeSpan(structQueryParam, isOver, false), aliasRight + timeDim); } return String.format("%s = date_add(%s,%s) ", aliasLeft + timeDim, @@ -266,7 +225,7 @@ public class CalculateAggConverter implements QueryConverter { } }).collect(Collectors.joining(" and ")); List groups = new ArrayList<>(); - for (String group : queryParam.getGroups()) { + for (String group : structQueryParam.getGroups()) { if (group.equalsIgnoreCase(timeDim)) { continue; } @@ -277,45 +236,49 @@ public class CalculateAggConverter implements QueryConverter { } @Override - public String sql(QueryParam queryParam, boolean isOver, boolean asWith, String metricSql) { + public String sql(StructQueryParam structQueryParam, boolean isOver, boolean asWith, + String metricSql) { if (!asWith) { return String.format( "select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ", - getOverSelect(queryParam, isOver), getAllSelect(queryParam, "t0."), - getAllJoinSelect(queryParam, "t1."), metricSql, metricSql, - getJoinOn(queryParam, isOver, "t0.", "t1."), getOrderBy(queryParam), - getLimit(queryParam)); + getOverSelect(structQueryParam, isOver), + getAllSelect(structQueryParam, "t0."), + getAllJoinSelect(structQueryParam, "t1."), metricSql, metricSql, + getJoinOn(structQueryParam, isOver, "t0.", "t1."), + getOrderBy(structQueryParam), getLimit(structQueryParam)); } return String.format( ",t0 as (select * from %s),t1 as (select * from %s) select %s from ( select %s , %s " + "from t0 left join t1 on %s ) metric_tb_src %s %s ", - metricSql, metricSql, getOverSelect(queryParam, isOver), - getAllSelect(queryParam, "t0."), getAllJoinSelect(queryParam, "t1."), - getJoinOn(queryParam, isOver, "t0.", "t1."), getOrderBy(queryParam), - getLimit(queryParam)); + metricSql, metricSql, getOverSelect(structQueryParam, isOver), + getAllSelect(structQueryParam, "t0."), + getAllJoinSelect(structQueryParam, "t1."), + getJoinOn(structQueryParam, isOver, "t0.", "t1."), getOrderBy(structQueryParam), + getLimit(structQueryParam)); } } public class MysqlEngineSql implements EngineSql { - public String getTimeSpan(QueryParam queryParam, boolean isOver, boolean isAdd) { - if (Objects.nonNull(queryParam.getDateInfo())) { + public String getTimeSpan(StructQueryParam structQueryParam, boolean isOver, + boolean isAdd) { + if (Objects.nonNull(structQueryParam.getDateInfo())) { String addStr = isAdd ? "" : "-"; - if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.DAY)) { + if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.DAY)) { return isOver ? addStr + "7 day" : addStr + "1 day"; } - if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK)) { + if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK)) { return isOver ? addStr + "1 month" : addStr + "7 day"; } - if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) { + if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) { return isOver ? addStr + "1 year" : addStr + "1 month"; } } return ""; } - public String getOverSelect(QueryParam queryParam, boolean isOver) { - String aggStr = queryParam.getAggregators().stream().map(f -> { + public String getOverSelect(StructQueryParam structQueryParam, boolean isOver) { + String aggStr = structQueryParam.getAggregators().stream().map(f -> { if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) { return String.format("if(%s_roll!=0, (%s-%s_roll)/%s_roll , 0) as %s_%s,%s", @@ -325,26 +288,26 @@ public class CalculateAggConverter implements QueryConverter { return f.getColumn(); } }).collect(Collectors.joining(",")); - return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr - : String.join(",", queryParam.getGroups()) + "," + aggStr; + return CollectionUtils.isEmpty(structQueryParam.getGroups()) ? aggStr + : String.join(",", structQueryParam.getGroups()) + "," + aggStr; } - public String getJoinOn(QueryParam queryParam, boolean isOver, String aliasLeft, + public String getJoinOn(StructQueryParam structQueryParam, boolean isOver, String aliasLeft, String aliasRight) { - String timeDim = getTimeDim(queryParam); - String timeSpan = "INTERVAL " + getTimeSpan(queryParam, isOver, true); - String aggStr = queryParam.getAggregators().stream().map(f -> { + String timeDim = getTimeDim(structQueryParam); + String timeSpan = "INTERVAL " + getTimeSpan(structQueryParam, isOver, true); + String aggStr = structQueryParam.getAggregators().stream().map(f -> { if (f.getFunc().equals(AggOperatorEnum.RATIO_OVER) || f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)) { - if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) { + if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.MONTH)) { return String.format( "%s = DATE_FORMAT(date_add(CONCAT(%s,'-01'), %s),'%%Y-%%m') ", aliasLeft + timeDim, aliasRight + timeDim, timeSpan); } - if (queryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK) + if (structQueryParam.getDateInfo().getPeriod().equals(DatePeriodEnum.WEEK) && isOver) { return String.format("to_monday(date_add(%s ,INTERVAL %s) ) = %s", - aliasLeft + timeDim, getTimeSpan(queryParam, isOver, false), + aliasLeft + timeDim, getTimeSpan(structQueryParam, isOver, false), aliasRight + timeDim); } return String.format("%s = date_add(%s,%s) ", aliasLeft + timeDim, @@ -354,7 +317,7 @@ public class CalculateAggConverter implements QueryConverter { } }).collect(Collectors.joining(" and ")); List groups = new ArrayList<>(); - for (String group : queryParam.getGroups()) { + for (String group : structQueryParam.getGroups()) { if (group.equalsIgnoreCase(timeDim)) { continue; } @@ -365,51 +328,53 @@ public class CalculateAggConverter implements QueryConverter { } @Override - public String sql(QueryParam queryParam, boolean isOver, boolean asWith, String metricSql) { + public String sql(StructQueryParam structQueryParam, boolean isOver, boolean asWith, + String metricSql) { String sql = String.format( "select %s from ( select %s , %s from %s t0 left join %s t1 on %s ) metric_tb_src %s %s ", - getOverSelect(queryParam, isOver), getAllSelect(queryParam, "t0."), - getAllJoinSelect(queryParam, "t1."), metricSql, metricSql, - getJoinOn(queryParam, isOver, "t0.", "t1."), getOrderBy(queryParam), - getLimit(queryParam)); + getOverSelect(structQueryParam, isOver), getAllSelect(structQueryParam, "t0."), + getAllJoinSelect(structQueryParam, "t1."), metricSql, metricSql, + getJoinOn(structQueryParam, isOver, "t0.", "t1."), getOrderBy(structQueryParam), + getLimit(structQueryParam)); return sql; } } - private String getAllJoinSelect(QueryParam queryParam, String alias) { - String aggStr = queryParam.getAggregators().stream() + private String getAllJoinSelect(StructQueryParam structQueryParam, String alias) { + String aggStr = structQueryParam.getAggregators().stream() .map(f -> getSelectField(f, alias) + " as " + getSelectField(f, "") + "_roll") .collect(Collectors.joining(",")); List groups = new ArrayList<>(); - for (String group : queryParam.getGroups()) { + for (String group : structQueryParam.getGroups()) { groups.add(alias + group + " as " + group + "_roll"); } return CollectionUtils.isEmpty(groups) ? aggStr : String.join(",", groups) + "," + aggStr; } - private String getGroupDimWithOutTime(QueryParam queryParam) { - String timeDim = getTimeDim(queryParam); - return queryParam.getGroups().stream().filter(f -> !f.equalsIgnoreCase(timeDim)) + private String getGroupDimWithOutTime(StructQueryParam structQueryParam) { + String timeDim = getTimeDim(structQueryParam); + return structQueryParam.getGroups().stream().filter(f -> !f.equalsIgnoreCase(timeDim)) .collect(Collectors.joining(",")); } - private static String getTimeDim(QueryParam queryParam) { + private static String getTimeDim(StructQueryParam structQueryParam) { DateModeUtils dateModeUtils = ContextUtils.getContext().getBean(DateModeUtils.class); - return dateModeUtils.getSysDateCol(queryParam.getDateInfo()); + return dateModeUtils.getSysDateCol(structQueryParam.getDateInfo()); } - private static String getLimit(QueryParam queryParam) { - if (queryParam != null && queryParam.getLimit() != null && queryParam.getLimit() > 0) { - return " limit " + String.valueOf(queryParam.getLimit()); + private static String getLimit(StructQueryParam structQueryParam) { + if (structQueryParam != null && structQueryParam.getLimit() != null + && structQueryParam.getLimit() > 0) { + return " limit " + String.valueOf(structQueryParam.getLimit()); } return ""; } - private String getAllSelect(QueryParam queryParam, String alias) { - String aggStr = queryParam.getAggregators().stream().map(f -> getSelectField(f, alias)) - .collect(Collectors.joining(",")); - return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr - : alias + String.join("," + alias, queryParam.getGroups()) + "," + aggStr; + private String getAllSelect(StructQueryParam structQueryParam, String alias) { + String aggStr = structQueryParam.getAggregators().stream() + .map(f -> getSelectField(f, alias)).collect(Collectors.joining(",")); + return CollectionUtils.isEmpty(structQueryParam.getGroups()) ? aggStr + : alias + String.join("," + alias, structQueryParam.getGroups()) + "," + aggStr; } private String getSelectField(final Aggregator agg, String alias) { @@ -421,32 +386,32 @@ public class CalculateAggConverter implements QueryConverter { return sqlGenerateUtils.getSelectField(agg); } - private String getGroupBy(QueryParam queryParam) { - if (CollectionUtils.isEmpty(queryParam.getGroups())) { + private String getGroupBy(StructQueryParam structQueryParam) { + if (CollectionUtils.isEmpty(structQueryParam.getGroups())) { return ""; } - return "group by " + String.join(",", queryParam.getGroups()); + return "group by " + String.join(",", structQueryParam.getGroups()); } - private static String getOrderBy(QueryParam queryParam) { - return "order by " + getTimeDim(queryParam) + " desc"; + private static String getOrderBy(StructQueryParam structQueryParam) { + return "order by " + getTimeDim(structQueryParam) + " desc"; } - private boolean isOverRatio(QueryParam queryParam) { - Long overCt = queryParam.getAggregators().stream() + private boolean isOverRatio(StructQueryParam structQueryParam) { + Long overCt = structQueryParam.getAggregators().stream() .filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_OVER)).count(); return overCt > 0; } - private void check(QueryParam queryParam) throws Exception { - Long ratioOverNum = queryParam.getAggregators().stream() + private void check(StructQueryParam structQueryParam) throws Exception { + Long ratioOverNum = structQueryParam.getAggregators().stream() .filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_OVER)).count(); - Long ratioRollNum = queryParam.getAggregators().stream() + Long ratioRollNum = structQueryParam.getAggregators().stream() .filter(f -> f.getFunc().equals(AggOperatorEnum.RATIO_ROLL)).count(); if (ratioOverNum > 0 && ratioRollNum > 0) { throw new Exception("not support over ratio and roll ratio together "); } - if (getTimeDim(queryParam).isEmpty()) { + if (getTimeDim(structQueryParam).isEmpty()) { throw new Exception("miss time filter"); } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/ParserDefaultConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/ParserDefaultConverter.java deleted file mode 100644 index 12c8dd722..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/ParserDefaultConverter.java +++ /dev/null @@ -1,74 +0,0 @@ -package com.tencent.supersonic.headless.core.translator.converter; - -import com.tencent.supersonic.common.pojo.ColumnOrder; -import com.tencent.supersonic.common.util.ContextUtils; -import com.tencent.supersonic.headless.api.pojo.QueryParam; -import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; -import com.tencent.supersonic.headless.core.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.translator.calcite.s2sql.DataModel; -import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.BeanUtils; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; - -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; - -/** QueryConverter default implement */ -@Component("ParserDefaultConverter") -@Slf4j -public class ParserDefaultConverter implements QueryConverter { - - @Override - public boolean accept(QueryStatement queryStatement) { - if (Objects.isNull(queryStatement.getQueryParam()) || queryStatement.getIsS2SQL()) { - return false; - } - CalculateAggConverter calculateConverterAgg = - ContextUtils.getBean(CalculateAggConverter.class); - return !calculateConverterAgg.accept(queryStatement); - } - - @Override - public void convert(QueryStatement queryStatement) throws Exception { - SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class); - QueryParam queryParam = queryStatement.getQueryParam(); - MetricQueryParam metricQueryParam = queryStatement.getMetricQueryParam(); - MetricQueryParam metricReq = - generateSqlCommand(queryStatement.getQueryParam(), queryStatement); - queryStatement.setMinMaxTime(sqlGenerateUtils.getBeginEndTime(queryParam, null)); - BeanUtils.copyProperties(metricReq, metricQueryParam); - } - - public MetricQueryParam generateSqlCommand(QueryParam queryParam, - QueryStatement queryStatement) { - SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class); - MetricQueryParam metricQueryParam = new MetricQueryParam(); - metricQueryParam.setMetrics(queryParam.getMetrics()); - metricQueryParam.setDimensions(queryParam.getGroups()); - String where = sqlGenerateUtils.generateWhere(queryParam, null); - log.info("in generateSqlCommend, complete where:{}", where); - - metricQueryParam.setWhere(where); - metricQueryParam.setOrder(queryParam.getOrders().stream() - .map(order -> new ColumnOrder(order.getColumn(), order.getDirection())) - .collect(Collectors.toList())); - metricQueryParam.setLimit(queryParam.getLimit()); - - // support detail query - if (queryParam.getQueryType().isNativeAggQuery() - && CollectionUtils.isEmpty(metricQueryParam.getMetrics())) { - Map modelMap = queryStatement.getOntology().getModelMap(); - for (Long modelId : modelMap.keySet()) { - String modelBizName = modelMap.get(modelId).getName(); - String internalMetricName = - sqlGenerateUtils.generateInternalMetricName(modelBizName); - metricQueryParam.getMetrics().add(internalMetricName); - } - } - - return metricQueryParam; - } -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlQueryConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlQueryConverter.java new file mode 100644 index 000000000..1b2e55362 --- /dev/null +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlQueryConverter.java @@ -0,0 +1,308 @@ +package com.tencent.supersonic.headless.core.translator.converter; + +import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper; +import com.tencent.supersonic.common.jsqlparser.SqlSelectFunctionHelper; +import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; +import com.tencent.supersonic.common.pojo.Constants; +import com.tencent.supersonic.common.pojo.enums.EngineType; +import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; +import com.tencent.supersonic.common.util.ContextUtils; +import com.tencent.supersonic.headless.api.pojo.Measure; +import com.tencent.supersonic.headless.api.pojo.SchemaItem; +import com.tencent.supersonic.headless.api.pojo.enums.AggOption; +import com.tencent.supersonic.headless.api.pojo.enums.MetricType; +import com.tencent.supersonic.headless.api.pojo.response.*; +import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor; +import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory; +import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import com.tencent.supersonic.headless.core.pojo.SqlQueryParam; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam; +import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@Component("SqlQueryConverter") +@Slf4j +public class SqlQueryConverter implements QueryConverter { + + @Override + public boolean accept(QueryStatement queryStatement) { + if (Objects.nonNull(queryStatement.getSqlQueryParam()) && queryStatement.getIsS2SQL()) { + return true; + } + return false; + } + + @Override + public void convert(QueryStatement queryStatement) throws Exception { + SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class); + convertNameToBizName(queryStatement); + rewriteFunction(queryStatement); + String reqSql = queryStatement.getSqlQueryParam().getSql(); + String tableName = SqlSelectHelper.getTableName(reqSql); + if (StringUtils.isEmpty(tableName)) { + return; + } + + // replace order by field with the select sequence number + queryStatement.setSql(SqlReplaceHelper.replaceAggAliasOrderbyField(reqSql)); + log.debug("replaceOrderAggSameAlias {} -> {}", reqSql, queryStatement.getSql()); + + SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp(); + // fill dataSetQuery + SqlQueryParam sqlQueryParam = queryStatement.getSqlQueryParam(); + sqlQueryParam.setTable(tableName.toLowerCase()); + if (!sqlGenerateUtils.isSupportWith( + EngineType.fromString(semanticSchemaResp.getDatabaseResp().getType().toUpperCase()), + semanticSchemaResp.getDatabaseResp().getVersion())) { + sqlQueryParam.setSupportWith(false); + sqlQueryParam.setWithAlias(false); + } + + // build ontologyQuery + List allFields = SqlSelectHelper.getAllSelectFields(queryStatement.getSql()); + List metricSchemas = getMetrics(semanticSchemaResp, allFields); + List metrics = + metricSchemas.stream().map(SchemaItem::getBizName).collect(Collectors.toList()); + AggOption aggOption = getAggOption(queryStatement, metricSchemas); + Set dimensions = getDimensions(semanticSchemaResp, allFields); + OntologyQueryParam ontologyQueryParam = new OntologyQueryParam(); + ontologyQueryParam.getMetrics().addAll(metrics); + ontologyQueryParam.getDimensions().addAll(dimensions); + ontologyQueryParam.setAggOption(aggOption); + ontologyQueryParam.setNativeQuery(!AggOption.isAgg(aggOption)); + + log.info("parse sqlQuery [{}] ", sqlQueryParam); + queryStatement.setOntologyQueryParam(ontologyQueryParam); + queryStatement.setSql(sqlQueryParam.getSql()); + generateDerivedMetric(sqlGenerateUtils, queryStatement); + } + + private AggOption getAggOption(QueryStatement queryStatement, + List metricSchemas) { + String sql = queryStatement.getSql(); + if (SqlSelectFunctionHelper.hasAggregateFunction(sql)) { + return AggOption.AGGREGATION; + } + + if (!SqlSelectFunctionHelper.hasAggregateFunction(sql) && !SqlSelectHelper.hasGroupBy(sql) + && !SqlSelectHelper.hasWith(sql) && !SqlSelectHelper.hasSubSelect(sql)) { + log.debug("getAggOption simple sql set to DEFAULT"); + return AggOption.DEFAULT; + } + + // if there is no group by in S2SQL,set MetricTable's aggOption to "NATIVE" + // if there is count() in S2SQL,set MetricTable's aggOption to "NATIVE" + if (!SqlSelectFunctionHelper.hasAggregateFunction(sql) + || SqlSelectFunctionHelper.hasFunction(sql, "count") + || SqlSelectFunctionHelper.hasFunction(sql, "count_distinct")) { + return AggOption.OUTER; + } + + if (SqlSelectHelper.hasSubSelect(sql) || SqlSelectHelper.hasWith(sql) + || SqlSelectHelper.hasGroupBy(sql)) { + return AggOption.OUTER; + } + long defaultAggNullCnt = metricSchemas.stream().filter( + m -> Objects.isNull(m.getDefaultAgg()) || StringUtils.isBlank(m.getDefaultAgg())) + .count(); + if (defaultAggNullCnt > 0) { + log.debug("getAggOption find null defaultAgg metric set to NATIVE"); + return AggOption.OUTER; + } + return AggOption.DEFAULT; + } + + private Set getDimensions(SemanticSchemaResp semanticSchemaResp, + List allFields) { + Map dimensionLowerToNameMap = semanticSchemaResp.getDimensions().stream() + .collect(Collectors.toMap(entry -> entry.getBizName().toLowerCase(), + SchemaItem::getBizName, (k1, k2) -> k1)); + dimensionLowerToNameMap.put(TimeDimensionEnum.DAY.getName(), + TimeDimensionEnum.DAY.getName()); + return allFields.stream() + .filter(entry -> dimensionLowerToNameMap.containsKey(entry.toLowerCase())) + .map(entry -> dimensionLowerToNameMap.get(entry.toLowerCase())) + .collect(Collectors.toSet()); + } + + private List getMetrics(SemanticSchemaResp semanticSchemaResp, + List allFields) { + Map metricLowerToNameMap = + semanticSchemaResp.getMetrics().stream().collect(Collectors + .toMap(entry -> entry.getBizName().toLowerCase(), entry -> entry)); + return allFields.stream() + .filter(entry -> metricLowerToNameMap.containsKey(entry.toLowerCase())) + .map(entry -> metricLowerToNameMap.get(entry.toLowerCase())) + .collect(Collectors.toList()); + } + + + private void generateDerivedMetric(SqlGenerateUtils sqlGenerateUtils, + QueryStatement queryStatement) { + SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp(); + SqlQueryParam dsParam = queryStatement.getSqlQueryParam(); + OntologyQueryParam ontology = queryStatement.getOntologyQueryParam(); + String sql = dsParam.getSql(); + + Set measures = new HashSet<>(); + Map replaces = generateDerivedMetric(sqlGenerateUtils, semanticSchemaResp, + ontology.getAggOption(), ontology.getMetrics(), ontology.getDimensions(), measures); + + if (!CollectionUtils.isEmpty(replaces)) { + // metricTable sql use measures replace metric + sql = SqlReplaceHelper.replaceSqlByExpression(sql, replaces); + ontology.setAggOption(AggOption.NATIVE); + // metricTable use measures replace metric + if (!CollectionUtils.isEmpty(measures)) { + ontology.getMetrics().addAll(measures); + } else { + // empty measure , fill default + ontology.setMetrics(new ArrayList<>()); + ontology.getMetrics().add(sqlGenerateUtils.generateInternalMetricName( + getDefaultModel(semanticSchemaResp, ontology.getDimensions()))); + } + } + + dsParam.setSql(sql); + } + + private Map generateDerivedMetric(SqlGenerateUtils sqlGenerateUtils, + SemanticSchemaResp semanticSchemaResp, AggOption aggOption, List metrics, + List dimensions, Set measures) { + Map result = new HashMap<>(); + List metricResps = semanticSchemaResp.getMetrics(); + List dimensionResps = semanticSchemaResp.getDimensions(); + + // Check if any metric is derived + boolean hasDerivedMetrics = + metricResps.stream().anyMatch(m -> metrics.contains(m.getBizName()) && MetricType + .isDerived(m.getMetricDefineType(), m.getMetricDefineByMeasureParams())); + if (!hasDerivedMetrics) { + return result; + } + + log.debug("begin to generateDerivedMetric {} [{}]", aggOption, metrics); + + Set allFields = new HashSet<>(); + Map allMeasures = new HashMap<>(); + semanticSchemaResp.getModelResps().forEach(modelResp -> { + allFields.addAll(modelResp.getFieldList()); + if (modelResp.getModelDetail().getMeasures() != null) { + modelResp.getModelDetail().getMeasures() + .forEach(measure -> allMeasures.put(measure.getBizName(), measure)); + } + }); + + Set derivedDimensions = new HashSet<>(); + Set derivedMetrics = new HashSet<>(); + Map visitedMetrics = new HashMap<>(); + + for (MetricResp metricResp : metricResps) { + if (metrics.contains(metricResp.getBizName())) { + boolean isDerived = MetricType.isDerived(metricResp.getMetricDefineType(), + metricResp.getMetricDefineByMeasureParams()); + if (isDerived) { + String expr = sqlGenerateUtils.generateDerivedMetric(metricResps, allFields, + allMeasures, dimensionResps, sqlGenerateUtils.getExpr(metricResp), + metricResp.getMetricDefineType(), aggOption, visitedMetrics, + derivedMetrics, derivedDimensions); + result.put(metricResp.getBizName(), expr); + log.debug("derived metric {}->{}", metricResp.getBizName(), expr); + } else { + measures.add(metricResp.getBizName()); + } + } + } + + measures.addAll(derivedMetrics); + derivedDimensions.stream().filter(dimension -> !dimensions.contains(dimension)) + .forEach(dimensions::add); + + return result; + } + + + private void convertNameToBizName(QueryStatement queryStatement) { + SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp(); + Map fieldNameToBizNameMap = getFieldNameToBizNameMap(semanticSchemaResp); + String sql = queryStatement.getSqlQueryParam().getSql(); + log.debug("dataSetId:{},convert name to bizName before:{}", queryStatement.getDataSetId(), + sql); + sql = SqlReplaceHelper.replaceFields(sql, fieldNameToBizNameMap, true); + log.debug("dataSetId:{},convert name to bizName after:{}", queryStatement.getDataSetId(), + sql); + sql = SqlReplaceHelper.replaceTable(sql, + Constants.TABLE_PREFIX + queryStatement.getDataSetId()); + log.debug("replaceTableName after:{}", sql); + queryStatement.getSqlQueryParam().setSql(sql); + } + + + private void rewriteFunction(QueryStatement queryStatement) { + SemanticSchemaResp semanticSchemaResp = queryStatement.getSemanticSchemaResp(); + DatabaseResp database = semanticSchemaResp.getDatabaseResp(); + String sql = queryStatement.getSqlQueryParam().getSql(); + if (Objects.isNull(database) || Objects.isNull(database.getType())) { + return; + } + String type = database.getType(); + DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(type.toLowerCase()); + if (Objects.nonNull(engineAdaptor)) { + String functionNameCorrector = engineAdaptor.functionNameCorrector(sql); + queryStatement.getSqlQueryParam().setSql(functionNameCorrector); + } + } + + + protected Map getFieldNameToBizNameMap(SemanticSchemaResp semanticSchemaResp) { + // support fieldName and field alias to bizName + Map dimensionResults = semanticSchemaResp.getDimensions().stream().flatMap( + entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName())) + .collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1)); + + Map metricResults = semanticSchemaResp.getMetrics().stream().flatMap( + entry -> getPairStream(entry.getAlias(), entry.getName(), entry.getBizName())) + .collect(Collectors.toMap(Pair::getLeft, Pair::getRight, (k1, k2) -> k1)); + + dimensionResults.putAll(TimeDimensionEnum.getChNameToNameMap()); + dimensionResults.putAll(TimeDimensionEnum.getNameToNameMap()); + dimensionResults.putAll(metricResults); + return dimensionResults; + } + + private Stream> getPairStream(String aliasStr, String name, + String bizName) { + Set> elements = new HashSet<>(); + elements.add(Pair.of(name, bizName)); + if (StringUtils.isNotBlank(aliasStr)) { + List aliasList = SchemaItem.getAliasList(aliasStr); + for (String alias : aliasList) { + elements.add(Pair.of(alias, bizName)); + } + } + return elements.stream(); + } + + private String getDefaultModel(SemanticSchemaResp semanticSchemaResp, List dimensions) { + if (!CollectionUtils.isEmpty(dimensions)) { + Map modelMatchCnt = new HashMap<>(); + for (ModelResp modelResp : semanticSchemaResp.getModelResps()) { + modelMatchCnt.put(modelResp.getBizName(), modelResp.getModelDetail().getDimensions() + .stream().filter(d -> dimensions.contains(d.getBizName())).count()); + } + return modelMatchCnt.entrySet().stream() + .sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())) + .map(m -> m.getKey()).findFirst().orElse(""); + } + return semanticSchemaResp.getModelResps().get(0).getBizName(); + } + +} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlVariableParseConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlVariableConverter.java similarity index 86% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlVariableParseConverter.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlVariableConverter.java index e2c5f9a06..299ea4609 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlVariableParseConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlVariableConverter.java @@ -14,12 +14,12 @@ import java.util.List; import java.util.Objects; @Slf4j -@Component("SqlVariableParseConverter") -public class SqlVariableParseConverter implements QueryConverter { +@Component("SqlVariableConverter") +public class SqlVariableConverter implements QueryConverter { @Override public boolean accept(QueryStatement queryStatement) { - if (Objects.isNull(queryStatement.getQueryParam())) { + if (Objects.isNull(queryStatement.getStructQueryParam()) && queryStatement.getIsS2SQL()) { return false; } return true; @@ -38,7 +38,7 @@ public class SqlVariableParseConverter implements QueryConverter { String sqlParsed = SqlVariableParseUtils.parse(modelResp.getModelDetail().getSqlQuery(), modelResp.getModelDetail().getSqlVariables(), - queryStatement.getQueryParam().getParams()); + queryStatement.getStructQueryParam().getParams()); DataModel dataModel = queryStatement.getOntology().getDataModelMap().get(modelResp.getBizName()); dataModel.setSqlQuery(sqlParsed); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/StructQueryConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/StructQueryConverter.java new file mode 100644 index 000000000..38aa00794 --- /dev/null +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/StructQueryConverter.java @@ -0,0 +1,74 @@ +package com.tencent.supersonic.headless.core.translator.converter; + +import com.tencent.supersonic.common.pojo.ColumnOrder; +import com.tencent.supersonic.common.pojo.enums.EngineType; +import com.tencent.supersonic.common.util.ContextUtils; +import com.tencent.supersonic.headless.api.pojo.enums.AggOption; +import com.tencent.supersonic.headless.core.pojo.Database; +import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import com.tencent.supersonic.headless.core.pojo.SqlQueryParam; +import com.tencent.supersonic.headless.core.pojo.StructQueryParam; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam; +import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.util.Objects; +import java.util.stream.Collectors; + +@Component("ParserDefaultConverter") +@Slf4j +public class StructQueryConverter implements QueryConverter { + + @Override + public boolean accept(QueryStatement queryStatement) { + if (Objects.nonNull(queryStatement.getStructQueryParam()) && !queryStatement.getIsS2SQL()) { + return true; + } + + return false; + } + + @Override + public void convert(QueryStatement queryStatement) throws Exception { + SqlGenerateUtils sqlGenerateUtils = ContextUtils.getBean(SqlGenerateUtils.class); + StructQueryParam structQueryParam = queryStatement.getStructQueryParam(); + + String dsTable = "t_1"; + SqlQueryParam sqlParam = new SqlQueryParam(); + sqlParam.setTable(dsTable); + String sql = String.format("select %s from %s %s %s %s", + sqlGenerateUtils.getSelect(structQueryParam), dsTable, + sqlGenerateUtils.getGroupBy(structQueryParam), + sqlGenerateUtils.getOrderBy(structQueryParam), + sqlGenerateUtils.getLimit(structQueryParam)); + Database database = queryStatement.getOntology().getDatabase(); + EngineType engineType = EngineType.fromString(database.getType().toUpperCase()); + if (!sqlGenerateUtils.isSupportWith(engineType, database.getVersion())) { + sqlParam.setSupportWith(false); + sql = String.format("select %s from %s t0 %s %s %s", + sqlGenerateUtils.getSelect(structQueryParam), dsTable, + sqlGenerateUtils.getGroupBy(structQueryParam), + sqlGenerateUtils.getOrderBy(structQueryParam), + sqlGenerateUtils.getLimit(structQueryParam)); + } + sqlParam.setSql(sql); + queryStatement.setSqlQueryParam(sqlParam); + + OntologyQueryParam ontologyQueryParam = new OntologyQueryParam(); + ontologyQueryParam.getDimensions().addAll(structQueryParam.getGroups()); + ontologyQueryParam.getMetrics().addAll(structQueryParam.getAggregators().stream() + .map(a -> a.getColumn()).collect(Collectors.toList())); + String where = sqlGenerateUtils.generateWhere(structQueryParam, null); + ontologyQueryParam.setWhere(where); + ontologyQueryParam.setAggOption(AggOption.AGGREGATION); + ontologyQueryParam.setNativeQuery(structQueryParam.getQueryType().isNativeAggQuery()); + ontologyQueryParam.setOrder(structQueryParam.getOrders().stream() + .map(order -> new ColumnOrder(order.getColumn(), order.getDirection())) + .collect(Collectors.toList())); + ontologyQueryParam.setLimit(structQueryParam.getLimit()); + queryStatement.setOntologyQueryParam(ontologyQueryParam); + log.info("parse structQuery [{}] ", queryStatement.getSqlQueryParam()); + } + +} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java index 60f4097d8..8adf84d3e 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java @@ -12,7 +12,6 @@ import com.tencent.supersonic.common.util.DateModeUtils; import com.tencent.supersonic.common.util.SqlFilterUtils; import com.tencent.supersonic.common.util.StringUtil; import com.tencent.supersonic.headless.api.pojo.Measure; -import com.tencent.supersonic.headless.api.pojo.QueryParam; import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; @@ -20,6 +19,7 @@ import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.MetricResp; import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; import com.tencent.supersonic.headless.core.config.ExecutorConfig; +import com.tencent.supersonic.headless.core.pojo.StructQueryParam; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.ImmutablePair; @@ -85,25 +85,26 @@ public class SqlGenerateUtils { return selectSql; } - public String getLimit(QueryParam queryParam) { - if (queryParam != null && queryParam.getLimit() != null && queryParam.getLimit() > 0) { - return " limit " + queryParam.getLimit(); + public String getLimit(StructQueryParam structQueryParam) { + if (structQueryParam != null && structQueryParam.getLimit() != null + && structQueryParam.getLimit() > 0) { + return " limit " + structQueryParam.getLimit(); } return ""; } - public String getSelect(QueryParam queryParam) { - String aggStr = queryParam.getAggregators().stream().map(this::getSelectField) + public String getSelect(StructQueryParam structQueryParam) { + String aggStr = structQueryParam.getAggregators().stream().map(this::getSelectField) .collect(Collectors.joining(",")); - return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr - : String.join(",", queryParam.getGroups()) + "," + aggStr; + return CollectionUtils.isEmpty(structQueryParam.getGroups()) ? aggStr + : String.join(",", structQueryParam.getGroups()) + "," + aggStr; } - public String getSelect(QueryParam queryParam, Map deriveMetrics) { - String aggStr = queryParam.getAggregators().stream() + public String getSelect(StructQueryParam structQueryParam, Map deriveMetrics) { + String aggStr = structQueryParam.getAggregators().stream() .map(a -> getSelectField(a, deriveMetrics)).collect(Collectors.joining(",")); - return CollectionUtils.isEmpty(queryParam.getGroups()) ? aggStr - : String.join(",", queryParam.getGroups()) + "," + aggStr; + return CollectionUtils.isEmpty(structQueryParam.getGroups()) ? aggStr + : String.join(",", structQueryParam.getGroups()) + "," + aggStr; } public String getSelectField(final Aggregator agg) { @@ -128,46 +129,46 @@ public class SqlGenerateUtils { return deriveMetrics.get(agg.getColumn()); } - public String getGroupBy(QueryParam queryParam) { - if (CollectionUtils.isEmpty(queryParam.getGroups())) { + public String getGroupBy(StructQueryParam structQueryParam) { + if (CollectionUtils.isEmpty(structQueryParam.getGroups())) { return ""; } - return "group by " + String.join(",", queryParam.getGroups()); + return "group by " + String.join(",", structQueryParam.getGroups()); } - public String getOrderBy(QueryParam queryParam) { - if (CollectionUtils.isEmpty(queryParam.getOrders())) { + public String getOrderBy(StructQueryParam structQueryParam) { + if (CollectionUtils.isEmpty(structQueryParam.getOrders())) { return ""; } - return "order by " + queryParam.getOrders().stream() + return "order by " + structQueryParam.getOrders().stream() .map(order -> " " + order.getColumn() + " " + order.getDirection() + " ") .collect(Collectors.joining(",")); } - public String getOrderBy(QueryParam queryParam, Map deriveMetrics) { - if (CollectionUtils.isEmpty(queryParam.getOrders())) { + public String getOrderBy(StructQueryParam structQueryParam, Map deriveMetrics) { + if (CollectionUtils.isEmpty(structQueryParam.getOrders())) { return ""; } - if (!queryParam.getOrders().stream() + if (!structQueryParam.getOrders().stream() .anyMatch(o -> deriveMetrics.containsKey(o.getColumn()))) { - return getOrderBy(queryParam); + return getOrderBy(structQueryParam); } - return "order by " + queryParam.getOrders().stream() + return "order by " + structQueryParam.getOrders().stream() .map(order -> " " + (deriveMetrics.containsKey(order.getColumn()) ? deriveMetrics.get(order.getColumn()) : order.getColumn()) + " " + order.getDirection() + " ") .collect(Collectors.joining(",")); } - public String generateWhere(QueryParam queryParam, ItemDateResp itemDateResp) { + public String generateWhere(StructQueryParam structQueryParam, ItemDateResp itemDateResp) { String whereClauseFromFilter = - sqlFilterUtils.getWhereClause(queryParam.getDimensionFilters()); - String whereFromDate = getDateWhereClause(queryParam.getDateInfo(), itemDateResp); - return mergeDateWhereClause(queryParam, whereClauseFromFilter, whereFromDate); + sqlFilterUtils.getWhereClause(structQueryParam.getDimensionFilters()); + String whereFromDate = getDateWhereClause(structQueryParam.getDateInfo(), itemDateResp); + return mergeDateWhereClause(structQueryParam, whereClauseFromFilter, whereFromDate); } - private String mergeDateWhereClause(QueryParam queryParam, String whereClauseFromFilter, - String whereFromDate) { + private String mergeDateWhereClause(StructQueryParam structQueryParam, + String whereClauseFromFilter, String whereFromDate) { if (StringUtils.isNotEmpty(whereFromDate) && StringUtils.isNotEmpty(whereClauseFromFilter)) { return String.format("%s AND (%s)", whereFromDate, whereClauseFromFilter); @@ -179,7 +180,7 @@ public class SqlGenerateUtils { return whereFromDate; } else if (Objects.isNull(whereFromDate) && StringUtils.isEmpty(whereClauseFromFilter)) { log.debug("the current date information is empty, enter the date initialization logic"); - return dateModeUtils.defaultRecentDateInfo(queryParam.getDateInfo()); + return dateModeUtils.defaultRecentDateInfo(structQueryParam.getDateInfo()); } return whereClauseFromFilter; } @@ -203,12 +204,12 @@ public class SqlGenerateUtils { return dateModeUtils.getDateWhereStr(dateInfo, dateDate); } - public Triple getBeginEndTime(QueryParam queryParam, + public Triple getBeginEndTime(StructQueryParam structQueryParam, ItemDateResp dataDate) { - if (Objects.isNull(queryParam.getDateInfo())) { + if (Objects.isNull(structQueryParam.getDateInfo())) { return Triple.of("", "", ""); } - DateConf dateConf = queryParam.getDateInfo(); + DateConf dateConf = structQueryParam.getDateInfo(); String dateInfo = dateModeUtils.getSysDateCol(dateConf); if (dateInfo.isEmpty()) { return Triple.of("", "", ""); diff --git a/headless/core/src/test/java/com/tencent/supersonic/chat/core/parser/aggregate/CalciteSqlParserTest.java b/headless/core/src/test/java/com/tencent/supersonic/chat/core/parser/aggregate/CalciteSqlParserTest.java index 5a3124bf8..72fa452c9 100644 --- a/headless/core/src/test/java/com/tencent/supersonic/chat/core/parser/aggregate/CalciteSqlParserTest.java +++ b/headless/core/src/test/java/com/tencent/supersonic/chat/core/parser/aggregate/CalciteSqlParserTest.java @@ -1,7 +1,6 @@ package com.tencent.supersonic.chat.core.parser.aggregate; import com.alibaba.fastjson.JSON; -import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.translator.calcite.CalciteQueryParser; import org.junit.jupiter.api.Test; @@ -318,7 +317,7 @@ public class CalciteSqlParserTest { + " \"updatedAt\": 1711367511146\n" + " }\n" + " }\n" + "}"; QueryStatement queryStatement = JSON.parseObject(json, QueryStatement.class); CalciteQueryParser calciteSqlParser = new CalciteQueryParser(); - calciteSqlParser.parse(queryStatement, AggOption.DEFAULT); + calciteSqlParser.parse(queryStatement); Assert.assertEquals(queryStatement.getSql().trim().replaceAll("\\s+", ""), "SELECT`imp_date`AS`sys_imp_date`,SUM(1)AS`pv`" + "FROM" + "`s2_pv_uv_statis`" + "GROUPBY`imp_date`,`imp_date`"); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java index c26b66c44..306ac693a 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/facade/service/impl/S2SemanticLayerService.java @@ -6,24 +6,10 @@ import com.tencent.supersonic.common.pojo.QueryColumn; import com.tencent.supersonic.common.pojo.User; import com.tencent.supersonic.common.pojo.enums.TaskStatusEnum; import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; -import com.tencent.supersonic.headless.api.pojo.DataSetSchema; -import com.tencent.supersonic.headless.api.pojo.Dim; -import com.tencent.supersonic.headless.api.pojo.MetaFilter; -import com.tencent.supersonic.headless.api.pojo.QueryParam; +import com.tencent.supersonic.headless.api.pojo.*; import com.tencent.supersonic.headless.api.pojo.enums.SemanticType; -import com.tencent.supersonic.headless.api.pojo.request.DimensionValueReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; -import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; -import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; -import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq; -import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; -import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; -import com.tencent.supersonic.headless.api.pojo.response.ItemResp; -import com.tencent.supersonic.headless.api.pojo.response.MetricResp; -import com.tencent.supersonic.headless.api.pojo.response.ModelResp; -import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; -import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; -import com.tencent.supersonic.headless.api.pojo.response.SemanticTranslateResp; +import com.tencent.supersonic.headless.api.pojo.request.*; +import com.tencent.supersonic.headless.api.pojo.response.*; import com.tencent.supersonic.headless.chat.knowledge.HanlpMapResult; import com.tencent.supersonic.headless.chat.knowledge.KnowledgeBaseService; import com.tencent.supersonic.headless.chat.knowledge.MapResult; @@ -33,6 +19,8 @@ import com.tencent.supersonic.headless.chat.knowledge.helper.NatureHelper; import com.tencent.supersonic.headless.core.cache.QueryCache; import com.tencent.supersonic.headless.core.executor.QueryExecutor; import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import com.tencent.supersonic.headless.core.pojo.SqlQueryParam; +import com.tencent.supersonic.headless.core.pojo.StructQueryParam; import com.tencent.supersonic.headless.core.translator.SemanticTranslator; import com.tencent.supersonic.headless.core.utils.ComponentFactory; import com.tencent.supersonic.headless.server.annotation.S2DataPermission; @@ -52,12 +40,7 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.beans.BeanUtils; import org.springframework.stereotype.Service; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; @Service @@ -307,30 +290,13 @@ public class S2SemanticLayerService implements SemanticLayerService { return queryStatement; } - private QueryStatement buildSqlQueryStatement(QuerySqlReq querySqlReq, User user) { - // If dataSetId or DataSetName is empty, parse dataSetId from the SQL - if (querySqlReq.needGetDataSetId()) { - Long dataSetId = dataSetService.getDataSetIdFromSql(querySqlReq.getSql(), user); - querySqlReq.setDataSetId(dataSetId); - } - - QueryStatement queryStatement = buildStructQueryStatement(querySqlReq); - queryStatement.setIsS2SQL(true); - queryStatement.setSql(querySqlReq.getSql()); - return queryStatement; - } - - private QueryStatement buildStructQueryStatement(SemanticQueryReq queryReq) { + private QueryStatement buildQueryStatement(SemanticQueryReq queryReq) { SchemaFilterReq schemaFilterReq = new SchemaFilterReq(); schemaFilterReq.setDataSetId(queryReq.getDataSetId()); schemaFilterReq.setModelIds(queryReq.getModelIds()); SemanticSchemaResp semanticSchemaResp = schemaService.fetchSemanticSchema(schemaFilterReq); QueryStatement queryStatement = new QueryStatement(); - QueryParam queryParam = new QueryParam(); - BeanUtils.copyProperties(queryReq, queryParam); - queryStatement.setQueryParam(queryParam); - queryStatement.setModelIds(queryReq.getModelIds()); queryStatement.setEnableOptimize(queryUtils.enableOptimize()); queryStatement.setDataSetId(queryReq.getDataSetId()); queryStatement.setSemanticSchemaResp(semanticSchemaResp); @@ -338,6 +304,31 @@ public class S2SemanticLayerService implements SemanticLayerService { return queryStatement; } + private QueryStatement buildSqlQueryStatement(QuerySqlReq querySqlReq, User user) { + QueryStatement queryStatement = buildQueryStatement(querySqlReq); + queryStatement.setIsS2SQL(true); + + SqlQueryParam sqlQueryParam = new SqlQueryParam(); + sqlQueryParam.setSql(querySqlReq.getSql()); + queryStatement.setSqlQueryParam(sqlQueryParam); + + // If dataSetId or DataSetName is empty, parse dataSetId from the SQL + if (querySqlReq.needGetDataSetId()) { + Long dataSetId = dataSetService.getDataSetIdFromSql(querySqlReq.getSql(), user); + querySqlReq.setDataSetId(dataSetId); + } + return queryStatement; + } + + private QueryStatement buildStructQueryStatement(SemanticQueryReq queryReq) { + QueryStatement queryStatement = buildQueryStatement(queryReq); + StructQueryParam structQueryParam = new StructQueryParam(); + BeanUtils.copyProperties(queryReq, structQueryParam); + queryStatement.setStructQueryParam(structQueryParam); + queryStatement.setIsS2SQL(false); + return queryStatement; + } + private QueryStatement buildMultiStructQueryStatement(QueryMultiStructReq queryMultiStructReq) { List queryStatements = new ArrayList<>(); for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricDrillDownChecker.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricDrillDownChecker.java index 12caf5b77..5c6de4df9 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricDrillDownChecker.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/MetricDrillDownChecker.java @@ -67,7 +67,8 @@ public class MetricDrillDownChecker { List metricResps = getMetrics(metricFields, semanticSchemaResp); if (!checkDrillDownDimension(dimensionBizName, metricResps, semanticSchemaResp)) { DimSchemaResp dimSchemaResp = semanticSchemaResp.getDimension(dimensionBizName); - if (Objects.nonNull(dimSchemaResp) && dimSchemaResp.isPartitionTime()) { + if (Objects.isNull(dimSchemaResp) + || (Objects.nonNull(dimSchemaResp) && dimSchemaResp.isPartitionTime())) { continue; } String errMsg = diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java index 656ccd081..c28e3e1ed 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/calcite/HeadlessParserServiceTest.java @@ -2,20 +2,13 @@ package com.tencent.supersonic.headless.server.calcite; import com.tencent.supersonic.common.pojo.ColumnOrder; import com.tencent.supersonic.common.pojo.enums.EngineType; -import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.api.pojo.response.SqlParserResp; -import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import com.tencent.supersonic.headless.core.translator.calcite.s2sql.OntologyQueryParam; import com.tencent.supersonic.headless.core.translator.calcite.sql.S2CalciteSchema; import com.tencent.supersonic.headless.core.translator.calcite.sql.SqlBuilder; import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager; -import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.DimensionYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.IdentifyYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.MeasureYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl; -import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl; +import com.tencent.supersonic.headless.server.pojo.yaml.*; import lombok.extern.slf4j.Slf4j; import java.util.ArrayList; @@ -26,7 +19,7 @@ import java.util.List; class HeadlessParserServiceTest { public static SqlParserResp parser(S2CalciteSchema semanticSchema, - MetricQueryParam metricQueryParam, boolean isAgg) { + OntologyQueryParam ontologyQueryParam, boolean isAgg) { SqlParserResp sqlParser = new SqlParserResp(); try { if (semanticSchema == null) { @@ -35,14 +28,14 @@ class HeadlessParserServiceTest { } SqlBuilder aggBuilder = new SqlBuilder(semanticSchema); QueryStatement queryStatement = new QueryStatement(); - queryStatement.setMetricQueryParam(metricQueryParam); - aggBuilder.build(queryStatement, AggOption.getAggregation(!isAgg)); + queryStatement.setOntologyQueryParam(ontologyQueryParam); + aggBuilder.build(queryStatement); EngineType engineType = EngineType.fromString(semanticSchema.getOntology().getDatabase().getType()); sqlParser.setSql(aggBuilder.getSql(engineType)); } catch (Exception e) { sqlParser.setErrMsg(e.getMessage()); - log.error("parser error metricQueryReq[{}] error [{}]", metricQueryParam, e); + log.error("parser error metricQueryReq[{}] error [{}]", ontologyQueryParam, e); } return sqlParser; } @@ -161,7 +154,7 @@ class HeadlessParserServiceTest { // HeadlessSchemaManager.update(headlessSchema, HeadlessSchemaManager.getMetrics(metric)); - MetricQueryParam metricCommand = new MetricQueryParam(); + OntologyQueryParam metricCommand = new OntologyQueryParam(); metricCommand.setDimensions(new ArrayList<>(Arrays.asList("sys_imp_date"))); metricCommand.setMetrics(new ArrayList<>(Arrays.asList("pv"))); metricCommand.setWhere( @@ -174,7 +167,7 @@ class HeadlessParserServiceTest { addDepartment(semanticSchema); - MetricQueryParam metricCommand2 = new MetricQueryParam(); + OntologyQueryParam metricCommand2 = new OntologyQueryParam(); metricCommand2.setDimensions(new ArrayList<>(Arrays.asList("sys_imp_date", "user_name__department", "user_name", "user_name__page"))); metricCommand2.setMetrics(new ArrayList<>(Arrays.asList("pv"))); diff --git a/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/QueryNLReqBuilderTest.java b/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/QueryNLReqBuilderTest.java index 7cec555f9..63fa85ed9 100644 --- a/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/QueryNLReqBuilderTest.java +++ b/headless/server/src/test/java/com/tencent/supersonic/headless/server/utils/QueryNLReqBuilderTest.java @@ -68,8 +68,5 @@ class QueryNLReqBuilderTest { DateModeUtils dateModeUtils = new DateModeUtils(); mockContextUtils.when(() -> ContextUtils.getBean(DateModeUtils.class)) .thenReturn(dateModeUtils); - dateModeUtils.setSysDateCol("sys_imp_date"); - dateModeUtils.setSysDateWeekCol("sys_imp_week"); - dateModeUtils.setSysDateMonthCol("sys_imp_month"); } } diff --git a/launchers/headless/src/main/resources/META-INF/spring.factories b/launchers/headless/src/main/resources/META-INF/spring.factories index 82b9e8397..d91c2e7d3 100644 --- a/launchers/headless/src/main/resources/META-INF/spring.factories +++ b/launchers/headless/src/main/resources/META-INF/spring.factories @@ -26,9 +26,10 @@ com.tencent.supersonic.headless.chat.parser.llm.DataSetResolver=\ com.tencent.supersonic.headless.core.translator.converter.QueryConverter=\ com.tencent.supersonic.headless.core.translator.converter.DefaultDimValueConverter,\ - com.tencent.supersonic.headless.core.translator.converter.SqlVariableParseConverter,\ - com.tencent.supersonic.headless.core.translator.converter.CalculateAggConverter,\ - com.tencent.supersonic.headless.core.translator.converter.ParserDefaultConverter + com.tencent.supersonic.headless.core.translator.converter.SqlVariableConverter,\ + com.tencent.supersonic.headless.core.translator.converter.MetricRatioConverter,\ + com.tencent.supersonic.headless.core.translator.converter.SqlQueryConverter,\ + com.tencent.supersonic.headless.core.translator.converter.StructQueryConverter com.tencent.supersonic.headless.core.translator.QueryOptimizer=\ com.tencent.supersonic.headless.core.translator.DetailQueryOptimizer @@ -46,4 +47,5 @@ com.tencent.supersonic.headless.core.cache.QueryCache=\ ### headless-server SPIs com.tencent.supersonic.headless.server.modeller.SemanticModeller=\ - com.tencent.supersonic.headless.server.modeller.RuleSemanticModeller \ No newline at end of file + com.tencent.supersonic.headless.server.modeller.RuleSemanticModeller, \ + com.tencent.supersonic.headless.server.modeller.LLMSemanticModeller \ No newline at end of file diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2CompanyDemo.java b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2CompanyDemo.java index 030eb0ad3..df425a2c9 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2CompanyDemo.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2CompanyDemo.java @@ -7,26 +7,12 @@ import com.tencent.supersonic.chat.server.agent.Agent; import com.tencent.supersonic.chat.server.agent.AgentToolType; import com.tencent.supersonic.chat.server.agent.DatasetTool; import com.tencent.supersonic.chat.server.agent.ToolConfig; -import com.tencent.supersonic.chat.server.processor.execute.DataInterpretProcessor; import com.tencent.supersonic.common.pojo.ChatApp; import com.tencent.supersonic.common.pojo.JoinCondition; import com.tencent.supersonic.common.pojo.ModelRela; -import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; -import com.tencent.supersonic.common.pojo.enums.AppModule; -import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; -import com.tencent.supersonic.common.pojo.enums.TimeMode; -import com.tencent.supersonic.common.pojo.enums.TypeEnums; +import com.tencent.supersonic.common.pojo.enums.*; import com.tencent.supersonic.common.util.ChatAppManager; -import com.tencent.supersonic.headless.api.pojo.AggregateTypeDefaultConfig; -import com.tencent.supersonic.headless.api.pojo.DataSetDetail; -import com.tencent.supersonic.headless.api.pojo.DataSetModelConfig; -import com.tencent.supersonic.headless.api.pojo.Dim; -import com.tencent.supersonic.headless.api.pojo.DimensionTimeTypeParams; -import com.tencent.supersonic.headless.api.pojo.Identify; -import com.tencent.supersonic.headless.api.pojo.Measure; -import com.tencent.supersonic.headless.api.pojo.ModelDetail; -import com.tencent.supersonic.headless.api.pojo.QueryConfig; -import com.tencent.supersonic.headless.api.pojo.TimeDefaultConfig; +import com.tencent.supersonic.headless.api.pojo.*; import com.tencent.supersonic.headless.api.pojo.enums.DimensionType; import com.tencent.supersonic.headless.api.pojo.enums.IdentifyType; import com.tencent.supersonic.headless.api.pojo.request.DataSetReq; @@ -40,11 +26,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; +import java.util.*; @Component @Slf4j @@ -272,7 +254,6 @@ public class S2CompanyDemo extends S2BaseDemo { Map chatAppConfig = Maps.newHashMap(ChatAppManager.getAllApps(AppModule.CHAT)); chatAppConfig.values().forEach(app -> app.setChatModelId(demoChatModel.getId())); - chatAppConfig.get(DataInterpretProcessor.APP_KEY).setEnable(true); agent.setChatAppConfig(chatAppConfig); agentService.createAgent(agent, defaultUser); diff --git a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2VisitsDemo.java b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2VisitsDemo.java index 7f7cb56b4..77bc0081b 100644 --- a/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2VisitsDemo.java +++ b/launchers/standalone/src/main/java/com/tencent/supersonic/demo/S2VisitsDemo.java @@ -146,7 +146,8 @@ public class S2VisitsDemo extends S2BaseDemo { agent.setStatus(1); agent.setEnableSearch(1); agent.setExamples(Lists.newArrayList("近15天超音数访问次数汇总", "按部门统计超音数的访问人数", "对比alice和lucy的停留时长", - "过去30天访问次数最高的部门top3", "近1个月总访问次数超过100次的部门有几个", "过去半个月每个核心用户的总停留时长")); + "过去30天访问次数最高的部门top3", "近1个月总访问次数超过100次的部门有几个", "过去半个月每个核心用户的总停留时长", + "今年以来访问次数最高的一天是哪一天")); // configure tools ToolConfig toolConfig = new ToolConfig(); diff --git a/launchers/standalone/src/main/resources/META-INF/spring.factories b/launchers/standalone/src/main/resources/META-INF/spring.factories index 0a989e182..0c35cf42d 100644 --- a/launchers/standalone/src/main/resources/META-INF/spring.factories +++ b/launchers/standalone/src/main/resources/META-INF/spring.factories @@ -26,9 +26,10 @@ com.tencent.supersonic.headless.chat.parser.llm.DataSetResolver=\ com.tencent.supersonic.headless.core.translator.converter.QueryConverter=\ com.tencent.supersonic.headless.core.translator.converter.DefaultDimValueConverter,\ - com.tencent.supersonic.headless.core.translator.converter.SqlVariableParseConverter,\ - com.tencent.supersonic.headless.core.translator.converter.CalculateAggConverter,\ - com.tencent.supersonic.headless.core.translator.converter.ParserDefaultConverter + com.tencent.supersonic.headless.core.translator.converter.SqlVariableConverter,\ + com.tencent.supersonic.headless.core.translator.converter.MetricRatioConverter,\ + com.tencent.supersonic.headless.core.translator.converter.SqlQueryConverter,\ + com.tencent.supersonic.headless.core.translator.converter.StructQueryConverter com.tencent.supersonic.headless.core.translator.QueryOptimizer=\ com.tencent.supersonic.headless.core.translator.DetailQueryOptimizer diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/BaseTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/BaseTest.java index a48825177..cab07b66c 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/BaseTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/BaseTest.java @@ -12,7 +12,6 @@ import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum; import com.tencent.supersonic.common.service.ChatModelService; import com.tencent.supersonic.headless.api.pojo.SchemaElement; import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; -import com.tencent.supersonic.headless.api.pojo.response.ParseResp; import com.tencent.supersonic.headless.api.pojo.response.QueryState; import com.tencent.supersonic.util.DataUtils; import org.springframework.beans.factory.annotation.Autowired; diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/DetailTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/DetailTest.java index 1cf356a6f..966943e10 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/DetailTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/DetailTest.java @@ -20,7 +20,7 @@ public class DetailTest extends BaseTest { @Test public void test_detail_dimension() throws Exception { - QueryResult actualResult = submitNewChat("周杰伦流派和代表作", DataUtils.tagAgentId); + QueryResult actualResult = submitNewChat("周杰伦流派和代表作", DataUtils.singerAgentId); QueryResult expectedResult = new QueryResult(); SemanticParseInfo expectedParseInfo = new SemanticParseInfo(); @@ -31,7 +31,7 @@ public class DetailTest extends BaseTest { expectedParseInfo.setAggType(AggregateTypeEnum.NONE); QueryFilter dimensionFilter = - DataUtils.getFilter("singer_name", FilterOperatorEnum.EQUALS, "周杰伦", "歌手名", 8L); + DataUtils.getFilter("singer_name", FilterOperatorEnum.EQUALS, "周杰伦", "歌手名", 17L); expectedParseInfo.getDimensionFilters().add(dimensionFilter); expectedParseInfo.getDimensions() @@ -43,7 +43,7 @@ public class DetailTest extends BaseTest { @Test public void test_detail_filter() throws Exception { - QueryResult actualResult = submitNewChat("国风歌手", DataUtils.tagAgentId); + QueryResult actualResult = submitNewChat("国风歌手", DataUtils.singerAgentId); QueryResult expectedResult = new QueryResult(); SemanticParseInfo expectedParseInfo = new SemanticParseInfo(); diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MetricTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MetricTest.java index d0e251eed..2d809bb70 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MetricTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/chat/MetricTest.java @@ -9,6 +9,7 @@ import com.tencent.supersonic.headless.api.pojo.SemanticParseInfo; import com.tencent.supersonic.headless.api.pojo.request.QueryFilter; import com.tencent.supersonic.headless.chat.query.rule.metric.MetricFilterQuery; import com.tencent.supersonic.headless.chat.query.rule.metric.MetricGroupByQuery; +import com.tencent.supersonic.headless.chat.query.rule.metric.MetricModelQuery; import com.tencent.supersonic.headless.chat.query.rule.metric.MetricTopNQuery; import com.tencent.supersonic.util.DataUtils; import org.junit.jupiter.api.Order; @@ -28,13 +29,28 @@ import static com.tencent.supersonic.common.pojo.enums.AggregateTypeEnum.SUM; public class MetricTest extends BaseTest { @Test - public void testMetric() throws Exception { - QueryResult actualResult = submitNewChat("超音数 访问次数", DataUtils.metricAgentId); + public void testMetricModel() throws Exception { + QueryResult actualResult = submitNewChat("超音数 访问次数", DataUtils.productAgentId); + + QueryResult expectedResult = new QueryResult(); + SemanticParseInfo expectedParseInfo = new SemanticParseInfo(); + expectedResult.setChatContext(expectedParseInfo); + + expectedResult.setQueryMode(MetricModelQuery.QUERY_MODE); + expectedParseInfo.setAggType(NONE); + expectedParseInfo.getMetrics().add(DataUtils.getSchemaElement("访问次数")); + + expectedParseInfo.setDateInfo( + DataUtils.getDateConf(DateConf.DateMode.BETWEEN, unit, period, startDay, endDay)); + expectedParseInfo.setQueryType(QueryType.AGGREGATE); + + assertQueryResult(expectedResult, actualResult); + assert actualResult.getQueryResults().size() == 1; } @Test public void testMetricFilter() throws Exception { - QueryResult actualResult = submitNewChat("alice的访问次数", DataUtils.metricAgentId); + QueryResult actualResult = submitNewChat("alice的访问次数", DataUtils.productAgentId); QueryResult expectedResult = new QueryResult(); SemanticParseInfo expectedParseInfo = new SemanticParseInfo(); @@ -57,7 +73,8 @@ public class MetricTest extends BaseTest { @Test public void testMetricGroupBy() throws Exception { - QueryResult actualResult = submitNewChat("近7天超音数各部门的访问次数", DataUtils.metricAgentId); + System.setProperty("s2.test", "true"); + QueryResult actualResult = submitNewChat("近7天超音数各部门的访问次数", DataUtils.productAgentId); QueryResult expectedResult = new QueryResult(); SemanticParseInfo expectedParseInfo = new SemanticParseInfo(); @@ -79,7 +96,7 @@ public class MetricTest extends BaseTest { @Test public void testMetricFilterCompare() throws Exception { - QueryResult actualResult = submitNewChat("对比alice和lucy的访问次数", DataUtils.metricAgentId); + QueryResult actualResult = submitNewChat("对比alice和lucy的访问次数", DataUtils.productAgentId); QueryResult expectedResult = new QueryResult(); SemanticParseInfo expectedParseInfo = new SemanticParseInfo(); @@ -107,7 +124,7 @@ public class MetricTest extends BaseTest { @Test @Order(3) public void testMetricTopN() throws Exception { - QueryResult actualResult = submitNewChat("近3天访问次数最多的用户", DataUtils.metricAgentId); + QueryResult actualResult = submitNewChat("近3天访问次数最多的用户", DataUtils.productAgentId); QueryResult expectedResult = new QueryResult(); SemanticParseInfo expectedParseInfo = new SemanticParseInfo(); @@ -128,7 +145,7 @@ public class MetricTest extends BaseTest { @Test public void testMetricGroupBySum() throws Exception { - QueryResult actualResult = submitNewChat("近7天超音数各部门的访问次数总和", DataUtils.metricAgentId); + QueryResult actualResult = submitNewChat("近7天超音数各部门的访问次数总和", DataUtils.productAgentId); QueryResult expectedResult = new QueryResult(); SemanticParseInfo expectedParseInfo = new SemanticParseInfo(); expectedResult.setChatContext(expectedParseInfo); @@ -154,7 +171,7 @@ public class MetricTest extends BaseTest { String dateStr = textFormat.format(format.parse(startDay)); QueryResult actualResult = - submitNewChat(String.format("alice在%s的访问次数", dateStr), DataUtils.metricAgentId); + submitNewChat(String.format("alice在%s的访问次数", dateStr), DataUtils.productAgentId); QueryResult expectedResult = new QueryResult(); SemanticParseInfo expectedParseInfo = new SemanticParseInfo(); diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByMetricTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByMetricTest.java index b4e3dd653..7353dd526 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByMetricTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByMetricTest.java @@ -20,6 +20,7 @@ public class QueryByMetricTest extends BaseTest { @Test public void testWithMetricAndDimensionBizNames() throws Exception { + System.setProperty("s2.test", "true"); QueryMetricReq queryMetricReq = new QueryMetricReq(); queryMetricReq.setMetricNames(Arrays.asList("stay_hours", "pv")); queryMetricReq.setDimensionNames(Arrays.asList("user_name", "department")); diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByStructTest.java b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByStructTest.java index 7e1b3a37d..43a50e590 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByStructTest.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/headless/QueryByStructTest.java @@ -46,6 +46,7 @@ public class QueryByStructTest extends BaseTest { @Test public void testDetailQuery() throws Exception { + System.setProperty("s2.test", "true"); QueryStructReq queryStructReq = buildQueryStructReq(Arrays.asList("user_name", "department"), QueryType.DETAIL); SemanticQueryResp semanticQueryResp = @@ -86,6 +87,7 @@ public class QueryByStructTest extends BaseTest { @Test public void testFilterQuery() throws Exception { + System.setProperty("s2.test", "true"); QueryStructReq queryStructReq = buildQueryStructReq(Arrays.asList("department")); List dimensionFilters = new ArrayList<>(); Filter filter = new Filter(); diff --git a/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java b/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java index 4e28f61a3..a882293c1 100644 --- a/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java +++ b/launchers/standalone/src/test/java/com/tencent/supersonic/util/DataUtils.java @@ -15,10 +15,10 @@ import static java.time.LocalDate.now; public class DataUtils { - public static final Integer metricAgentId = 1; - public static final Integer tagAgentId = 2; + public static final Integer productAgentId = 1; + public static final Integer companyAgentId = 2; + public static final Integer singerAgentId = 3; public static final Integer ONE_TURNS_CHAT_ID = 10; - public static final Integer MULTI_TURNS_CHAT_ID = 11; private static final User user_test = User.getDefaultUser(); public static User getUser() { @@ -40,7 +40,7 @@ public class DataUtils { public static ChatParseReq getChatParseReq(Integer id, String query, boolean enableLLM) { ChatParseReq chatParseReq = new ChatParseReq(); chatParseReq.setQueryText(query); - chatParseReq.setAgentId(metricAgentId); + chatParseReq.setAgentId(productAgentId); chatParseReq.setChatId(id); chatParseReq.setUser(user_test); chatParseReq.setDisableLLM(!enableLLM); diff --git a/launchers/standalone/src/test/resources/s2-config.yaml b/launchers/standalone/src/test/resources/s2-config.yaml index e570dfb12..a95a4df7e 100644 --- a/launchers/standalone/src/test/resources/s2-config.yaml +++ b/launchers/standalone/src/test/resources/s2-config.yaml @@ -21,7 +21,7 @@ s2: date: true demo: - names: S2VisitsDemo,S2SingerDemo + names: S2VisitsDemo,S2SingerDemo,S2CompanyDemo enableLLM: false authentication: