diff --git a/common/src/main/java/com/tencent/supersonic/common/calcite/Configuration.java b/common/src/main/java/com/tencent/supersonic/common/calcite/Configuration.java index 516baeccf..4bae99e58 100644 --- a/common/src/main/java/com/tencent/supersonic/common/calcite/Configuration.java +++ b/common/src/main/java/com/tencent/supersonic/common/calcite/Configuration.java @@ -81,7 +81,7 @@ public class Configuration { .setUnquotedCasing(Casing.TO_UPPER).setConformance(sqlDialect.getConformance()) .setLex(Lex.BIG_QUERY); if (EngineType.HANADB.equals(engineType)) { - parserConfig = parserConfig.setQuoting(Quoting.DOUBLE_QUOTE); + parserConfig = parserConfig.setQuoting(Quoting.DOUBLE_QUOTE); } parserConfig = parserConfig.setQuotedCasing(Casing.UNCHANGED); parserConfig = parserConfig.setUnquotedCasing(Casing.UNCHANGED); diff --git a/common/src/main/java/com/tencent/supersonic/common/calcite/SqlDialectFactory.java b/common/src/main/java/com/tencent/supersonic/common/calcite/SqlDialectFactory.java index c549ce88a..ab7aea92f 100644 --- a/common/src/main/java/com/tencent/supersonic/common/calcite/SqlDialectFactory.java +++ b/common/src/main/java/com/tencent/supersonic/common/calcite/SqlDialectFactory.java @@ -21,10 +21,11 @@ public class SqlDialectFactory { .withDatabaseProduct(DatabaseProduct.BIG_QUERY).withLiteralQuoteString("'") .withLiteralEscapedQuoteString("''").withUnquotedCasing(Casing.UNCHANGED) .withQuotedCasing(Casing.UNCHANGED).withCaseSensitive(false); - public static final Context HANADB_CONTEXT = SqlDialect.EMPTY_CONTEXT - .withDatabaseProduct(DatabaseProduct.BIG_QUERY).withLiteralQuoteString("'") - .withIdentifierQuoteString("\"").withLiteralEscapedQuoteString("''").withUnquotedCasing(Casing.UNCHANGED) - .withQuotedCasing(Casing.UNCHANGED).withCaseSensitive(true); + public static final Context HANADB_CONTEXT = + SqlDialect.EMPTY_CONTEXT.withDatabaseProduct(DatabaseProduct.BIG_QUERY) + .withLiteralQuoteString("'").withIdentifierQuoteString("\"") + .withLiteralEscapedQuoteString("''").withUnquotedCasing(Casing.UNCHANGED) + .withQuotedCasing(Casing.UNCHANGED).withCaseSensitive(true); private static Map sqlDialectMap; static { diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java index 8c9fc1753..26a334e27 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/ModelDetail.java @@ -18,6 +18,8 @@ public class ModelDetail { private String queryType; + private String dbType; + private String sqlQuery; private String tableQuery; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaItem.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaItem.java index bf954b8cf..f69931e3c 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaItem.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/SchemaItem.java @@ -4,8 +4,7 @@ import com.google.common.base.Objects; import com.tencent.supersonic.common.pojo.RecordInfo; import com.tencent.supersonic.common.pojo.enums.SensitiveLevelEnum; import com.tencent.supersonic.common.pojo.enums.TypeEnums; -import lombok.Data; -import lombok.ToString; +import lombok.*; import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; @@ -17,19 +16,20 @@ import java.util.List; public class SchemaItem extends RecordInfo { private static String aliasSplit = ","; - private Long id; - private String name; + protected Long id; - private String bizName; + protected String name; - private String description; + protected String bizName; - private Integer status; + protected String description; - private TypeEnums typeEnum; + protected Integer status; - private Integer sensitiveLevel = SensitiveLevelEnum.LOW.getCode(); + protected TypeEnums typeEnum; + + protected Integer sensitiveLevel = SensitiveLevelEnum.LOW.getCode(); @Override public boolean equals(Object o) { diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DatabaseResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DatabaseResp.java index e60fb122c..d64419f40 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DatabaseResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DatabaseResp.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.api.pojo.response; import com.google.common.collect.Lists; import com.tencent.supersonic.common.pojo.RecordInfo; +import com.tencent.supersonic.common.util.AESEncryptionUtil; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -64,4 +65,8 @@ public class DatabaseResp extends RecordInfo { } return ""; } + + public String passwordDecrypt() { + return AESEncryptionUtil.aesDecryptECB(password); + } } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelResp.java index afead8d78..3ee0c863f 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/ModelResp.java @@ -1,17 +1,9 @@ package com.tencent.supersonic.headless.api.pojo.response; import com.google.common.collect.Lists; -import com.tencent.supersonic.headless.api.pojo.Dimension; -import com.tencent.supersonic.headless.api.pojo.DrillDownDimension; -import com.tencent.supersonic.headless.api.pojo.Field; -import com.tencent.supersonic.headless.api.pojo.Identify; -import com.tencent.supersonic.headless.api.pojo.ModelDetail; -import com.tencent.supersonic.headless.api.pojo.SchemaItem; +import com.tencent.supersonic.headless.api.pojo.*; import com.tencent.supersonic.headless.api.pojo.enums.IdentifyType; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import lombok.ToString; +import lombok.*; import org.springframework.util.CollectionUtils; import java.util.ArrayList; @@ -26,6 +18,7 @@ import java.util.stream.Collectors; @ToString(callSuper = true) @AllArgsConstructor @NoArgsConstructor +@Builder public class ModelResp extends SchemaItem { private Long domainId; @@ -62,6 +55,14 @@ public class ModelResp extends SchemaItem { return isOpen != null && isOpen == 1; } + public List getMeasures() { + return modelDetail.getMeasures(); + } + + public List getIdentifiers() { + return modelDetail.getIdentifiers(); + } + public List getTimeDimension() { if (modelDetail == null) { return Lists.newArrayList(); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/AbstractAccelerator.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/AbstractAccelerator.java index 175ff2497..be6847ab4 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/AbstractAccelerator.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/AbstractAccelerator.java @@ -3,10 +3,10 @@ package com.tencent.supersonic.headless.core.executor; import com.tencent.supersonic.common.calcite.Configuration; import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.headless.core.pojo.Materialization; +import com.tencent.supersonic.headless.core.translator.parser.TimeRange; import com.tencent.supersonic.headless.core.translator.parser.calcite.S2CalciteTable; import com.tencent.supersonic.headless.core.translator.parser.calcite.S2CalciteTable.Builder; import com.tencent.supersonic.headless.core.translator.parser.calcite.SchemaBuilder; -import com.tencent.supersonic.headless.core.translator.parser.s2sql.TimeRange; import lombok.extern.slf4j.Slf4j; import org.apache.calcite.adapter.enumerable.EnumerableRules; import org.apache.calcite.config.CalciteConnectionConfigImpl; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java index a218fa0fb..947fa58ab 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/executor/JdbcExecutor.java @@ -1,8 +1,8 @@ package com.tencent.supersonic.headless.core.executor; import com.tencent.supersonic.common.util.ContextUtils; +import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; -import com.tencent.supersonic.headless.core.pojo.Database; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.utils.ComponentFactory; import com.tencent.supersonic.headless.core.utils.SqlUtils; @@ -38,7 +38,7 @@ public class JdbcExecutor implements QueryExecutor { SqlUtils sqlUtils = ContextUtils.getBean(SqlUtils.class); String sql = StringUtils.normalizeSpace(queryStatement.getSql()); log.info("executing SQL: {}", sql); - Database database = queryStatement.getOntology().getDatabase(); + DatabaseResp database = queryStatement.getOntology().getDatabase(); SemanticQueryResp queryResultWithColumns = new SemanticQueryResp(); try { SqlUtils sqlUtil = sqlUtils.init(database); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/DataModel.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/DataModel.java deleted file mode 100644 index 16030951f..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/DataModel.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.tencent.supersonic.headless.core.pojo; - -import com.tencent.supersonic.headless.api.pojo.Identify; -import com.tencent.supersonic.headless.api.pojo.Measure; -import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; -import com.tencent.supersonic.headless.core.translator.parser.s2sql.Materialization; -import lombok.Builder; -import lombok.Data; - -import java.util.List; - -@Data -@Builder -public class DataModel { - - private Long id; - - private String name; - - private Long modelId; - - private String type; - - private String sqlQuery; - - private String tableQuery; - - private List identifiers; - - private List dimensions; - - private List measures; - - private String aggTime; - - private com.tencent.supersonic.headless.core.translator.parser.s2sql.Materialization.TimePartType timePartType = - Materialization.TimePartType.None; -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Database.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Database.java deleted file mode 100644 index 5227166cc..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Database.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.tencent.supersonic.headless.core.pojo; - -import com.google.common.collect.Lists; -import com.tencent.supersonic.common.pojo.RecordInfo; -import com.tencent.supersonic.common.pojo.enums.EngineType; -import com.tencent.supersonic.common.util.AESEncryptionUtil; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.util.List; - -@Data -@AllArgsConstructor -@NoArgsConstructor -@Builder -public class Database extends RecordInfo { - - private Long id; - - private Long domainId; - - private String name; - - private String description; - - private String version; - - private String url; - - private String username; - - private String password; - - private String database; - - private String schema; - /** mysql,clickhouse */ - private EngineType type; - - private List admins = Lists.newArrayList(); - - private List viewers = Lists.newArrayList(); - - public String passwordDecrypt() { - return AESEncryptionUtil.aesDecryptECB(password); - } -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/JdbcDataSource.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/JdbcDataSource.java index 96b553b1d..99dd36ade 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/JdbcDataSource.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/JdbcDataSource.java @@ -2,6 +2,7 @@ package com.tencent.supersonic.headless.core.pojo; import com.alibaba.druid.pool.DruidDataSource; import com.tencent.supersonic.headless.api.pojo.enums.DataType; +import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.core.utils.JdbcDataSourceUtils; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -106,7 +107,7 @@ public class JdbcDataSource { } } - public void removeDatasource(Database database) { + public void removeDatasource(DatabaseResp database) { String key = getDataSourceKey(database); @@ -128,7 +129,7 @@ public class JdbcDataSource { } } - public DruidDataSource getDataSource(Database database) throws RuntimeException { + public DruidDataSource getDataSource(DatabaseResp database) throws RuntimeException { String name = database.getName(); String jdbcUrl = database.getUrl(); @@ -239,7 +240,7 @@ public class JdbcDataSource { return druidDataSource; } - private String getDataSourceKey(Database database) { + private String getDataSourceKey(DatabaseResp database) { return JdbcDataSourceUtils.getKey(database.getName(), database.getUrl(), database.getUsername(), database.passwordDecrypt(), "", false); } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Ontology.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Ontology.java index 4067b7e46..e75e7c008 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Ontology.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/Ontology.java @@ -1,7 +1,9 @@ package com.tencent.supersonic.headless.core.pojo; +import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import lombok.Data; import java.util.*; @@ -14,12 +16,16 @@ import java.util.stream.Collectors; @Data public class Ontology { - private Database database; - private Map dataModelMap = new HashMap<>(); - private List metrics = new ArrayList<>(); + private DatabaseResp database; + private Map modelMap = new HashMap<>(); + private Map> metricMap = new HashMap<>(); private Map> dimensionMap = new HashMap<>(); private List joinRelations; + public List getMetrics() { + return metricMap.values().stream().flatMap(Collection::stream).collect(Collectors.toList()); + } + public List getDimensions() { return dimensionMap.values().stream().flatMap(Collection::stream) .collect(Collectors.toList()); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java index e780f7aac..3cc5eb3b4 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/DefaultSemanticTranslator.java @@ -68,7 +68,8 @@ public class DefaultSemanticTranslator implements SemanticTranslator { List> tables = new ArrayList<>(); tables.add(Pair.of(ontologyInnerTable, ontologyInnerSql)); if (sqlQuery.isSupportWith()) { - EngineType engineType = queryStatement.getOntology().getDatabase().getType(); + EngineType engineType = + EngineType.fromString(queryStatement.getOntology().getDatabase().getType()); if (!SqlMergeWithUtils.hasWith(engineType, ontologyQuerySql)) { String withSql = "with " + tables.stream() .map(t -> String.format("%s as (%s)", t.getLeft(), t.getRight())) diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/MetricRatioConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/MetricRatioConverter.java index 72b6f7f66..868d8c0d4 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/MetricRatioConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/MetricRatioConverter.java @@ -6,7 +6,7 @@ import com.tencent.supersonic.common.pojo.enums.DatePeriodEnum; import com.tencent.supersonic.common.pojo.enums.EngineType; import com.tencent.supersonic.common.util.ContextUtils; import com.tencent.supersonic.headless.api.pojo.enums.AggOption; -import com.tencent.supersonic.headless.core.pojo.Database; +import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.core.pojo.OntologyQuery; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.SqlQuery; @@ -59,8 +59,9 @@ public class MetricRatioConverter implements QueryConverter { @Override public void convert(QueryStatement queryStatement) throws Exception { - Database database = queryStatement.getOntology().getDatabase(); - generateRatioSql(queryStatement, database.getType(), database.getVersion()); + DatabaseResp database = queryStatement.getOntology().getDatabase(); + generateRatioSql(queryStatement, EngineType.fromString(database.getType()), + database.getVersion()); } /** Ratio */ diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlVariableConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlVariableConverter.java index d2bf2a22c..8a35c2b93 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlVariableConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlVariableConverter.java @@ -3,7 +3,6 @@ package com.tencent.supersonic.headless.core.translator.converter; import com.tencent.supersonic.headless.api.pojo.enums.ModelDefineType; import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; -import com.tencent.supersonic.headless.core.pojo.DataModel; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.utils.SqlVariableParseUtils; import lombok.extern.slf4j.Slf4j; @@ -36,9 +35,9 @@ public class SqlVariableConverter implements QueryConverter { SqlVariableParseUtils.parse(modelResp.getModelDetail().getSqlQuery(), modelResp.getModelDetail().getSqlVariables(), queryStatement.getStructQuery().getParams()); - DataModel dataModel = - queryStatement.getOntology().getDataModelMap().get(modelResp.getBizName()); - dataModel.setSqlQuery(sqlParsed); + ModelResp dataModel = + queryStatement.getOntology().getModelMap().get(modelResp.getBizName()); + dataModel.getModelDetail().setSqlQuery(sqlParsed); } } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/StructQueryConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/StructQueryConverter.java index 45de899bf..f84a263dd 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/StructQueryConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/StructQueryConverter.java @@ -1,7 +1,8 @@ package com.tencent.supersonic.headless.core.translator.converter; +import com.tencent.supersonic.common.pojo.enums.EngineType; import com.tencent.supersonic.common.util.ContextUtils; -import com.tencent.supersonic.headless.core.pojo.Database; +import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.SqlQuery; import com.tencent.supersonic.headless.core.pojo.StructQuery; @@ -40,8 +41,9 @@ public class StructQueryConverter implements QueryConverter { sqlGenerateUtils.generateWhere(structQuery, null), sqlGenerateUtils.getGroupBy(structQuery), sqlGenerateUtils.getOrderBy(structQuery), sqlGenerateUtils.getLimit(structQuery)); - Database database = queryStatement.getOntology().getDatabase(); - if (!sqlGenerateUtils.isSupportWith(database.getType(), database.getVersion())) { + DatabaseResp database = queryStatement.getOntology().getDatabase(); + if (!sqlGenerateUtils.isSupportWith(EngineType.fromString(database.getType()), + database.getVersion())) { sqlQuery.setSupportWith(false); sql = String.format("select %s from %s t0 %s %s %s", sqlGenerateUtils.getSelect(structQuery), dsTable, diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/Constants.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/Constants.java similarity index 94% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/Constants.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/Constants.java index 0ad7b9d0f..10691a740 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/Constants.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/Constants.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.translator.parser.s2sql; +package com.tencent.supersonic.headless.core.translator.parser; public class Constants { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/Materialization.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/Materialization.java similarity index 94% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/Materialization.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/Materialization.java index c882ed1d7..22dba6edd 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/Materialization.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/Materialization.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.translator.parser.s2sql; +package com.tencent.supersonic.headless.core.translator.parser; import lombok.Builder; import lombok.Data; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/MaterializationElement.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/MaterializationElement.java similarity index 73% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/MaterializationElement.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/MaterializationElement.java index 620b0d767..5824a313f 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/MaterializationElement.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/MaterializationElement.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.translator.parser.s2sql; +package com.tencent.supersonic.headless.core.translator.parser; import lombok.Builder; import lombok.Data; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/TimeRange.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/TimeRange.java similarity index 66% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/TimeRange.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/TimeRange.java index 742c7df36..f000a3a05 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/s2sql/TimeRange.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/TimeRange.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.translator.parser.s2sql; +package com.tencent.supersonic.headless.core.translator.parser; import lombok.Builder; import lombok.Data; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/CalciteQueryParser.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/CalciteQueryParser.java index c02fc7b3b..89d4fffd5 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/CalciteQueryParser.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/CalciteQueryParser.java @@ -6,7 +6,6 @@ import com.tencent.supersonic.headless.core.translator.parser.QueryParser; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; -/** the calcite parse implements */ @Component("CalciteQueryParser") @Slf4j public class CalciteQueryParser implements QueryParser { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/DataModelNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/DataModelNode.java similarity index 79% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/DataModelNode.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/DataModelNode.java index 459db41aa..1d95a5f51 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/DataModelNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/DataModelNode.java @@ -1,4 +1,4 @@ -package com.tencent.supersonic.headless.core.translator.parser.calcite.node; +package com.tencent.supersonic.headless.core.translator.parser.calcite; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -6,17 +6,19 @@ import com.google.common.collect.Sets; import com.tencent.supersonic.common.calcite.Configuration; import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; import com.tencent.supersonic.common.pojo.enums.EngineType; +import com.tencent.supersonic.headless.api.pojo.Dimension; import com.tencent.supersonic.headless.api.pojo.Identify; import com.tencent.supersonic.headless.api.pojo.Measure; import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; -import com.tencent.supersonic.headless.core.pojo.DataModel; +import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.core.pojo.JoinRelation; import com.tencent.supersonic.headless.core.pojo.Ontology; import com.tencent.supersonic.headless.core.pojo.OntologyQuery; -import com.tencent.supersonic.headless.core.translator.parser.calcite.SchemaBuilder; -import com.tencent.supersonic.headless.core.translator.parser.s2sql.Constants; +import com.tencent.supersonic.headless.core.translator.parser.Constants; import lombok.extern.slf4j.Slf4j; -import org.apache.calcite.sql.*; +import org.apache.calcite.sql.SqlDataTypeSpec; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlUserDefinedTypeNameSpec; import org.apache.calcite.sql.parser.SqlParser; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.validate.SqlValidatorScope; @@ -28,28 +30,32 @@ import java.util.stream.Collectors; @Slf4j public class DataModelNode extends SemanticNode { - public static SqlNode build(DataModel dataModel, SqlValidatorScope scope) throws Exception { + public static SqlNode build(ModelResp dataModel, SqlValidatorScope scope) throws Exception { String sqlTable = ""; - if (dataModel.getSqlQuery() != null && !dataModel.getSqlQuery().isEmpty()) { - sqlTable = dataModel.getSqlQuery(); - } else if (dataModel.getTableQuery() != null && !dataModel.getTableQuery().isEmpty()) { - if (dataModel.getType().equalsIgnoreCase(EngineType.POSTGRESQL.getName())) { - String fullTableName = - String.join(".public.", dataModel.getTableQuery().split("\\.")); + if (dataModel.getModelDetail().getSqlQuery() != null + && !dataModel.getModelDetail().getSqlQuery().isEmpty()) { + sqlTable = dataModel.getModelDetail().getSqlQuery(); + } else if (dataModel.getModelDetail().getTableQuery() != null + && !dataModel.getModelDetail().getTableQuery().isEmpty()) { + if (dataModel.getModelDetail().getDbType() + .equalsIgnoreCase(EngineType.POSTGRESQL.getName())) { + String fullTableName = String.join(".public.", + dataModel.getModelDetail().getTableQuery().split("\\.")); sqlTable = "select * from " + fullTableName; } else { - sqlTable = "select * from " + dataModel.getTableQuery(); + sqlTable = "select * from " + dataModel.getModelDetail().getTableQuery(); } } if (sqlTable.isEmpty()) { throw new Exception("DataModelNode build error [tableSqlNode not found]"); } - SqlNode source = getTable(sqlTable, scope, EngineType.fromString(dataModel.getType())); + SqlNode source = getTable(sqlTable, scope, + EngineType.fromString(dataModel.getModelDetail().getDbType())); addSchema(scope, dataModel, sqlTable); return buildAs(dataModel.getName(), source); } - private static void addSchema(SqlValidatorScope scope, DataModel datasource, String table) + private static void addSchema(SqlValidatorScope scope, ModelResp datasource, String table) throws Exception { Map> sqlTable = SqlSelectHelper.getFieldsWithSubQuery(table); for (Map.Entry> entry : sqlTable.entrySet()) { @@ -63,22 +69,22 @@ public class DataModelNode extends SemanticNode { } } - private static void addSchemaTable(SqlValidatorScope scope, DataModel datasource, String db, + private static void addSchemaTable(SqlValidatorScope scope, ModelResp dataModel, String db, String tb, Set fields) throws Exception { Set dateInfo = new HashSet<>(); Set dimensions = new HashSet<>(); Set metrics = new HashSet<>(); - EngineType engineType = EngineType.fromString(datasource.getType()); - for (DimSchemaResp d : datasource.getDimensions()) { + EngineType engineType = EngineType.fromString(dataModel.getModelDetail().getDbType()); + for (Dimension d : dataModel.getModelDetail().getDimensions()) { List identifiers = expand(SemanticNode.parse(d.getExpr(), scope, engineType), scope); identifiers.forEach(i -> dimensions.add(i.toString())); dimensions.add(d.getName()); } - for (Identify i : datasource.getIdentifiers()) { + for (Identify i : dataModel.getIdentifiers()) { dimensions.add(i.getName()); } - for (Measure m : datasource.getMeasures()) { + for (Measure m : dataModel.getMeasures()) { List identifiers = expand(SemanticNode.parse(m.getExpr(), scope, engineType), scope); identifiers.forEach(i -> { @@ -93,7 +99,7 @@ public class DataModelNode extends SemanticNode { for (String field : fields) { if (!metrics.contains(field) && !dimensions.contains(field)) { dimensions.add(field); - log.info("add column {} {}", datasource.getName(), field); + log.info("add column {} {}", dataModel.getName(), field); } } SchemaBuilder.addSourceView(scope.getValidator().getCatalogReader().getRootSchema(), db, tb, @@ -121,10 +127,6 @@ public class DataModelNode extends SemanticNode { return sqlNode; } - public static String getNames(List dataModelList) { - return dataModelList.stream().map(DataModel::getName).collect(Collectors.joining("_")); - } - public static void getQueryDimensionMeasure(Ontology ontology, OntologyQuery ontologyQuery, Set queryDimensions, Set queryMeasures) { ontologyQuery.getMetrics().forEach(m -> { @@ -139,7 +141,7 @@ public class DataModelNode extends SemanticNode { }); } - public static List getQueryDataModelsV2(Ontology ontology, OntologyQuery query) { + public static List getQueryDataModelsV2(Ontology ontology, OntologyQuery query) { // first, sort models based on the number of query metrics Map modelMetricCount = Maps.newHashMap(); query.getMetrics().forEach(m -> { @@ -171,11 +173,11 @@ public class DataModelNode extends SemanticNode { Set dataModelNames = Sets.newLinkedHashSet(); dataModelNames.addAll(dimDataModels); dataModelNames.addAll(metricsDataModels); - return dataModelNames.stream().map(bizName -> ontology.getDataModelMap().get(bizName)) + return dataModelNames.stream().map(bizName -> ontology.getModelMap().get(bizName)) .collect(Collectors.toList()); } - public static List getQueryDataModels(Ontology ontology, + public static List getQueryDataModels(Ontology ontology, OntologyQuery ontologyQuery) { // get query measures and dimensions Set queryMeasures = new HashSet<>(); @@ -183,7 +185,7 @@ public class DataModelNode extends SemanticNode { getQueryDimensionMeasure(ontology, ontologyQuery, queryDimensions, queryMeasures); // first, find the base model - DataModel baseDataModel = findBaseModel(ontology, ontologyQuery); + ModelResp baseDataModel = findBaseModel(ontology, ontologyQuery); if (Objects.isNull(baseDataModel)) { throw new RuntimeException( String.format("could not find matching dataModel, dimensions:%s, measures:%s", @@ -196,7 +198,7 @@ public class DataModelNode extends SemanticNode { } // second, traverse the ontology to find other related dataModels - List relatedDataModels = findRelatedModelsByRelation(ontology, ontologyQuery, + List relatedDataModels = findRelatedModelsByRelation(ontology, ontologyQuery, baseDataModel, queryDimensions, queryMeasures); if (CollectionUtils.isEmpty(relatedDataModels)) { relatedDataModels = findRelatedModelsByIdentifier(ontology, baseDataModel, @@ -210,8 +212,8 @@ public class DataModelNode extends SemanticNode { return relatedDataModels; } - private static DataModel findBaseModel(Ontology ontology, OntologyQuery query) { - DataModel dataModel = null; + private static ModelResp findBaseModel(Ontology ontology, OntologyQuery query) { + ModelResp dataModel = null; // first, try to find the model with the most query metrics Map modelMetricCount = Maps.newHashMap(); query.getMetrics().forEach(m -> { @@ -226,7 +228,7 @@ public class DataModelNode extends SemanticNode { .sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())).map(e -> e.getKey()) .findFirst(); if (baseModelName.isPresent()) { - dataModel = ontology.getDataModelMap().get(baseModelName.get()); + dataModel = ontology.getModelMap().get(baseModelName.get()); } else { // second, try to find the model with the most query dimensions Map modelDimCount = Maps.newHashMap(); @@ -242,20 +244,20 @@ public class DataModelNode extends SemanticNode { .sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())) .map(e -> e.getKey()).findFirst(); if (baseModelName.isPresent()) { - dataModel = ontology.getDataModelMap().get(baseModelName.get()); + dataModel = ontology.getModelMap().get(baseModelName.get()); } } return dataModel; } - private static boolean checkMatch(DataModel baseDataModel, Set queryMeasures, + private static boolean checkMatch(ModelResp baseDataModel, Set queryMeasures, Set queryDimension) { boolean isAllMatch = true; Set baseMeasures = baseDataModel.getMeasures().stream().map(Measure::getName) .collect(Collectors.toSet()); - Set baseDimensions = baseDataModel.getDimensions().stream() - .map(DimSchemaResp::getName).collect(Collectors.toSet()); + Set baseDimensions = baseDataModel.getModelDetail().getDimensions().stream() + .map(Dimension::getName).collect(Collectors.toSet()); baseDataModel.getIdentifiers().forEach(i -> baseDimensions.add(i.getName())); baseMeasures.retainAll(queryMeasures); @@ -282,11 +284,11 @@ public class DataModelNode extends SemanticNode { return isAllMatch; } - private static List findRelatedModelsByRelation(Ontology ontology, - OntologyQuery ontologyQuery, DataModel baseDataModel, Set queryDimensions, + private static List findRelatedModelsByRelation(Ontology ontology, + OntologyQuery ontologyQuery, ModelResp baseDataModel, Set queryDimensions, Set queryMeasures) { Set joinDataModelNames = new HashSet<>(); - List joinDataModels = new ArrayList<>(); + List joinDataModels = new ArrayList<>(); Set before = new HashSet<>(); before.add(baseDataModel.getName()); @@ -305,14 +307,15 @@ public class DataModelNode extends SemanticNode { } boolean isMatch = false; boolean isRight = before.contains(joinRelation.getLeft()); - DataModel other = isRight ? ontology.getDataModelMap().get(joinRelation.getRight()) - : ontology.getDataModelMap().get(joinRelation.getLeft()); + ModelResp other = isRight ? ontology.getModelMap().get(joinRelation.getRight()) + : ontology.getModelMap().get(joinRelation.getLeft()); String joinDimName = isRight ? joinRelation.getJoinCondition().get(0).getRight() : joinRelation.getJoinCondition().get(0).getLeft(); if (!queryDimensions.isEmpty()) { - Set linkDimension = other.getDimensions().stream() - .map(DimSchemaResp::getName).collect(Collectors.toSet()); - other.getIdentifiers().forEach(i -> linkDimension.add(i.getName())); + Set linkDimension = other.getModelDetail().getDimensions().stream() + .map(Dimension::getName).collect(Collectors.toSet()); + other.getModelDetail().getIdentifiers() + .forEach(i -> linkDimension.add(i.getName())); linkDimension.retainAll(queryDimensions); if (!linkDimension.isEmpty()) { isMatch = true; @@ -320,8 +323,8 @@ public class DataModelNode extends SemanticNode { // ontologyQuery.getDimensions().add(joinDimName); } } - Set linkMeasure = other.getMeasures().stream().map(Measure::getName) - .collect(Collectors.toSet()); + Set linkMeasure = other.getModelDetail().getMeasures().stream() + .map(Measure::getName).collect(Collectors.toSet()); linkMeasure.retainAll(queryMeasures); if (!linkMeasure.isEmpty()) { isMatch = true; @@ -360,7 +363,7 @@ public class DataModelNode extends SemanticNode { orders.entrySet().stream() .sorted((entry1, entry2) -> entry2.getValue().compareTo(entry1.getValue())) // 倒序排序 .forEach(d -> { - joinDataModels.add(ontology.getDataModelMap().get(d.getKey())); + joinDataModels.add(ontology.getModelMap().get(d.getKey())); }); } return joinDataModels; @@ -381,36 +384,37 @@ public class DataModelNode extends SemanticNode { } } - private static List findRelatedModelsByIdentifier(Ontology ontology, - DataModel baseDataModel, Set queryDimension, Set measures) { - Set baseIdentifiers = baseDataModel.getIdentifiers().stream().map(Identify::getName) - .collect(Collectors.toSet()); + private static List findRelatedModelsByIdentifier(Ontology ontology, + ModelResp baseDataModel, Set queryDimension, Set measures) { + Set baseIdentifiers = baseDataModel.getModelDetail().getIdentifiers().stream() + .map(Identify::getName).collect(Collectors.toSet()); if (baseIdentifiers.isEmpty()) { return Collections.EMPTY_LIST; } Set linkDataSourceName = new HashSet<>(); - List linkDataModels = new ArrayList<>(); - for (Map.Entry entry : ontology.getDataModelMap().entrySet()) { + List linkDataModels = new ArrayList<>(); + for (Map.Entry entry : ontology.getModelMap().entrySet()) { if (entry.getKey().equalsIgnoreCase(baseDataModel.getName())) { continue; } - long identifierNum = entry.getValue().getIdentifiers().stream().map(Identify::getName) - .filter(baseIdentifiers::contains).count(); + long identifierNum = entry.getValue().getModelDetail().getIdentifiers().stream() + .map(Identify::getName).filter(baseIdentifiers::contains).count(); if (identifierNum > 0) { boolean isMatch = false; if (!queryDimension.isEmpty()) { - Set linkDimension = entry.getValue().getDimensions().stream() - .map(DimSchemaResp::getName).collect(Collectors.toSet()); - entry.getValue().getIdentifiers().forEach(i -> linkDimension.add(i.getName())); + Set linkDimension = entry.getValue().getModelDetail().getDimensions() + .stream().map(Dimension::getName).collect(Collectors.toSet()); + entry.getValue().getModelDetail().getIdentifiers() + .forEach(i -> linkDimension.add(i.getName())); linkDimension.retainAll(queryDimension); if (!linkDimension.isEmpty()) { isMatch = true; } } if (!measures.isEmpty()) { - Set linkMeasure = entry.getValue().getMeasures().stream() - .map(Measure::getName).collect(Collectors.toSet()); + Set linkMeasure = entry.getValue().getModelDetail().getMeasures() + .stream().map(Measure::getName).collect(Collectors.toSet()); linkMeasure.retainAll(measures); if (!linkMeasure.isEmpty()) { isMatch = true; @@ -432,10 +436,10 @@ public class DataModelNode extends SemanticNode { } } for (String linkName : linkDataSourceName) { - linkDataModels.add(ontology.getDataModelMap().get(linkName)); + linkDataModels.add(ontology.getModelMap().get(linkName)); } if (!CollectionUtils.isEmpty(linkDataModels)) { - List all = new ArrayList<>(); + List all = new ArrayList<>(); all.add(baseDataModel); all.addAll(linkDataModels); return all; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/FilterToGroupScanRule.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/FilterToGroupScanRule.java index 3cf783931..522ad3dad 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/FilterToGroupScanRule.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/FilterToGroupScanRule.java @@ -25,7 +25,9 @@ import java.util.List; import java.util.Objects; import java.util.Optional; -/** push down the time filter into group using the RuntimeOptions defined minMaxTime */ +/** + * push down the time filter into group using the RuntimeOptions defined minMaxTime + */ public class FilterToGroupScanRule extends RelRule implements TransformationRule { public static FilterTableScanRule.Config DEFAULT = diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/S2CalciteSchema.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/S2CalciteSchema.java index 34556a6e2..128a4c1a2 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/S2CalciteSchema.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/S2CalciteSchema.java @@ -2,7 +2,7 @@ package com.tencent.supersonic.headless.core.translator.parser.calcite; import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; -import com.tencent.supersonic.headless.core.pojo.DataModel; +import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.core.pojo.JoinRelation; import com.tencent.supersonic.headless.core.pojo.Ontology; import lombok.Builder; @@ -29,8 +29,8 @@ public class S2CalciteSchema extends AbstractSchema { return this; } - public Map getDataModels() { - return ontology.getDataModelMap(); + public Map getDataModels() { + return ontology.getModelMap(); } public List getMetrics() { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/SchemaBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/SchemaBuilder.java index e7bcfb3b3..e3c0592ff 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/SchemaBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/SchemaBuilder.java @@ -26,14 +26,14 @@ public class SchemaBuilder { public static final String MATERIALIZATION_SYS_FIELD_DATE = "C1"; public static final String MATERIALIZATION_SYS_FIELD_DATA = "C2"; - public static SqlValidatorScope getScope(S2CalciteSchema schema) throws Exception { + public static SqlValidatorScope getScope(S2CalciteSchema schema) { Map nameToTypeMap = new HashMap<>(); CalciteSchema rootSchema = CalciteSchema.createRootSchema(true, false); rootSchema.add(schema.getSchemaKey(), schema); Prepare.CatalogReader catalogReader = new CalciteCatalogReader(rootSchema, Collections.singletonList(schema.getSchemaKey()), Configuration.typeFactory, Configuration.config); - EngineType engineType = schema.getOntology().getDatabase().getType(); + EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType()); S2SQLSqlValidatorImpl s2SQLSqlValidator = new S2SQLSqlValidatorImpl(Configuration.operatorTable, catalogReader, Configuration.typeFactory, Configuration.getValidatorConfig(engineType)); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/SemanticNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/SemanticNode.java similarity index 98% rename from headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/SemanticNode.java rename to headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/SemanticNode.java index 88f2a7d80..04f996825 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/SemanticNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/SemanticNode.java @@ -1,12 +1,10 @@ -package com.tencent.supersonic.headless.core.translator.parser.calcite.node; +package com.tencent.supersonic.headless.core.translator.parser.calcite; import com.tencent.supersonic.common.calcite.Configuration; import com.tencent.supersonic.common.calcite.SemanticSqlDialect; import com.tencent.supersonic.common.calcite.SqlDialectFactory; import com.tencent.supersonic.common.pojo.enums.EngineType; -import com.tencent.supersonic.headless.core.translator.parser.calcite.FilterToGroupScanRule; -import com.tencent.supersonic.headless.core.translator.parser.calcite.S2CalciteSchema; -import com.tencent.supersonic.headless.core.translator.parser.s2sql.Constants; +import com.tencent.supersonic.headless.core.translator.parser.Constants; import lombok.extern.slf4j.Slf4j; import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.plan.hep.HepPlanner; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/SqlBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/SqlBuilder.java index e46923e8e..fc700ecdd 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/SqlBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/SqlBuilder.java @@ -2,84 +2,65 @@ package com.tencent.supersonic.headless.core.translator.parser.calcite; import com.tencent.supersonic.common.calcite.Configuration; import com.tencent.supersonic.common.pojo.enums.EngineType; -import com.tencent.supersonic.headless.core.pojo.DataModel; -import com.tencent.supersonic.headless.core.pojo.Database; -import com.tencent.supersonic.headless.core.pojo.OntologyQuery; -import com.tencent.supersonic.headless.core.pojo.QueryStatement; -import com.tencent.supersonic.headless.core.translator.parser.calcite.node.DataModelNode; -import com.tencent.supersonic.headless.core.translator.parser.calcite.node.SemanticNode; -import com.tencent.supersonic.headless.core.translator.parser.calcite.render.JoinRender; -import com.tencent.supersonic.headless.core.translator.parser.calcite.render.Renderer; +import com.tencent.supersonic.headless.api.pojo.Dimension; +import com.tencent.supersonic.headless.api.pojo.Identify; +import com.tencent.supersonic.headless.api.pojo.enums.IdentifyType; +import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; +import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.ModelResp; +import com.tencent.supersonic.headless.core.pojo.*; +import com.tencent.supersonic.headless.core.translator.parser.Constants; import lombok.extern.slf4j.Slf4j; -import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.*; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParser; +import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.validate.SqlValidatorScope; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Triple; +import org.springframework.util.CollectionUtils; -import java.util.LinkedList; -import java.util.List; -import java.util.ListIterator; -import java.util.Objects; +import java.util.*; +import java.util.stream.Collectors; @Slf4j public class SqlBuilder { private final S2CalciteSchema schema; - private OntologyQuery ontologyQuery; - private SqlValidatorScope scope; - private SqlNode parserNode; + private final SqlValidatorScope scope; public SqlBuilder(S2CalciteSchema schema) { this.schema = schema; + this.scope = SchemaBuilder.getScope(schema); } public String buildOntologySql(QueryStatement queryStatement) throws Exception { - this.ontologyQuery = queryStatement.getOntologyQuery(); + OntologyQuery ontologyQuery = queryStatement.getOntologyQuery(); if (ontologyQuery.getLimit() == null) { ontologyQuery.setLimit(0L); } - buildParseNode(); - Database database = queryStatement.getOntology().getDatabase(); - optimizeParseNode(database.getType()); - return getSql(database.getType()); - } - - private void buildParseNode() throws Exception { // find relevant data models - scope = SchemaBuilder.getScope(schema); - List dataModels = + List dataModels = DataModelNode.getQueryDataModelsV2(schema.getOntology(), ontologyQuery); if (dataModels == null || dataModels.isEmpty()) { throw new Exception("data model not found"); } - LinkedList builders = new LinkedList<>(); - builders.add(new JoinRender()); - ListIterator it = builders.listIterator(); - int i = 0; - Renderer previous = null; - while (it.hasNext()) { - Renderer renderer = it.next(); - if (previous != null) { - previous.render(ontologyQuery, dataModels, scope, schema); - renderer.setTable(previous.builderAs(DataModelNode.getNames(dataModels) + "_" + i)); - i++; - } - previous = renderer; - } - builders.getLast().render(ontologyQuery, dataModels, scope, schema); - parserNode = builders.getLast().build(); - } - - public String getSql(EngineType engineType) { + TableView tableView = render(ontologyQuery, dataModels, scope, schema); + SqlNode parserNode = tableView.build(); + DatabaseResp database = queryStatement.getOntology().getDatabase(); + EngineType engineType = EngineType.fromString(database.getType()); + parserNode = optimizeParseNode(parserNode, engineType); return SemanticNode.getSql(parserNode, engineType); } - private void optimizeParseNode(EngineType engineType) { + private SqlNode optimizeParseNode(SqlNode parserNode, EngineType engineType) { if (Objects.isNull(schema.getRuntimeOptions()) || Objects.isNull(schema.getRuntimeOptions().getEnableOptimize()) || !schema.getRuntimeOptions().getEnableOptimize()) { - return; + return parserNode; } SqlNode optimizeNode = null; @@ -94,8 +75,233 @@ public class SqlBuilder { } if (Objects.nonNull(optimizeNode)) { - parserNode = optimizeNode; + return optimizeNode; } + + return parserNode; + } + + private TableView render(OntologyQuery ontologyQuery, List dataModels, + SqlValidatorScope scope, S2CalciteSchema schema) throws Exception { + SqlNode left = null; + TableView leftTable = null; + TableView outerTable = new TableView(); + Map outerSelect = new HashMap<>(); + Map beforeModels = new HashMap<>(); + EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType()); + + for (int i = 0; i < dataModels.size(); i++) { + final ModelResp dataModel = dataModels.get(i); + final Set queryDimensions = + ontologyQuery.getDimensionsByModel(dataModel.getId()); + final Set queryMetrics = + ontologyQuery.getMetricsByModel(dataModel.getId()); + + List primary = new ArrayList<>(); + for (Identify identify : dataModel.getIdentifiers()) { + primary.add(identify.getName()); + } + + TableView tableView = + renderOne(queryMetrics, queryDimensions, dataModel, scope, schema); + log.info("tableView {}", StringUtils.normalizeSpace(tableView.getTable().toString())); + String alias = Constants.JOIN_TABLE_PREFIX + dataModel.getName(); + tableView.setAlias(alias); + tableView.setPrimary(primary); + tableView.setDataModel(dataModel); + for (String field : tableView.getFields()) { + outerSelect.put(field, SemanticNode.parse(alias + "." + field, scope, engineType)); + } + if (left == null) { + left = SemanticNode.buildAs(tableView.getAlias(), getTable(tableView)); + } else { + left = buildJoin(left, leftTable, tableView, beforeModels, dataModel, schema, + scope); + } + leftTable = tableView; + beforeModels.put(dataModel.getName(), leftTable.getAlias()); + } + + for (Map.Entry entry : outerSelect.entrySet()) { + outerTable.getSelect().add(entry.getValue()); + } + outerTable.setTable(left); + + return outerTable; + } + + private SqlNode getTable(TableView tableView) { + return SemanticNode.getTable(tableView.getTable()); + } + + private SqlNode buildJoin(SqlNode leftNode, TableView leftTable, TableView rightTable, + Map before, ModelResp dataModel, S2CalciteSchema schema, + SqlValidatorScope scope) throws Exception { + EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType()); + SqlNode condition = + getCondition(leftTable, rightTable, dataModel, schema, scope, engineType); + SqlLiteral sqlLiteral = SemanticNode.getJoinSqlLiteral(""); + JoinRelation matchJoinRelation = getMatchJoinRelation(before, rightTable, schema); + SqlNode joinRelationCondition; + if (!CollectionUtils.isEmpty(matchJoinRelation.getJoinCondition())) { + sqlLiteral = SemanticNode.getJoinSqlLiteral(matchJoinRelation.getJoinType()); + joinRelationCondition = getCondition(matchJoinRelation, scope, engineType); + condition = joinRelationCondition; + } + + return new SqlJoin(SqlParserPos.ZERO, leftNode, + SqlLiteral.createBoolean(false, SqlParserPos.ZERO), sqlLiteral, + SemanticNode.buildAs(rightTable.getAlias(), getTable(rightTable)), + SqlLiteral.createSymbol(JoinConditionType.ON, SqlParserPos.ZERO), condition); + } + + private JoinRelation getMatchJoinRelation(Map before, TableView tableView, + S2CalciteSchema schema) { + JoinRelation matchJoinRelation = JoinRelation.builder().build(); + if (!CollectionUtils.isEmpty(schema.getJoinRelations())) { + for (JoinRelation joinRelation : schema.getJoinRelations()) { + if (joinRelation.getRight().equalsIgnoreCase(tableView.getDataModel().getName()) + && before.containsKey(joinRelation.getLeft())) { + matchJoinRelation.setJoinCondition(joinRelation.getJoinCondition().stream() + .map(r -> Triple.of( + before.get(joinRelation.getLeft()) + "." + r.getLeft(), + r.getMiddle(), tableView.getAlias() + "." + r.getRight())) + .collect(Collectors.toList())); + matchJoinRelation.setJoinType(joinRelation.getJoinType()); + // Added join condition judgment to solve the problem of join condition order + } else if (joinRelation.getLeft() + .equalsIgnoreCase(tableView.getDataModel().getName()) + && before.containsKey(joinRelation.getRight())) { + matchJoinRelation.setJoinCondition(joinRelation.getJoinCondition().stream() + .map(r -> Triple.of( + before.get(joinRelation.getRight()) + "." + r.getRight(), + r.getMiddle(), tableView.getAlias() + "." + r.getLeft())) + .collect(Collectors.toList())); + matchJoinRelation.setJoinType(joinRelation.getJoinType()); + } + } + } + return matchJoinRelation; + } + + private SqlNode getCondition(JoinRelation joinRelation, SqlValidatorScope scope, + EngineType engineType) throws Exception { + SqlNode condition = null; + for (Triple con : joinRelation.getJoinCondition()) { + List ons = new ArrayList<>(); + ons.add(SemanticNode.parse(con.getLeft(), scope, engineType)); + ons.add(SemanticNode.parse(con.getRight(), scope, engineType)); + if (Objects.isNull(condition)) { + condition = new SqlBasicCall(SemanticNode.getBinaryOperator(con.getMiddle()), ons, + SqlParserPos.ZERO, null); + continue; + } + SqlNode addCondition = new SqlBasicCall(SemanticNode.getBinaryOperator(con.getMiddle()), + ons, SqlParserPos.ZERO, null); + condition = new SqlBasicCall(SqlStdOperatorTable.AND, + new ArrayList<>(Arrays.asList(condition, addCondition)), SqlParserPos.ZERO, + null); + } + return condition; + } + + private SqlNode getCondition(TableView left, TableView right, ModelResp dataModel, + S2CalciteSchema schema, SqlValidatorScope scope, EngineType engineType) + throws Exception { + + Set selectLeft = SemanticNode.getSelect(left.getTable()); + Set selectRight = SemanticNode.getSelect(right.getTable()); + selectLeft.retainAll(selectRight); + SqlNode condition = null; + for (String on : selectLeft) { + if (!isDimension(on, dataModel, schema)) { + continue; + } + if (isForeign(on, left.getDataModel().getIdentifiers())) { + if (!isPrimary(on, right.getDataModel().getIdentifiers())) { + continue; + } + } + if (isForeign(on, right.getDataModel().getIdentifiers())) { + if (!isPrimary(on, left.getDataModel().getIdentifiers())) { + continue; + } + } + List ons = new ArrayList<>(); + ons.add(SemanticNode.parse(left.getAlias() + "." + on, scope, engineType)); + ons.add(SemanticNode.parse(right.getAlias() + "." + on, scope, engineType)); + if (condition == null) { + condition = + new SqlBasicCall(SqlStdOperatorTable.EQUALS, ons, SqlParserPos.ZERO, null); + continue; + } + SqlNode addCondition = + new SqlBasicCall(SqlStdOperatorTable.EQUALS, ons, SqlParserPos.ZERO, null); + condition = new SqlBasicCall(SqlStdOperatorTable.AND, + new ArrayList<>(Arrays.asList(condition, addCondition)), SqlParserPos.ZERO, + null); + } + return condition; + } + + public static TableView renderOne(Set queryMetrics, + Set queryDimensions, ModelResp dataModel, SqlValidatorScope scope, + S2CalciteSchema schema) { + TableView tableView = new TableView(); + EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType()); + Set queryFields = tableView.getFields(); + queryMetrics.stream().forEach(m -> queryFields.addAll(m.getFields())); + queryDimensions.stream().forEach(m -> queryFields.add(m.getBizName())); + + try { + for (String field : queryFields) { + tableView.getSelect().add(SemanticNode.parse(field, scope, engineType)); + } + tableView.setTable(DataModelNode.build(dataModel, scope)); + } catch (Exception e) { + log.error("Failed to create sqlNode for data model {}", dataModel); + } + + return tableView; + } + + private static boolean isDimension(String name, ModelResp dataModel, S2CalciteSchema schema) { + Optional dimension = dataModel.getModelDetail().getDimensions().stream() + .filter(d -> d.getName().equalsIgnoreCase(name)).findFirst(); + if (dimension.isPresent()) { + return true; + } + Optional identify = dataModel.getIdentifiers().stream() + .filter(i -> i.getName().equalsIgnoreCase(name)).findFirst(); + if (identify.isPresent()) { + return true; + } + if (schema.getDimensions().containsKey(dataModel.getName())) { + Optional dataSourceDim = schema.getDimensions().get(dataModel.getName()) + .stream().filter(d -> d.getName().equalsIgnoreCase(name)).findFirst(); + if (dataSourceDim.isPresent()) { + return true; + } + } + return false; + } + + private static boolean isForeign(String name, List identifies) { + Optional identify = + identifies.stream().filter(i -> i.getName().equalsIgnoreCase(name)).findFirst(); + if (identify.isPresent()) { + return IdentifyType.foreign.equals(identify.get().getType()); + } + return false; + } + + private static boolean isPrimary(String name, List identifies) { + Optional identify = + identifies.stream().filter(i -> i.getName().equalsIgnoreCase(name)).findFirst(); + if (identify.isPresent()) { + return IdentifyType.primary.equals(identify.get().getType()); + } + return false; } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/TableView.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/TableView.java index 81c8c7373..f55e8f799 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/TableView.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/TableView.java @@ -2,7 +2,7 @@ package com.tencent.supersonic.headless.core.translator.parser.calcite; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import com.tencent.supersonic.headless.core.pojo.DataModel; +import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import lombok.Data; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; @@ -13,7 +13,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Set; -/** basic query project */ @Data public class TableView { @@ -28,7 +27,7 @@ public class TableView { private String alias; private List primary; - private DataModel dataModel; + private ModelResp dataModel; public SqlNode build() { List selectNodeList = new ArrayList<>(); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/AggFunctionNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/AggFunctionNode.java deleted file mode 100644 index 75fc1cf65..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/AggFunctionNode.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.tencent.supersonic.headless.core.translator.parser.calcite.node; - -import com.tencent.supersonic.common.pojo.enums.EngineType; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.validate.SqlValidatorScope; - -import java.util.Objects; - -public class AggFunctionNode extends SemanticNode { - - public static SqlNode build(String agg, String name, SqlValidatorScope scope, - EngineType engineType) throws Exception { - if (Objects.isNull(agg) || agg.isEmpty()) { - return parse(name, scope, engineType); - } - if (AggFunction.COUNT_DISTINCT.name().equalsIgnoreCase(agg)) { - return parse(AggFunction.COUNT.name() + " ( " + AggFunction.DISTINCT.name() + " " + name - + " ) ", scope, engineType); - } - return parse(agg + " ( " + name + " ) ", scope, engineType); - } - - public static enum AggFunction { - AVG, COUNT_DISTINCT, MAX, MIN, SUM, COUNT, DISTINCT - } -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/DimensionNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/DimensionNode.java deleted file mode 100644 index 4665cd099..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/DimensionNode.java +++ /dev/null @@ -1,54 +0,0 @@ -package com.tencent.supersonic.headless.core.translator.parser.calcite.node; - -import com.tencent.supersonic.common.pojo.enums.DataTypeEnums; -import com.tencent.supersonic.common.pojo.enums.EngineType; -import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; -import com.tencent.supersonic.headless.core.translator.parser.s2sql.Constants; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.validate.SqlValidatorScope; - -import java.util.List; -import java.util.Objects; - -public class DimensionNode extends SemanticNode { - - public static SqlNode build(DimSchemaResp dimension, SqlValidatorScope scope, - EngineType engineType) throws Exception { - return parse(dimension.getExpr(), scope, engineType); - } - - public static List expand(DimSchemaResp dimension, SqlValidatorScope scope, - EngineType engineType) throws Exception { - SqlNode sqlNode = parse(dimension.getExpr(), scope, engineType); - return expand(sqlNode, scope); - } - - public static SqlNode buildName(DimSchemaResp dimension, SqlValidatorScope scope, - EngineType engineType) throws Exception { - return parse(dimension.getExpr(), scope, engineType); - } - - public static SqlNode buildNameAs(String alias, DimSchemaResp dimension, - SqlValidatorScope scope, EngineType engineType) throws Exception { - if ("".equals(alias)) { - return buildName(dimension, scope, engineType); - } - SqlNode sqlNode = parse(dimension.getExpr(), scope, engineType); - return buildAs(alias, sqlNode); - } - - public static SqlNode buildArray(DimSchemaResp dimension, SqlValidatorScope scope, - EngineType engineType) throws Exception { - if (Objects.nonNull(dimension.getDataType()) - && dimension.getDataType().equals(DataTypeEnums.ARRAY)) { - SqlNode sqlNode = parse(dimension.getExpr(), scope, engineType); - if (isIdentifier(sqlNode)) { - return buildAs(dimension.getName(), - parse(dimension.getExpr() + Constants.DIMENSION_ARRAY_SINGLE_SUFFIX, scope, - engineType)); - } - throw new Exception("array dimension expr should only identify"); - } - return build(dimension, scope, engineType); - } -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/ExtendNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/ExtendNode.java deleted file mode 100644 index f5ff9ca7d..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/ExtendNode.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.tencent.supersonic.headless.core.translator.parser.calcite.node; - -import org.apache.calcite.sql.SqlCall; -import org.apache.calcite.sql.SqlInternalOperator; -import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.SqlNodeList; -import org.apache.calcite.sql.SqlOperator; -import org.apache.calcite.sql.SqlWriter; -import org.apache.calcite.sql.SqlWriter.Frame; -import org.apache.calcite.sql.SqlWriter.FrameTypeEnum; - -public class ExtendNode extends SqlInternalOperator { - - public ExtendNode() { - super(SqlKind.EXTEND.lowerName, SqlKind.EXTEND); - } - - public void unparse(SqlWriter writer, SqlCall call, int leftPrec, int rightPrec) { - SqlOperator operator = call.getOperator(); - Frame frame = writer.startList(FrameTypeEnum.SIMPLE); - call.operand(0).unparse(writer, leftPrec, operator.getLeftPrec()); - writer.setNeedWhitespace(true); - writer.sep(operator.getName()); - SqlNodeList list = (SqlNodeList) call.operand(1); - Frame frameArgs = writer.startList("(", ")"); - for (int i = 0; i < list.size(); i++) { - list.get(i).unparse(writer, 0, 0); - if (i < list.size() - 1) { - writer.sep(","); - } - } - writer.endList(frameArgs); - writer.endList(frame); - } -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/FilterNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/FilterNode.java deleted file mode 100644 index 063966ae3..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/FilterNode.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.tencent.supersonic.headless.core.translator.parser.calcite.node; - -import org.apache.calcite.sql.SqlBasicCall; -import org.apache.calcite.sql.SqlIdentifier; -import org.apache.calcite.sql.SqlNode; - -import java.util.Set; - -public class FilterNode extends SemanticNode { - - public static void getFilterField(SqlNode sqlNode, Set fields) { - if (sqlNode instanceof SqlIdentifier) { - SqlIdentifier sqlIdentifier = (SqlIdentifier) sqlNode; - fields.add(sqlIdentifier.names.get(0).toLowerCase()); - return; - } - if (sqlNode instanceof SqlBasicCall) { - SqlBasicCall sqlBasicCall = (SqlBasicCall) sqlNode; - for (SqlNode operand : sqlBasicCall.getOperandList()) { - getFilterField(operand, fields); - } - } - } - -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/IdentifyNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/IdentifyNode.java deleted file mode 100644 index c4f5f7881..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/IdentifyNode.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.tencent.supersonic.headless.core.translator.parser.calcite.node; - -import com.tencent.supersonic.common.pojo.enums.EngineType; -import com.tencent.supersonic.headless.api.pojo.Identify; -import com.tencent.supersonic.headless.api.pojo.enums.IdentifyType; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.validate.SqlValidatorScope; - -import java.util.List; -import java.util.Optional; - -public class IdentifyNode extends SemanticNode { - - public static SqlNode build(Identify identify, SqlValidatorScope scope, EngineType engineType) - throws Exception { - return parse(identify.getName(), scope, engineType); - } - - public static boolean isForeign(String name, List identifies) { - Optional identify = - identifies.stream().filter(i -> i.getName().equalsIgnoreCase(name)).findFirst(); - if (identify.isPresent()) { - return IdentifyType.foreign.equals(identify.get().getType()); - } - return false; - } - - public static boolean isPrimary(String name, List identifies) { - Optional identify = - identifies.stream().filter(i -> i.getName().equalsIgnoreCase(name)).findFirst(); - if (identify.isPresent()) { - return IdentifyType.primary.equals(identify.get().getType()); - } - return false; - } -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/LateralViewExplodeNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/LateralViewExplodeNode.java deleted file mode 100644 index 5a2db5a72..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/LateralViewExplodeNode.java +++ /dev/null @@ -1,79 +0,0 @@ -package com.tencent.supersonic.headless.core.translator.parser.calcite.node; - -import org.apache.calcite.linq4j.Ord; -import org.apache.calcite.sql.SqlCall; -import org.apache.calcite.sql.SqlIdentifier; -import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.SqlNodeList; -import org.apache.calcite.sql.SqlOperator; -import org.apache.calcite.sql.SqlWriter; - -import java.util.Iterator; -import java.util.Map; -import java.util.Objects; - -/** extend node to handle lateral explode dataSet */ -public class LateralViewExplodeNode extends ExtendNode { - - public final String sqlNameView = "view"; - public final String sqlNameExplode = "explode"; - public final String sqlNameExplodeSplit = "explode_split"; - private Map delimiterMap; - - public LateralViewExplodeNode(Map delimiterMap) { - super(); - this.delimiterMap = delimiterMap; - } - - public void unparse(SqlWriter writer, SqlCall call, int leftPrec, int rightPrec) { - SqlOperator operator = call.getOperator(); - writer.setNeedWhitespace(true); - assert call.operandCount() == 2; - writer.sep(SqlKind.SELECT.lowerName); - writer.sep(SqlIdentifier.STAR.toString()); - writer.sep("from"); - SqlWriter.Frame frame = writer.startList(SqlWriter.FrameTypeEnum.SIMPLE); - call.operand(0).unparse(writer, leftPrec, operator.getLeftPrec()); - writer.setNeedWhitespace(true); - writer.sep(SqlKind.LATERAL.lowerName); - writer.sep(sqlNameView); - SqlNodeList list = (SqlNodeList) call.operand(1); - Ord node; - Iterator var = Ord.zip(list).iterator(); - while (var.hasNext()) { - node = (Ord) var.next(); - if (node.i > 0 && node.i % 2 > 0) { - writer.sep(SqlKind.AS.lowerName); - ((SqlNode) node.e).unparse(writer, 0, 0); - continue; - } - if (node.i > 0 && node.i % 2 == 0) { - writer.sep(SqlKind.LATERAL.lowerName); - writer.sep(sqlNameView); - } - explode(writer, (SqlNode) node.e); - } - writer.endList(frame); - } - - public void explode(SqlWriter writer, SqlNode sqlNode) { - String delimiter = - Objects.nonNull(delimiterMap) && delimiterMap.containsKey(sqlNode.toString()) - ? delimiterMap.get(sqlNode.toString()) - : ""; - if (delimiter.isEmpty()) { - writer.sep(sqlNameExplode); - } else { - writer.sep(sqlNameExplodeSplit); - } - SqlWriter.Frame frame = writer.startList("(", ")"); - sqlNode.unparse(writer, 0, 0); - if (!delimiter.isEmpty()) { - writer.sep(","); - writer.sep(String.format("'%s'", delimiter)); - } - writer.endList(frame); - writer.sep("tmp_sgl_" + sqlNode.toString()); - } -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/MeasureNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/MeasureNode.java deleted file mode 100644 index 69124d724..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/MeasureNode.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.tencent.supersonic.headless.core.translator.parser.calcite.node; - -import com.tencent.supersonic.common.pojo.enums.EngineType; -import com.tencent.supersonic.headless.api.pojo.Measure; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.validate.SqlValidatorScope; - -public class MeasureNode extends SemanticNode { - - public static SqlNode buildNonAgg(String alias, Measure measure, SqlValidatorScope scope, - EngineType engineType) throws Exception { - return getExpr(measure, alias, scope, engineType); - } - - private static SqlNode getExpr(Measure measure, String alias, SqlValidatorScope scope, - EngineType enginType) throws Exception { - if (measure.getExpr() == null) { - return parse((alias.isEmpty() ? "" : alias + ".") + measure.getName(), scope, - enginType); - } - return parse(measure.getExpr(), scope, enginType); - } -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/MetricNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/MetricNode.java deleted file mode 100644 index 32eb5723c..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/node/MetricNode.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.tencent.supersonic.headless.core.translator.parser.calcite.node; - -import com.tencent.supersonic.common.pojo.enums.EngineType; -import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType; -import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; -import com.tencent.supersonic.headless.core.translator.parser.calcite.S2CalciteSchema; -import lombok.Data; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.validate.SqlValidatorScope; - -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - -@Data -public class MetricNode extends SemanticNode { - - private MetricSchemaResp metric; - private Map aggNode = new HashMap<>(); - private Map nonAggNode = new HashMap<>(); - private Map measureFilter = new HashMap<>(); - private Map aggFunction = new HashMap<>(); - - public static SqlNode build(MetricSchemaResp metric, SqlValidatorScope scope, - EngineType engineType) throws Exception { - return parse(metric.getExpr(), scope, engineType); - } - - public static Boolean isMetricField(String name, S2CalciteSchema schema) { - Optional metric = schema.getMetrics().stream() - .filter(m -> m.getName().equalsIgnoreCase(name)).findFirst(); - return metric.isPresent() - && metric.get().getMetricDefineType().equals(MetricDefineType.FIELD); - } - -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/render/JoinRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/render/JoinRender.java deleted file mode 100644 index b4e7e3748..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/render/JoinRender.java +++ /dev/null @@ -1,236 +0,0 @@ -package com.tencent.supersonic.headless.core.translator.parser.calcite.render; - -import com.tencent.supersonic.common.pojo.enums.EngineType; -import com.tencent.supersonic.headless.api.pojo.Identify; -import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; -import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; -import com.tencent.supersonic.headless.core.pojo.DataModel; -import com.tencent.supersonic.headless.core.pojo.JoinRelation; -import com.tencent.supersonic.headless.core.pojo.OntologyQuery; -import com.tencent.supersonic.headless.core.translator.parser.calcite.S2CalciteSchema; -import com.tencent.supersonic.headless.core.translator.parser.calcite.TableView; -import com.tencent.supersonic.headless.core.translator.parser.calcite.node.DataModelNode; -import com.tencent.supersonic.headless.core.translator.parser.calcite.node.IdentifyNode; -import com.tencent.supersonic.headless.core.translator.parser.calcite.node.SemanticNode; -import com.tencent.supersonic.headless.core.translator.parser.s2sql.Constants; -import lombok.extern.slf4j.Slf4j; -import org.apache.calcite.sql.*; -import org.apache.calcite.sql.fun.SqlStdOperatorTable; -import org.apache.calcite.sql.parser.SqlParserPos; -import org.apache.calcite.sql.validate.SqlValidatorScope; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Triple; -import org.springframework.util.CollectionUtils; - -import java.util.*; -import java.util.stream.Collectors; - -/** process the join conditions when the source number is greater than 1 */ -@Slf4j -public class JoinRender extends Renderer { - - @Override - public void render(OntologyQuery ontologyQuery, List dataModels, - SqlValidatorScope scope, S2CalciteSchema schema) throws Exception { - SqlNode left = null; - TableView leftTable = null; - Map outerSelect = new HashMap<>(); - Map beforeModels = new HashMap<>(); - EngineType engineType = schema.getOntology().getDatabase().getType(); - - for (int i = 0; i < dataModels.size(); i++) { - final DataModel dataModel = dataModels.get(i); - final Set queryDimensions = - ontologyQuery.getDimensionsByModel(dataModel.getId()); - final Set queryMetrics = - ontologyQuery.getMetricsByModel(dataModel.getId()); - - List primary = new ArrayList<>(); - for (Identify identify : dataModel.getIdentifiers()) { - primary.add(identify.getName()); - } - - TableView tableView = - renderOne(queryMetrics, queryDimensions, dataModel, scope, schema); - log.info("tableView {}", StringUtils.normalizeSpace(tableView.getTable().toString())); - String alias = Constants.JOIN_TABLE_PREFIX + dataModel.getName(); - tableView.setAlias(alias); - tableView.setPrimary(primary); - tableView.setDataModel(dataModel); - for (String field : tableView.getFields()) { - outerSelect.put(field, SemanticNode.parse(alias + "." + field, scope, engineType)); - } - if (left == null) { - left = SemanticNode.buildAs(tableView.getAlias(), getTable(tableView)); - } else { - left = buildJoin(left, leftTable, tableView, beforeModels, dataModel, schema, - scope); - } - leftTable = tableView; - beforeModels.put(dataModel.getName(), leftTable.getAlias()); - } - - for (Map.Entry entry : outerSelect.entrySet()) { - tableView.getSelect().add(entry.getValue()); - } - tableView.setTable(left); - } - - private SqlNode getTable(TableView tableView) { - return SemanticNode.getTable(tableView.getTable()); - } - - private SqlNode buildJoin(SqlNode leftNode, TableView leftTable, TableView rightTable, - Map before, DataModel dataModel, S2CalciteSchema schema, - SqlValidatorScope scope) throws Exception { - EngineType engineType = schema.getOntology().getDatabase().getType(); - SqlNode condition = - getCondition(leftTable, rightTable, dataModel, schema, scope, engineType); - SqlLiteral sqlLiteral = SemanticNode.getJoinSqlLiteral(""); - JoinRelation matchJoinRelation = getMatchJoinRelation(before, rightTable, schema); - SqlNode joinRelationCondition; - if (!CollectionUtils.isEmpty(matchJoinRelation.getJoinCondition())) { - sqlLiteral = SemanticNode.getJoinSqlLiteral(matchJoinRelation.getJoinType()); - joinRelationCondition = getCondition(matchJoinRelation, scope, engineType); - condition = joinRelationCondition; - } - - return new SqlJoin(SqlParserPos.ZERO, leftNode, - SqlLiteral.createBoolean(false, SqlParserPos.ZERO), sqlLiteral, - SemanticNode.buildAs(rightTable.getAlias(), getTable(rightTable)), - SqlLiteral.createSymbol(JoinConditionType.ON, SqlParserPos.ZERO), condition); - } - - private JoinRelation getMatchJoinRelation(Map before, TableView tableView, - S2CalciteSchema schema) { - JoinRelation matchJoinRelation = JoinRelation.builder().build(); - if (!CollectionUtils.isEmpty(schema.getJoinRelations())) { - for (JoinRelation joinRelation : schema.getJoinRelations()) { - if (joinRelation.getRight().equalsIgnoreCase(tableView.getDataModel().getName()) - && before.containsKey(joinRelation.getLeft())) { - matchJoinRelation.setJoinCondition(joinRelation.getJoinCondition().stream() - .map(r -> Triple.of( - before.get(joinRelation.getLeft()) + "." + r.getLeft(), - r.getMiddle(), tableView.getAlias() + "." + r.getRight())) - .collect(Collectors.toList())); - matchJoinRelation.setJoinType(joinRelation.getJoinType()); - // Added join condition judgment to solve the problem of join condition order - } else if (joinRelation.getLeft() - .equalsIgnoreCase(tableView.getDataModel().getName()) - && before.containsKey(joinRelation.getRight())) { - matchJoinRelation.setJoinCondition(joinRelation.getJoinCondition().stream() - .map(r -> Triple.of( - before.get(joinRelation.getRight()) + "." + r.getRight(), - r.getMiddle(), tableView.getAlias() + "." + r.getLeft())) - .collect(Collectors.toList())); - matchJoinRelation.setJoinType(joinRelation.getJoinType()); - } - } - } - return matchJoinRelation; - } - - private SqlNode getCondition(JoinRelation joinRelation, SqlValidatorScope scope, - EngineType engineType) throws Exception { - SqlNode condition = null; - for (Triple con : joinRelation.getJoinCondition()) { - List ons = new ArrayList<>(); - ons.add(SemanticNode.parse(con.getLeft(), scope, engineType)); - ons.add(SemanticNode.parse(con.getRight(), scope, engineType)); - if (Objects.isNull(condition)) { - condition = new SqlBasicCall(SemanticNode.getBinaryOperator(con.getMiddle()), ons, - SqlParserPos.ZERO, null); - continue; - } - SqlNode addCondition = new SqlBasicCall(SemanticNode.getBinaryOperator(con.getMiddle()), - ons, SqlParserPos.ZERO, null); - condition = new SqlBasicCall(SqlStdOperatorTable.AND, - new ArrayList<>(Arrays.asList(condition, addCondition)), SqlParserPos.ZERO, - null); - } - return condition; - } - - private SqlNode getCondition(TableView left, TableView right, DataModel dataModel, - S2CalciteSchema schema, SqlValidatorScope scope, EngineType engineType) - throws Exception { - - Set selectLeft = SemanticNode.getSelect(left.getTable()); - Set selectRight = SemanticNode.getSelect(right.getTable()); - selectLeft.retainAll(selectRight); - SqlNode condition = null; - for (String on : selectLeft) { - if (!isDimension(on, dataModel, schema)) { - continue; - } - if (IdentifyNode.isForeign(on, left.getDataModel().getIdentifiers())) { - if (!IdentifyNode.isPrimary(on, right.getDataModel().getIdentifiers())) { - continue; - } - } - if (IdentifyNode.isForeign(on, right.getDataModel().getIdentifiers())) { - if (!IdentifyNode.isPrimary(on, left.getDataModel().getIdentifiers())) { - continue; - } - } - List ons = new ArrayList<>(); - ons.add(SemanticNode.parse(left.getAlias() + "." + on, scope, engineType)); - ons.add(SemanticNode.parse(right.getAlias() + "." + on, scope, engineType)); - if (condition == null) { - condition = - new SqlBasicCall(SqlStdOperatorTable.EQUALS, ons, SqlParserPos.ZERO, null); - continue; - } - SqlNode addCondition = - new SqlBasicCall(SqlStdOperatorTable.EQUALS, ons, SqlParserPos.ZERO, null); - condition = new SqlBasicCall(SqlStdOperatorTable.AND, - new ArrayList<>(Arrays.asList(condition, addCondition)), SqlParserPos.ZERO, - null); - } - return condition; - } - - - public static TableView renderOne(Set queryMetrics, - Set queryDimensions, DataModel dataModel, SqlValidatorScope scope, - S2CalciteSchema schema) { - TableView tableView = new TableView(); - EngineType engineType = schema.getOntology().getDatabase().getType(); - Set queryFields = tableView.getFields(); - queryMetrics.stream().forEach(m -> queryFields.addAll(m.getFields())); - queryDimensions.stream().forEach(m -> queryFields.add(m.getBizName())); - - try { - for (String field : queryFields) { - tableView.getSelect().add(SemanticNode.parse(field, scope, engineType)); - } - tableView.setTable(DataModelNode.build(dataModel, scope)); - } catch (Exception e) { - log.error("Failed to create sqlNode for data model {}", dataModel); - } - - return tableView; - } - - public static boolean isDimension(String name, DataModel dataModel, S2CalciteSchema schema) { - Optional dimension = dataModel.getDimensions().stream() - .filter(d -> d.getName().equalsIgnoreCase(name)).findFirst(); - if (dimension.isPresent()) { - return true; - } - Optional identify = dataModel.getIdentifiers().stream() - .filter(i -> i.getName().equalsIgnoreCase(name)).findFirst(); - if (identify.isPresent()) { - return true; - } - if (schema.getDimensions().containsKey(dataModel.getName())) { - Optional dataSourceDim = schema.getDimensions().get(dataModel.getName()) - .stream().filter(d -> d.getName().equalsIgnoreCase(name)).findFirst(); - if (dataSourceDim.isPresent()) { - return true; - } - } - return false; - } - -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/render/Renderer.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/render/Renderer.java deleted file mode 100644 index 49390fc60..000000000 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/render/Renderer.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.tencent.supersonic.headless.core.translator.parser.calcite.render; - -import com.tencent.supersonic.headless.core.pojo.DataModel; -import com.tencent.supersonic.headless.core.pojo.OntologyQuery; -import com.tencent.supersonic.headless.core.translator.parser.calcite.S2CalciteSchema; -import com.tencent.supersonic.headless.core.translator.parser.calcite.TableView; -import com.tencent.supersonic.headless.core.translator.parser.calcite.node.SemanticNode; -import lombok.Data; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.validate.SqlValidatorScope; - -import java.util.List; - -/** process TableView */ -@Data -public abstract class Renderer { - - protected TableView tableView = new TableView(); - - public void setTable(SqlNode table) { - tableView.setTable(table); - } - - public SqlNode build() { - return tableView.build(); - } - - public SqlNode builderAs(String alias) throws Exception { - return SemanticNode.buildAs(alias, tableView.build()); - } - - public abstract void render(OntologyQuery ontologyQuery, List dataModels, - SqlValidatorScope scope, S2CalciteSchema schema) throws Exception; -} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/JdbcDataSourceUtils.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/JdbcDataSourceUtils.java index 1d4729c4a..8dee1e82a 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/JdbcDataSourceUtils.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/JdbcDataSourceUtils.java @@ -5,7 +5,7 @@ import javax.sql.DataSource; import com.alibaba.druid.util.StringUtils; import com.tencent.supersonic.common.util.MD5Util; import com.tencent.supersonic.headless.api.pojo.enums.DataType; -import com.tencent.supersonic.headless.core.pojo.Database; +import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.core.pojo.JdbcDataSource; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -18,14 +18,7 @@ import java.util.HashSet; import java.util.Set; import java.util.regex.Matcher; -import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL; -import static com.tencent.supersonic.common.pojo.Constants.COLON; -import static com.tencent.supersonic.common.pojo.Constants.DOUBLE_SLASH; -import static com.tencent.supersonic.common.pojo.Constants.EMPTY; -import static com.tencent.supersonic.common.pojo.Constants.JDBC_PREFIX_FORMATTER; -import static com.tencent.supersonic.common.pojo.Constants.NEW_LINE_CHAR; -import static com.tencent.supersonic.common.pojo.Constants.PATTERN_JDBC_TYPE; -import static com.tencent.supersonic.common.pojo.Constants.SPACE; +import static com.tencent.supersonic.common.pojo.Constants.*; /** tools functions about jdbc */ @Slf4j @@ -39,7 +32,7 @@ public class JdbcDataSourceUtils { this.jdbcDataSource = jdbcDataSource; } - public static boolean testDatabase(Database database) { + public static boolean testDatabase(DatabaseResp database) { try { Class.forName(getDriverClassName(database.getUrl())); @@ -146,11 +139,11 @@ public class JdbcDataSourceUtils { return MD5Util.getMD5(sb.toString(), true, 64); } - public DataSource getDataSource(Database database) throws RuntimeException { + public DataSource getDataSource(DatabaseResp database) throws RuntimeException { return jdbcDataSource.getDataSource(database); } - public Connection getConnection(Database database) throws RuntimeException { + public Connection getConnection(DatabaseResp database) throws RuntimeException { Connection conn = getConnectionWithRetry(database); if (conn == null) { try { @@ -166,7 +159,7 @@ public class JdbcDataSourceUtils { return conn; } - private Connection getConnectionWithRetry(Database database) { + private Connection getConnectionWithRetry(DatabaseResp database) { int rc = 1; for (;;) { @@ -193,7 +186,7 @@ public class JdbcDataSourceUtils { } } - public void releaseDataSource(Database database) { + public void releaseDataSource(DatabaseResp database) { jdbcDataSource.removeDatasource(database); } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlUtils.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlUtils.java index 339586bf0..99d62841c 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlUtils.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlUtils.java @@ -3,11 +3,10 @@ package com.tencent.supersonic.headless.core.utils; import javax.sql.DataSource; import com.tencent.supersonic.common.pojo.QueryColumn; -import com.tencent.supersonic.common.pojo.enums.EngineType; import com.tencent.supersonic.common.util.DateUtils; import com.tencent.supersonic.headless.api.pojo.enums.DataType; +import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; -import com.tencent.supersonic.headless.core.pojo.Database; import com.tencent.supersonic.headless.core.pojo.JdbcDataSource; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -24,11 +23,7 @@ import java.sql.SQLException; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Date; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL; @@ -38,7 +33,7 @@ import static com.tencent.supersonic.common.pojo.Constants.AT_SYMBOL; public class SqlUtils { @Getter - private Database database; + private DatabaseResp database; @Autowired private JdbcDataSource jdbcDataSource; @@ -57,15 +52,15 @@ public class SqlUtils { public SqlUtils() {} - public SqlUtils(Database database) { + public SqlUtils(DatabaseResp database) { this.database = database; this.dataTypeEnum = DataType.urlOf(database.getUrl()); } - public SqlUtils init(Database database) { + public SqlUtils init(DatabaseResp database) { return SqlUtilsBuilder.getBuilder() .withName(database.getId() + AT_SYMBOL + database.getName()) - .withType(database.getType().getName()).withJdbcUrl(database.getUrl()) + .withType(database.getType()).withJdbcUrl(database.getUrl()) .withUsername(database.getUsername()).withPassword(database.getPassword()) .withJdbcDataSource(this.jdbcDataSource).withResultLimit(this.resultLimit) .withIsQueryLogEnable(this.isQueryLogEnable).build(); @@ -225,9 +220,9 @@ public class SqlUtils { } public SqlUtils build() { - Database database = Database.builder().name(this.name) - .type(EngineType.fromString(this.type.toUpperCase())).url(this.jdbcUrl) - .username(this.username).password(this.password).build(); + DatabaseResp database = DatabaseResp.builder().name(this.name) + .type(this.type.toUpperCase()).url(this.jdbcUrl).username(this.username) + .password(this.password).build(); SqlUtils sqlUtils = new SqlUtils(database); sqlUtils.jdbcDataSource = this.jdbcDataSource; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java index 4f357a76d..b371da13e 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java @@ -3,23 +3,16 @@ package com.tencent.supersonic.headless.server.manager; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.tencent.supersonic.common.pojo.ModelRela; -import com.tencent.supersonic.common.pojo.enums.DataTypeEnums; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; import com.tencent.supersonic.headless.api.pojo.*; +import com.tencent.supersonic.headless.api.pojo.enums.DimensionType; import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType; -import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; -import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; -import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; -import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; -import com.tencent.supersonic.headless.core.pojo.DataModel; +import com.tencent.supersonic.headless.api.pojo.response.*; import com.tencent.supersonic.headless.core.pojo.JoinRelation; import com.tencent.supersonic.headless.core.pojo.Ontology; import com.tencent.supersonic.headless.core.translator.parser.calcite.S2CalciteSchema; -import com.tencent.supersonic.headless.core.translator.parser.s2sql.*; -import com.tencent.supersonic.headless.core.translator.parser.s2sql.Materialization.TimePartType; import com.tencent.supersonic.headless.server.pojo.yaml.*; import com.tencent.supersonic.headless.server.service.SchemaService; -import com.tencent.supersonic.headless.server.utils.DatabaseConverter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Triple; import org.springframework.stereotype.Service; @@ -41,7 +34,15 @@ public class SemanticSchemaManager { public Ontology buildOntology(SemanticSchemaResp semanticSchemaResp) { Ontology ontology = new Ontology(); - ontology.setMetrics(semanticSchemaResp.getMetrics()); + Map> model2Metrics = Maps.newHashMap(); + semanticSchemaResp.getMetrics().forEach(dim -> { + if (!model2Metrics.containsKey(dim.getModelBizName())) { + model2Metrics.put(dim.getModelBizName(), Lists.newArrayList()); + } + model2Metrics.get(dim.getModelBizName()).add(dim); + }); + ontology.setMetricMap(model2Metrics); + Map> model2Dimensions = Maps.newHashMap(); semanticSchemaResp.getDimensions().forEach(dim -> { if (!model2Dimensions.containsKey(dim.getModelBizName())) { @@ -58,16 +59,16 @@ public class SemanticSchemaManager { schemaService.getSchemaYamlTpl(semanticSchemaResp, dimensionYamlTpls, dataModelYamlTpls, metricYamlTpls, modelIdName); DatabaseResp databaseResp = semanticSchemaResp.getDatabaseResp(); - ontology.setDatabase(DatabaseConverter.convert(databaseResp)); + ontology.setDatabase(databaseResp); if (!CollectionUtils.isEmpty(semanticSchemaResp.getModelRelas())) { ontology.setJoinRelations( getJoinRelation(semanticSchemaResp.getModelRelas(), modelIdName)); } if (!dataModelYamlTpls.isEmpty()) { - Map dataModelMap = + Map dataModelMap = dataModelYamlTpls.stream().map(SemanticSchemaManager::getDataModel).collect( - Collectors.toMap(DataModel::getName, item -> item, (k1, k2) -> k1)); - ontology.setDataModelMap(dataModelMap); + Collectors.toMap(ModelResp::getName, item -> item, (k1, k2) -> k1)); + ontology.setModelMap(dataModelMap); } return ontology; @@ -77,30 +78,30 @@ public class SemanticSchemaManager { return getMetricsByMetricYamlTpl(t); } - public static List getDimensions(final List t) { + public static List getDimensions(final List t) { return getDimension(t); } - public static DataModel getDataModel(final DataModelYamlTpl d) { - DataModel dataModel = DataModel.builder().id(d.getId()).modelId(d.getSourceId()) - .type(d.getType()).sqlQuery(d.getSqlQuery()).name(d.getName()) - .tableQuery(d.getTableQuery()).identifiers(getIdentify(d.getIdentifiers())) - .measures(getMeasureParams(d.getMeasures())) - .dimensions(getDimensions(d.getDimensions())).build(); - dataModel.setAggTime(getDataModelAggTime(dataModel.getDimensions())); - if (Objects.nonNull(d.getModelSourceTypeEnum())) { - dataModel.setTimePartType(TimePartType.of(d.getModelSourceTypeEnum().name())); - } - return dataModel; - } + public static ModelResp getDataModel(final DataModelYamlTpl d) { + // ModelResp dataModel = ModelResp.builder()(d.getId()).modelId(d.getSourceId()) + // .type(d.getType()).sqlQuery(d.getSqlQuery()).name(d.getName()) + // .tableQuery(d.getTableQuery()).identifiers(getIdentify(d.getIdentifiers())) + // .measures(getMeasureParams(d.getMeasures())) + // .dimensions(getDimensions(d.getDimensions())).build(); + ModelResp dataModel = new ModelResp(); + dataModel.setId(d.getId()); + dataModel.setName(d.getName()); + ModelDetail modelDetail = new ModelDetail(); + dataModel.setModelDetail(modelDetail); - private static String getDataModelAggTime(List dimensions) { - Optional timeDimension = - dimensions.stream().filter(DimSchemaResp::isTimeDimension).findFirst(); - if (timeDimension.isPresent() && Objects.nonNull(timeDimension.get().getTypeParams())) { - return timeDimension.get().getTypeParams().getTimeGranularity(); - } - return Constants.DIMENSION_TYPE_TIME_GRANULARITY_NONE; + modelDetail.setDbType(d.getType()); + modelDetail.setSqlQuery(d.getSqlQuery()); + modelDetail.setTableQuery(d.getTableQuery()); + modelDetail.getIdentifiers().addAll(getIdentify(d.getIdentifiers())); + modelDetail.getMeasures().addAll(getMeasureParams(d.getMeasures())); + modelDetail.getDimensions().addAll(getDimensions(d.getDimensions())); + + return dataModel; } private static List getMetricsByMetricYamlTpl( @@ -170,24 +171,16 @@ public class SemanticSchemaManager { return measures; } - private static List getDimension(List dimensionYamlTpls) { - List dimensions = new ArrayList<>(); + private static List getDimension(List dimensionYamlTpls) { + List dimensions = new ArrayList<>(); for (DimensionYamlTpl dimensionYamlTpl : dimensionYamlTpls) { - DimSchemaResp dimension = new DimSchemaResp(); - // dimension.setType(dimensionYamlTpl.getType()); + Dimension dimension = new Dimension(); + if (Objects.nonNull(dimensionYamlTpl.getType())) { + dimension.setType(DimensionType.valueOf(dimensionYamlTpl.getType())); + } dimension.setExpr(dimensionYamlTpl.getExpr()); dimension.setName(dimensionYamlTpl.getName()); dimension.setBizName(dimensionYamlTpl.getBizName()); - dimension.setDefaultValues(dimensionYamlTpl.getDefaultValues()); - if (Objects.nonNull(dimensionYamlTpl.getDataType())) { - dimension.setDataType(dimensionYamlTpl.getDataType()); - } - if (Objects.isNull(dimension.getDataType())) { - dimension.setDataType(DataTypeEnums.UNKNOWN); - } - if (Objects.nonNull(dimensionYamlTpl.getExt())) { - dimension.setExt(dimensionYamlTpl.getExt()); - } dimension.setTypeParams(dimensionYamlTpl.getTypeParams()); dimensions.add(dimension); } @@ -247,11 +240,11 @@ public class SemanticSchemaManager { } } - public static void update(S2CalciteSchema schema, DataModel datasourceYamlTpl) + public static void update(S2CalciteSchema schema, ModelResp datasourceYamlTpl) throws Exception { if (schema != null) { String dataSourceName = datasourceYamlTpl.getName(); - Optional> datasourceYamlTplMap = + Optional> datasourceYamlTplMap = schema.getDataModels().entrySet().stream() .filter(t -> t.getKey().equalsIgnoreCase(dataSourceName)).findFirst(); if (datasourceYamlTplMap.isPresent()) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java index a686d8345..3ac8712d9 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/DatabaseServiceImpl.java @@ -16,17 +16,12 @@ import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor; import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory; -import com.tencent.supersonic.headless.core.pojo.Database; import com.tencent.supersonic.headless.core.utils.JdbcDataSourceUtils; import com.tencent.supersonic.headless.core.utils.SqlUtils; import com.tencent.supersonic.headless.core.utils.SqlVariableParseUtils; import com.tencent.supersonic.headless.server.persistence.dataobject.DatabaseDO; import com.tencent.supersonic.headless.server.persistence.mapper.DatabaseDOMapper; -import com.tencent.supersonic.headless.server.pojo.DatabaseParameter; -import com.tencent.supersonic.headless.server.pojo.DbParameterFactory; -import com.tencent.supersonic.headless.server.pojo.DbParametersBuilder; -import com.tencent.supersonic.headless.server.pojo.DefaultParametersBuilder; -import com.tencent.supersonic.headless.server.pojo.ModelFilter; +import com.tencent.supersonic.headless.server.pojo.*; import com.tencent.supersonic.headless.server.service.DatabaseService; import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.utils.DatabaseConverter; @@ -58,7 +53,7 @@ public class DatabaseServiceImpl extends ServiceImpl