From e11aeafbc013a2f04d79ae4cb86923b786c8f9ef Mon Sep 17 00:00:00 2001 From: jipeli <54889677+jipeli@users.noreply.github.com> Date: Wed, 8 May 2024 11:52:31 +0800 Subject: [PATCH] (improvement)(Headless) explode split dimension (#956) --- .../util/jsqlparser/SqlSelectHelper.java | 20 +++++++ .../core/parser/calcite/s2sql/Constants.java | 1 + .../core/parser/calcite/s2sql/Dimension.java | 2 + .../calcite/sql/node/DataSourceNode.java | 30 +++++------ .../node/extend/LateralViewExplodeNode.java | 22 ++++++-- .../calcite/sql/render/SourceRender.java | 52 +++++++++++-------- .../server/manager/SemanticSchemaManager.java | 5 +- .../server/pojo/yaml/DimensionYamlTpl.java | 3 ++ 8 files changed, 93 insertions(+), 42 deletions(-) diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlSelectHelper.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlSelectHelper.java index 2de97c526..66099c79c 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlSelectHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlSelectHelper.java @@ -21,8 +21,12 @@ import net.sf.jsqlparser.expression.WhenClause; import net.sf.jsqlparser.expression.operators.conditional.AndExpression; import net.sf.jsqlparser.expression.operators.conditional.OrExpression; import net.sf.jsqlparser.expression.operators.conditional.XorExpression; +import net.sf.jsqlparser.expression.operators.relational.Between; import net.sf.jsqlparser.expression.operators.relational.ComparisonOperator; import net.sf.jsqlparser.expression.operators.relational.ExpressionList; +import net.sf.jsqlparser.expression.operators.relational.InExpression; +import net.sf.jsqlparser.expression.operators.relational.IsBooleanExpression; +import net.sf.jsqlparser.expression.operators.relational.IsNullExpression; import net.sf.jsqlparser.parser.CCJSqlParserUtil; import net.sf.jsqlparser.schema.Column; import net.sf.jsqlparser.schema.Table; @@ -564,6 +568,22 @@ public class SqlSelectHelper { getColumnFromExpr(expr.getLeftExpression(), columns); getColumnFromExpr(expr.getRightExpression(), columns); } + if (expression instanceof InExpression) { + InExpression inExpression = (InExpression) expression; + getColumnFromExpr(inExpression.getLeftExpression(), columns); + } + if (expression instanceof Between) { + Between between = (Between) expression; + getColumnFromExpr(between.getLeftExpression(), columns); + } + if (expression instanceof IsBooleanExpression) { + IsBooleanExpression isBooleanExpression = (IsBooleanExpression) expression; + getColumnFromExpr(isBooleanExpression.getLeftExpression(), columns); + } + if (expression instanceof IsNullExpression) { + IsNullExpression isNullExpression = (IsNullExpression) expression; + getColumnFromExpr(isNullExpression.getLeftExpression(), columns); + } if (expression instanceof Parenthesis) { Parenthesis expr = (Parenthesis) expression; getColumnFromExpr(expr.getExpression(), columns); diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/Constants.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/Constants.java index ff151c0e9..61988330e 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/Constants.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/Constants.java @@ -17,5 +17,6 @@ public class Constants { public static final String SQL_PARSER_TABLE = "parsed_tb"; public static final String SQL_PARSER_DB = "parsed_db"; public static final String SQL_PARSER_FIELD = "parsed_field"; + public static final String DIMENSION_DELIMITER = "dimension_delimiter"; } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/Dimension.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/Dimension.java index a9c9a1d95..d3c6ec8ef 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/Dimension.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/Dimension.java @@ -3,6 +3,7 @@ package com.tencent.supersonic.headless.core.parser.calcite.s2sql; import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticItem; import java.util.List; +import java.util.Map; import lombok.Builder; import lombok.Data; @@ -19,6 +20,7 @@ public class Dimension implements SemanticItem { private DataType dataType = DataType.UNKNOWN; private String bizName; private List defaultValues; + private Map ext; @Override public String getName() { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DataSourceNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DataSourceNode.java index b3315f3a8..8977fb1cc 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DataSourceNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DataSourceNode.java @@ -13,17 +13,6 @@ import com.tencent.supersonic.headless.core.parser.calcite.schema.SchemaBuilder; import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.parser.calcite.sql.node.extend.LateralViewExplodeNode; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; -import lombok.extern.slf4j.Slf4j; -import org.apache.calcite.sql.SqlBasicCall; -import org.apache.calcite.sql.SqlDataTypeSpec; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.SqlNodeList; -import org.apache.calcite.sql.SqlUserDefinedTypeNameSpec; -import org.apache.calcite.sql.parser.SqlParser; -import org.apache.calcite.sql.parser.SqlParserPos; -import org.apache.calcite.sql.validate.SqlValidatorScope; -import org.springframework.util.CollectionUtils; - import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; @@ -35,6 +24,16 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.calcite.sql.SqlBasicCall; +import org.apache.calcite.sql.SqlDataTypeSpec; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlUserDefinedTypeNameSpec; +import org.apache.calcite.sql.parser.SqlParser; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.validate.SqlValidatorScope; +import org.springframework.util.CollectionUtils; @Slf4j public class DataSourceNode extends SemanticNode { @@ -120,22 +119,23 @@ public class DataSourceNode extends SemanticNode { dimensions, metrics); } - public static SqlNode buildExtend(DataSource datasource, Set exprList, + public static SqlNode buildExtend(DataSource datasource, Map exprList, SqlValidatorScope scope) throws Exception { if (CollectionUtils.isEmpty(exprList)) { return build(datasource, scope); } EngineType engineType = EngineType.fromString(datasource.getType()); - SqlNode dataSet = new SqlBasicCall(new LateralViewExplodeNode(), Arrays.asList(build(datasource, scope), + SqlNode dataSet = new SqlBasicCall(new LateralViewExplodeNode(exprList), Arrays.asList(build(datasource, scope), new SqlNodeList(getExtendField(exprList, scope, engineType), SqlParserPos.ZERO)), SqlParserPos.ZERO); return buildAs(datasource.getName() + Constants.DIMENSION_ARRAY_SINGLE_SUFFIX, dataSet); } - public static List getExtendField(Set exprList, SqlValidatorScope scope, EngineType engineType) + public static List getExtendField(Map exprList, SqlValidatorScope scope, + EngineType engineType) throws Exception { List sqlNodeList = new ArrayList<>(); - for (String expr : exprList) { + for (String expr : exprList.keySet()) { sqlNodeList.add(parse(expr, scope, engineType)); sqlNodeList.add(new SqlDataTypeSpec( new SqlUserDefinedTypeNameSpec(expr + Constants.DIMENSION_ARRAY_SINGLE_SUFFIX, SqlParserPos.ZERO), diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/extend/LateralViewExplodeNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/extend/LateralViewExplodeNode.java index 87458bf01..4cf1206d0 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/extend/LateralViewExplodeNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/extend/LateralViewExplodeNode.java @@ -2,6 +2,8 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql.node.extend; import com.tencent.supersonic.headless.core.parser.calcite.sql.node.ExtendNode; import java.util.Iterator; +import java.util.Map; +import java.util.Objects; import org.apache.calcite.linq4j.Ord; import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlIdentifier; @@ -16,11 +18,14 @@ import org.apache.calcite.sql.SqlWriter; */ public class LateralViewExplodeNode extends ExtendNode { - public final String sqlNameView = "dataSet"; + public final String sqlNameView = "view"; public final String sqlNameExplode = "explode"; + public final String sqlNameExplodeSplit = "explode_split"; + private Map delimiterMap; - public LateralViewExplodeNode() { + public LateralViewExplodeNode(Map delimiterMap) { super(); + this.delimiterMap = delimiterMap; } public void unparse(SqlWriter writer, SqlCall call, int leftPrec, int rightPrec) { @@ -55,9 +60,20 @@ public class LateralViewExplodeNode extends ExtendNode { } public void explode(SqlWriter writer, SqlNode sqlNode) { - writer.sep(sqlNameExplode); + String delimiter = + Objects.nonNull(delimiterMap) && delimiterMap.containsKey(sqlNode.toString()) ? delimiterMap.get( + sqlNode.toString()) : ""; + if (delimiter.isEmpty()) { + writer.sep(sqlNameExplode); + } else { + writer.sep(sqlNameExplodeSplit); + } SqlWriter.Frame frame = writer.startList("(", ")"); sqlNode.unparse(writer, 0, 0); + if (!delimiter.isEmpty()) { + writer.sep(","); + writer.sep(String.format("'%s'", delimiter)); + } writer.endList(frame); writer.sep("tmp_sgl_" + sqlNode.toString()); } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/SourceRender.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/SourceRender.java index ab70c3cb2..cadcccbd3 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/SourceRender.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/render/SourceRender.java @@ -1,8 +1,9 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql.render; +import static com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants.DIMENSION_DELIMITER; + import com.tencent.supersonic.headless.api.pojo.enums.EngineType; -import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension; @@ -19,19 +20,22 @@ import com.tencent.supersonic.headless.core.parser.calcite.sql.node.FilterNode; import com.tencent.supersonic.headless.core.parser.calcite.sql.node.IdentifyNode; import com.tencent.supersonic.headless.core.parser.calcite.sql.node.MetricNode; import com.tencent.supersonic.headless.core.parser.calcite.sql.node.SemanticNode; -import lombok.extern.slf4j.Slf4j; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.validate.SqlValidatorScope; -import org.springframework.util.CollectionUtils; - +import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.validate.SqlValidatorScope; +import org.springframework.util.CollectionUtils; /** * process the table dataSet from the defined data model schema @@ -49,7 +53,7 @@ public class SourceRender extends Renderer { List queryMetrics = new ArrayList<>(reqMetrics); List queryDimensions = new ArrayList<>(reqDimensions); List fieldWhere = new ArrayList<>(fieldWheres); - Set extendFields = new HashSet<>(); + Map extendFields = new HashMap<>(); if (!fieldWhere.isEmpty()) { Set dimensions = new HashSet<>(); Set metrics = new HashSet<>(); @@ -95,7 +99,8 @@ public class SourceRender extends Renderer { } private static void buildDimension(String alias, String dimension, DataSource datasource, SemanticSchema schema, - boolean nonAgg, Set extendFields, TableView dataSet, TableView output, SqlValidatorScope scope) + boolean nonAgg, Map extendFields, TableView dataSet, TableView output, + SqlValidatorScope scope) throws Exception { List dimensionList = schema.getDimension().get(datasource.getName()); EngineType engineType = EngineType.fromString(schema.getSemanticModel().getDatabase().getType()); @@ -106,6 +111,7 @@ public class SourceRender extends Renderer { continue; } dataSet.getMeasure().add(DimensionNode.build(dim, scope, engineType)); + addExtendFields(dim, extendFields); if (nonAgg) { output.getMeasure().add(DimensionNode.buildName(dim, scope, engineType)); isAdd = true; @@ -141,9 +147,7 @@ public class SourceRender extends Renderer { Optional dimensionOptional = getDimensionByName(dimension, datasource); if (dimensionOptional.isPresent()) { dataSet.getMeasure().add(DimensionNode.buildArray(dimensionOptional.get(), scope, engineType)); - if (dimensionOptional.get().getDataType().isArray()) { - extendFields.add(dimensionOptional.get().getExpr()); - } + addExtendFields(dimensionOptional.get(), extendFields); if (nonAgg) { output.getMeasure().add(DimensionNode.buildName(dimensionOptional.get(), scope, engineType)); return; @@ -152,16 +156,21 @@ public class SourceRender extends Renderer { } } - private static boolean isWhereHasMetric(List fields, DataSource datasource) { - Long metricNum = datasource.getMeasures().stream().filter(m -> fields.contains(m.getName().toLowerCase())) - .count(); - Long measureNum = datasource.getMeasures().stream().filter(m -> fields.contains(m.getName().toLowerCase())) - .count(); - return metricNum > 0 || measureNum > 0; + private static void addExtendFields(Dimension dimension, Map extendFields) { + if (dimension.getDataType().isArray()) { + if (Objects.nonNull(dimension.getExt()) && dimension.getExt() + .containsKey(DIMENSION_DELIMITER)) { + extendFields.put(dimension.getExpr(), + (String) dimension.getExt().get(DIMENSION_DELIMITER)); + } else { + extendFields.put(dimension.getExpr(), ""); + } + } } private static List getWhereMeasure(List fields, List queryMetrics, - List queryDimensions, Set extendFields, DataSource datasource, SqlValidatorScope scope, + List queryDimensions, Map extendFields, DataSource datasource, + SqlValidatorScope scope, SemanticSchema schema, boolean nonAgg) throws Exception { Iterator iterator = fields.iterator(); List whereNode = new ArrayList<>(); @@ -195,9 +204,7 @@ public class SourceRender extends Renderer { Optional dimensionOptional = getDimensionByName(where, datasource); if (dimensionOptional.isPresent()) { whereNode.add(DimensionNode.buildArray(dimensionOptional.get(), scope, engineType)); - if (dimensionOptional.get().getDataType().isArray()) { - extendFields.add(dimensionOptional.get().getExpr()); - } + addExtendFields(dimensionOptional.get(), extendFields); } } return whereNode; @@ -205,7 +212,8 @@ public class SourceRender extends Renderer { private static void mergeWhere(List fields, TableView dataSet, TableView outputSet, List queryMetrics, - List queryDimensions, Set extendFields, DataSource datasource, SqlValidatorScope scope, + List queryDimensions, Map extendFields, DataSource datasource, + SqlValidatorScope scope, SemanticSchema schema, boolean nonAgg) throws Exception { List whereNode = getWhereMeasure(fields, queryMetrics, queryDimensions, extendFields, datasource, diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java index 88245bef4..a993e54bd 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java @@ -31,7 +31,6 @@ import com.tencent.supersonic.headless.server.pojo.yaml.MetricTypeParamsYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl; import com.tencent.supersonic.headless.server.service.Catalog; import com.tencent.supersonic.headless.server.utils.DatabaseConverter; - import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; @@ -42,7 +41,6 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; - import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Triple; import org.springframework.stereotype.Service; @@ -292,6 +290,9 @@ public class SemanticSchemaManager { if (Objects.isNull(dimension.getDataType())) { dimension.setDataType(DataType.UNKNOWN); } + if (Objects.nonNull(dimensionYamlTpl.getExt())) { + dimension.setExt(dimensionYamlTpl.getExt()); + } dimension.setDimensionTimeTypeParams(getDimensionTimeTypeParams(dimensionYamlTpl.getTypeParams())); dimensions.add(dimension); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DimensionYamlTpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DimensionYamlTpl.java index 85efdeaa2..8f2882634 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DimensionYamlTpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/yaml/DimensionYamlTpl.java @@ -3,6 +3,7 @@ package com.tencent.supersonic.headless.server.pojo.yaml; import com.tencent.supersonic.common.pojo.enums.DataTypeEnums; import java.util.List; +import java.util.Map; import lombok.Data; @@ -24,4 +25,6 @@ public class DimensionYamlTpl { private DataTypeEnums dataType; private List defaultValues; + + private Map ext; }