diff --git a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/QueryExpressionReplaceVisitor.java b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/QueryExpressionReplaceVisitor.java index f47d99bf4..f0ce0465e 100644 --- a/common/src/main/java/com/tencent/supersonic/common/jsqlparser/QueryExpressionReplaceVisitor.java +++ b/common/src/main/java/com/tencent/supersonic/common/jsqlparser/QueryExpressionReplaceVisitor.java @@ -1,13 +1,10 @@ package com.tencent.supersonic.common.jsqlparser; -import net.sf.jsqlparser.expression.Alias; -import net.sf.jsqlparser.expression.BinaryExpression; -import net.sf.jsqlparser.expression.Expression; -import net.sf.jsqlparser.expression.ExpressionVisitorAdapter; -import net.sf.jsqlparser.expression.Function; +import net.sf.jsqlparser.expression.*; import net.sf.jsqlparser.parser.CCJSqlParserUtil; import net.sf.jsqlparser.schema.Column; import net.sf.jsqlparser.statement.select.SelectItem; +import org.apache.commons.lang3.StringUtils; import java.util.Map; import java.util.Objects; @@ -50,7 +47,8 @@ public class QueryExpressionReplaceVisitor extends ExpressionVisitorAdapter { String columnName = ""; if (expression instanceof Function) { Function leftFunc = (Function) expression; - if (leftFunc.getParameters().getExpressions().get(0) instanceof Column) { + if (Objects.nonNull(leftFunc.getParameters()) + && leftFunc.getParameters().getExpressions().get(0) instanceof Column) { Column column = (Column) leftFunc.getParameters().getExpressions().get(0); columnName = column.getColumnName(); toReplace = getReplaceExpr(leftFunc, fieldExprMap); @@ -75,7 +73,10 @@ public class QueryExpressionReplaceVisitor extends ExpressionVisitorAdapter { public static Expression replace(Expression expression, Map fieldExprMap) { String toReplace = ""; if (expression instanceof Function) { - toReplace = getReplaceExpr((Function) expression, fieldExprMap); + Function function = (Function) expression; + if (function.getParameters().getExpressions().get(0) instanceof Column) { + toReplace = getReplaceExpr((Function) expression, fieldExprMap); + } } if (expression instanceof Column) { toReplace = getReplaceExpr((Column) expression, fieldExprMap); @@ -109,6 +110,16 @@ public class QueryExpressionReplaceVisitor extends ExpressionVisitorAdapter { public static String getReplaceExpr(Function function, Map fieldExprMap) { Column column = (Column) function.getParameters().getExpressions().get(0); - return getReplaceExpr(column, fieldExprMap); + String expr = getReplaceExpr(column, fieldExprMap); + // if metric expr itself has agg function then replace original function in the SQL + if (StringUtils.isBlank(expr)) { + return expr; + } else if (!SqlSelectFunctionHelper.getAggregateFunctions(expr).isEmpty()) { + return expr; + } else { + String col = getReplaceExpr(column, fieldExprMap); + column.setColumnName(col); + return function.toString(); + } } } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Dimension.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Dimension.java index 6ed5b0450..d67bb201d 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Dimension.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Dimension.java @@ -32,6 +32,16 @@ public class Dimension { this.type = type; this.isCreateDimension = isCreateDimension; this.bizName = bizName; + this.expr = bizName; + } + + public Dimension(String name, String bizName, String expr, DimensionType type, + Integer isCreateDimension) { + this.name = name; + this.type = type; + this.isCreateDimension = isCreateDimension; + this.bizName = bizName; + this.expr = expr; } public Dimension(String name, String bizName, DimensionType type, Integer isCreateDimension, diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Measure.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Measure.java index b39521be3..24d8665fe 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Measure.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Measure.java @@ -23,11 +23,20 @@ public class Measure { private String alias; + public Measure(String name, String bizName, String expr, String agg, Integer isCreateMetric) { + this.name = name; + this.agg = agg; + this.isCreateMetric = isCreateMetric; + this.bizName = bizName; + this.expr = expr; + } + public Measure(String name, String bizName, String agg, Integer isCreateMetric) { this.name = name; this.agg = agg; this.isCreateMetric = isCreateMetric; this.bizName = bizName; + this.expr = bizName; } public Measure(String bizName, String constraint) { diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimSchemaResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimSchemaResp.java index 3500c825e..201c31a82 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimSchemaResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimSchemaResp.java @@ -1,13 +1,17 @@ package com.tencent.supersonic.headless.api.pojo.response; +import com.google.common.collect.Sets; import lombok.Data; import lombok.ToString; +import java.util.Set; + @Data @ToString(callSuper = true) public class DimSchemaResp extends DimensionResp { private Long useCnt = 0L; + private Set fields = Sets.newHashSet(); @Override public boolean equals(Object o) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/DimExpressionConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/DimExpressionConverter.java new file mode 100644 index 000000000..da96e2d10 --- /dev/null +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/DimExpressionConverter.java @@ -0,0 +1,73 @@ +package com.tencent.supersonic.headless.core.translator.converter; + +import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper; +import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; +import com.tencent.supersonic.headless.api.pojo.Dimension; +import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; +import com.tencent.supersonic.headless.core.pojo.OntologyQuery; +import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import com.tencent.supersonic.headless.core.pojo.SqlQuery; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + +import java.util.*; + +/** + * This converter replaces dimension bizName in the S2SQL with calculation expression (if + * configured). + */ +@Component("DimExpressionConverter") +@Slf4j +public class DimExpressionConverter implements QueryConverter { + @Override + public boolean accept(QueryStatement queryStatement) { + return Objects.nonNull(queryStatement.getSqlQuery()) + && StringUtils.isNotBlank(queryStatement.getSqlQuery().getSql()); + } + + @Override + public void convert(QueryStatement queryStatement) throws Exception { + + SemanticSchemaResp semanticSchema = queryStatement.getSemanticSchema(); + SqlQuery sqlQuery = queryStatement.getSqlQuery(); + OntologyQuery ontologyQuery = queryStatement.getOntologyQuery(); + + Map bizName2Expr = getDimensionExpressions(semanticSchema, ontologyQuery); + if (!CollectionUtils.isEmpty(bizName2Expr)) { + String sql = SqlReplaceHelper.replaceSqlByExpression(sqlQuery.getSql(), bizName2Expr); + sqlQuery.setSql(sql); + } + } + + private Map getDimensionExpressions(SemanticSchemaResp semanticSchema, + OntologyQuery ontologyQuery) { + + Set queryDimensions = ontologyQuery.getDimensions(); + Set queryFields = ontologyQuery.getFields(); + log.debug("begin to generateDerivedMetric {} [{}]", queryDimensions); + + Set allFields = new HashSet<>(); + Map dimensionMap = new HashMap<>(); + semanticSchema.getModelResps().forEach(modelResp -> { + allFields.addAll(modelResp.getFieldList()); + if (modelResp.getModelDetail().getDimensions() != null) { + modelResp.getModelDetail().getDimensions() + .forEach(dimension -> dimensionMap.put(dimension.getBizName(), dimension)); + } + }); + + + Map dim2Expr = new HashMap<>(); + for (DimSchemaResp queryDim : queryDimensions) { + queryDim.getFields().addAll(SqlSelectHelper.getFieldsFromExpr(queryDim.getExpr())); + dim2Expr.put(queryDim.getBizName(), queryDim.getExpr()); + queryFields.addAll(queryDim.getFields()); + } + + return dim2Expr; + } + +} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/MetricExpressionConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/MetricExpressionConverter.java index 3556b018e..7d57529d3 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/MetricExpressionConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/MetricExpressionConverter.java @@ -2,11 +2,8 @@ package com.tencent.supersonic.headless.core.translator.converter; import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper; import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper; -import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.headless.api.pojo.Measure; -import com.tencent.supersonic.headless.api.pojo.enums.AggOption; import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType; -import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.core.pojo.OntologyQuery; @@ -20,7 +17,7 @@ import org.springframework.util.CollectionUtils; import java.util.*; /** - * This converter replaces metric fields in the S2SQL with calculation expressions (if configured). + * This converter replaces metric bizName in the S2SQL with calculation expression (if configured). */ @Component("MetricExpressionConverter") @Slf4j @@ -38,11 +35,10 @@ public class MetricExpressionConverter implements QueryConverter { SqlQuery sqlQuery = queryStatement.getSqlQuery(); OntologyQuery ontologyQuery = queryStatement.getOntologyQuery(); - Map metric2Expr = getMetricExpressions(semanticSchema, ontologyQuery); - if (!CollectionUtils.isEmpty(metric2Expr)) { - String sql = SqlReplaceHelper.replaceSqlByExpression(sqlQuery.getSql(), metric2Expr); + Map bizName2Expr = getMetricExpressions(semanticSchema, ontologyQuery); + if (!CollectionUtils.isEmpty(bizName2Expr)) { + String sql = SqlReplaceHelper.replaceSqlByExpression(sqlQuery.getSql(), bizName2Expr); sqlQuery.setSql(sql); - ontologyQuery.setAggOption(AggOption.NATIVE); } } @@ -50,12 +46,9 @@ public class MetricExpressionConverter implements QueryConverter { OntologyQuery ontologyQuery) { List allMetrics = semanticSchema.getMetrics(); - List allDimensions = semanticSchema.getDimensions(); - AggOption aggOption = ontologyQuery.getAggOption(); Set queryMetrics = ontologyQuery.getMetrics(); - Set queryDimensions = ontologyQuery.getDimensions(); Set queryFields = ontologyQuery.getFields(); - log.debug("begin to generateDerivedMetric {} [{}]", aggOption, queryMetrics); + log.debug("begin to generateDerivedMetric {} [{}]", queryMetrics); Set allFields = new HashSet<>(); Map allMeasures = new HashMap<>(); @@ -70,13 +63,12 @@ public class MetricExpressionConverter implements QueryConverter { Map visitedMetrics = new HashMap<>(); Map metric2Expr = new HashMap<>(); for (MetricSchemaResp queryMetric : queryMetrics) { - String fieldExpr = buildFieldExpr(allMetrics, allFields, allMeasures, allDimensions, - queryMetric.getExpr(), queryMetric.getMetricDefineType(), aggOption, - visitedMetrics, queryDimensions, queryFields); + String fieldExpr = buildFieldExpr(allMetrics, allMeasures, queryMetric.getExpr(), + queryMetric.getMetricDefineType(), visitedMetrics, queryFields); // add all fields referenced in the expression queryMetric.getFields().addAll(SqlSelectHelper.getFieldsFromExpr(fieldExpr)); log.debug("derived metric {}->{}", queryMetric.getBizName(), fieldExpr); - if (queryMetric.isDerived()) { + if (!queryMetric.getBizName().equals(fieldExpr)) { metric2Expr.put(queryMetric.getBizName(), fieldExpr); } } @@ -85,18 +77,16 @@ public class MetricExpressionConverter implements QueryConverter { } private String buildFieldExpr(final List metricResps, - final Set allFields, final Map allMeasures, - final List dimensionResps, final String expression, - final MetricDefineType metricDefineType, AggOption aggOption, - Map visitedMetric, Set queryDimensions, + final Map allMeasures, final String metricExpr, + final MetricDefineType metricDefineType, Map visitedMetric, Set queryFields) { - Set fields = SqlSelectHelper.getFieldsFromExpr(expression); + Set fields = SqlSelectHelper.getFieldsFromExpr(metricExpr); if (!CollectionUtils.isEmpty(fields)) { Map replace = new HashMap<>(); for (String field : fields) { - queryFields.add(field); switch (metricDefineType) { case METRIC: + // if defineType=METRIC, field should be the bizName of its parent metric Optional metricItem = metricResps.stream() .filter(m -> m.getBizName().equalsIgnoreCase(field)).findFirst(); if (metricItem.isPresent()) { @@ -105,49 +95,39 @@ public class MetricExpressionConverter implements QueryConverter { break; } replace.put(field, - buildFieldExpr(metricResps, allFields, allMeasures, - dimensionResps, metricItem.get().getExpr(), - metricItem.get().getMetricDefineType(), aggOption, - visitedMetric, queryDimensions, queryFields)); + buildFieldExpr(metricResps, allMeasures, + metricItem.get().getExpr(), + metricItem.get().getMetricDefineType(), visitedMetric, + queryFields)); visitedMetric.put(field, replace.get(field)); } break; case MEASURE: + // if defineType=MEASURE, field should be the bizName of its measure if (allMeasures.containsKey(field)) { Measure measure = allMeasures.get(field); - if (AggOperatorEnum.COUNT_DISTINCT.getOperator() - .equalsIgnoreCase(measure.getAgg())) { - return AggOption.NATIVE.equals(aggOption) ? measure.getExpr() - : AggOperatorEnum.COUNT.getOperator() + " ( " - + AggOperatorEnum.DISTINCT + " " + measure.getExpr() - + " ) "; + String expr = metricExpr; + if (Objects.nonNull(measure.getAgg())) { + expr = String.format("%s (%s)", measure.getAgg(), metricExpr); } - String expr = AggOption.NATIVE.equals(aggOption) ? measure.getExpr() - : measure.getAgg() + " ( " + measure.getExpr() + " ) "; - replace.put(field, expr); + queryFields.add(field); } break; case FIELD: - if (allFields.contains(field)) { - Optional dimensionItem = dimensionResps.stream() - .filter(d -> d.getBizName().equals(field)).findFirst(); - if (dimensionItem.isPresent()) { - queryDimensions.add(dimensionItem.get()); - } - } + queryFields.add(field); break; default: break; } } if (!CollectionUtils.isEmpty(replace)) { - String expr = SqlReplaceHelper.replaceExpression(expression, replace); - log.debug("derived measure {}->{}", expression, expr); + String expr = SqlReplaceHelper.replaceExpression(metricExpr, replace); + log.debug("derived measure {}->{}", metricExpr, expr); return expr; } } - return expression; + return metricExpr; } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlQueryConverter.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlQueryConverter.java index f31e863af..e03e03cc8 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlQueryConverter.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlQueryConverter.java @@ -65,13 +65,7 @@ public class SqlQueryConverter implements QueryConverter { ontologyQuery.getDimensions().addAll(queryDimensions); AggOption sqlQueryAggOption = getAggOption(sqlQuery.getSql(), queryMetrics); - // if sql query itself has aggregation, ontology query just returns detail - if (sqlQueryAggOption.equals(AggOption.AGGREGATION)) { - ontologyQuery.setAggOption(AggOption.NATIVE); - } else if (sqlQueryAggOption.equals(AggOption.NATIVE) && !queryMetrics.isEmpty()) { - ontologyQuery.setAggOption(AggOption.DEFAULT); - } - + ontologyQuery.setAggOption(sqlQueryAggOption); queryStatement.setOntologyQuery(ontologyQuery); log.info("parse sqlQuery [{}] ", sqlQuery); } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/SqlBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/SqlBuilder.java index fc700ecdd..74a162041 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/SqlBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/SqlBuilder.java @@ -251,7 +251,7 @@ public class SqlBuilder { EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType()); Set queryFields = tableView.getFields(); queryMetrics.stream().forEach(m -> queryFields.addAll(m.getFields())); - queryDimensions.stream().forEach(m -> queryFields.add(m.getBizName())); + queryDimensions.stream().forEach(d -> queryFields.addAll(d.getFields())); try { for (String field : queryFields) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java index cd1f5cd8d..e8300cbd2 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java @@ -28,7 +28,8 @@ public class ModelConverter { public static ModelDO convert(ModelReq modelReq, User user) { ModelDO modelDO = new ModelDO(); - ModelDetail modelDetail = createModelDetail(modelReq); + // ModelDetail modelDetail = createModelDetail(modelReq); + ModelDetail modelDetail = modelReq.getModelDetail(); modelReq.createdBy(user.getName()); BeanMapper.mapper(modelReq, modelDO); modelDO.setStatus(StatusEnum.ONLINE.getCode()); @@ -107,7 +108,7 @@ public class ModelConverter { dimensionReq.setSemanticType(SemanticType.CATEGORY.name()); } dimensionReq.setModelId(modelDO.getId()); - dimensionReq.setExpr(dim.getBizName()); + dimensionReq.setExpr(dim.getExpr()); dimensionReq.setType(dim.getType().name()); dimensionReq .setDescription(Objects.isNull(dim.getDescription()) ? "" : dim.getDescription()); @@ -118,11 +119,11 @@ public class ModelConverter { public static MetricReq convert(Measure measure, ModelDO modelDO) { MetricReq metricReq = new MetricReq(); metricReq.setName(measure.getName()); - metricReq.setBizName(measure.getExpr()); + metricReq.setBizName(measure.getBizName()); metricReq.setDescription(measure.getName()); metricReq.setModelId(modelDO.getId()); MetricDefineByMeasureParams exprTypeParams = new MetricDefineByMeasureParams(); - exprTypeParams.setExpr(measure.getBizName()); + exprTypeParams.setExpr(measure.getExpr()); exprTypeParams.setMeasures(Lists.newArrayList(measure)); metricReq.setMetricDefineByMeasureParams(exprTypeParams); metricReq.setMetricDefineType(MetricDefineType.MEASURE); @@ -163,11 +164,14 @@ public class ModelConverter { getIdentifyType(fieldType).name(), columnSchema.getColumnName(), 1); modelDetail.getIdentifiers().add(identify); } else if (FieldType.measure.equals(fieldType)) { - Measure measure = new Measure(columnSchema.getName(), columnSchema.getColumnName(), - columnSchema.getAgg().getOperator(), 1); + Measure measure = new Measure(columnSchema.getName(), + modelReq.getBizName() + "_" + columnSchema.getColumnName(), + columnSchema.getColumnName(), columnSchema.getAgg().getOperator(), 1); modelDetail.getMeasures().add(measure); } else { - Dimension dim = new Dimension(columnSchema.getName(), columnSchema.getColumnName(), + Dimension dim = new Dimension(columnSchema.getName(), + modelReq.getBizName() + "_" + columnSchema.getColumnName(), + columnSchema.getColumnName(), DimensionType.valueOf(columnSchema.getFiledType().name()), 1); modelDetail.getDimensions().add(dim); } diff --git a/launchers/standalone/src/main/resources/META-INF/spring.factories b/launchers/standalone/src/main/resources/META-INF/spring.factories index ca6e48105..22845e747 100644 --- a/launchers/standalone/src/main/resources/META-INF/spring.factories +++ b/launchers/standalone/src/main/resources/META-INF/spring.factories @@ -30,6 +30,7 @@ com.tencent.supersonic.headless.core.translator.converter.QueryConverter=\ com.tencent.supersonic.headless.core.translator.converter.StructQueryConverter,\ com.tencent.supersonic.headless.core.translator.converter.SqlQueryConverter,\ com.tencent.supersonic.headless.core.translator.converter.DefaultDimValueConverter,\ + com.tencent.supersonic.headless.core.translator.converter.DimExpressionConverter,\ com.tencent.supersonic.headless.core.translator.converter.MetricExpressionConverter,\ com.tencent.supersonic.headless.core.translator.converter.MetricRatioConverter