[improvement][headless]bizName of dimension and metric shall not be the same as the field name in the database table.

common/src/main/java/com/tencent/supersonic/common/jsqlparser/QueryExpressionReplaceVisitor.java

@@ -1,13 +1,10 @@
 package com.tencent.supersonic.common.jsqlparser;
 
-import net.sf.jsqlparser.expression.Alias;
+import net.sf.jsqlparser.expression.*;
-import net.sf.jsqlparser.expression.BinaryExpression;
-import net.sf.jsqlparser.expression.Expression;
-import net.sf.jsqlparser.expression.ExpressionVisitorAdapter;
-import net.sf.jsqlparser.expression.Function;
 import net.sf.jsqlparser.parser.CCJSqlParserUtil;
 import net.sf.jsqlparser.schema.Column;
 import net.sf.jsqlparser.statement.select.SelectItem;
+import org.apache.commons.lang3.StringUtils;
 
 import java.util.Map;
 import java.util.Objects;
@@ -50,7 +47,8 @@ public class QueryExpressionReplaceVisitor extends ExpressionVisitorAdapter {
         String columnName = "";
         if (expression instanceof Function) {
             Function leftFunc = (Function) expression;
-            if (leftFunc.getParameters().getExpressions().get(0) instanceof Column) {
+            if (Objects.nonNull(leftFunc.getParameters())
+                    && leftFunc.getParameters().getExpressions().get(0) instanceof Column) {
                 Column column = (Column) leftFunc.getParameters().getExpressions().get(0);
                 columnName = column.getColumnName();
                 toReplace = getReplaceExpr(leftFunc, fieldExprMap);
@@ -75,7 +73,10 @@ public class QueryExpressionReplaceVisitor extends ExpressionVisitorAdapter {
     public static Expression replace(Expression expression, Map<String, String> fieldExprMap) {
         String toReplace = "";
         if (expression instanceof Function) {
-            toReplace = getReplaceExpr((Function) expression, fieldExprMap);
+            Function function = (Function) expression;
+            if (function.getParameters().getExpressions().get(0) instanceof Column) {
+                toReplace = getReplaceExpr((Function) expression, fieldExprMap);
+            }
         }
         if (expression instanceof Column) {
             toReplace = getReplaceExpr((Column) expression, fieldExprMap);
@@ -109,6 +110,16 @@ public class QueryExpressionReplaceVisitor extends ExpressionVisitorAdapter {
 
     public static String getReplaceExpr(Function function, Map<String, String> fieldExprMap) {
         Column column = (Column) function.getParameters().getExpressions().get(0);
-        return getReplaceExpr(column, fieldExprMap);
+        String expr = getReplaceExpr(column, fieldExprMap);
+        // if metric expr itself has agg function then replace original function in the SQL
+        if (StringUtils.isBlank(expr)) {
+            return expr;
+        } else if (!SqlSelectFunctionHelper.getAggregateFunctions(expr).isEmpty()) {
+            return expr;
+        } else {
+            String col = getReplaceExpr(column, fieldExprMap);
+            column.setColumnName(col);
+            return function.toString();
+        }
     }
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Dimension.java

@@ -32,6 +32,16 @@ public class Dimension {
         this.type = type;
         this.isCreateDimension = isCreateDimension;
         this.bizName = bizName;
+        this.expr = bizName;
+    }
+
+    public Dimension(String name, String bizName, String expr, DimensionType type,
+            Integer isCreateDimension) {
+        this.name = name;
+        this.type = type;
+        this.isCreateDimension = isCreateDimension;
+        this.bizName = bizName;
+        this.expr = expr;
     }
 
     public Dimension(String name, String bizName, DimensionType type, Integer isCreateDimension,

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Measure.java

@@ -23,11 +23,20 @@ public class Measure {
 
     private String alias;
 
+    public Measure(String name, String bizName, String expr, String agg, Integer isCreateMetric) {
+        this.name = name;
+        this.agg = agg;
+        this.isCreateMetric = isCreateMetric;
+        this.bizName = bizName;
+        this.expr = expr;
+    }
+
     public Measure(String name, String bizName, String agg, Integer isCreateMetric) {
         this.name = name;
         this.agg = agg;
         this.isCreateMetric = isCreateMetric;
         this.bizName = bizName;
+        this.expr = bizName;
     }
 
     public Measure(String bizName, String constraint) {

headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimSchemaResp.java

@@ -1,13 +1,17 @@
 package com.tencent.supersonic.headless.api.pojo.response;
 
+import com.google.common.collect.Sets;
 import lombok.Data;
 import lombok.ToString;
 
+import java.util.Set;
+
 @Data
 @ToString(callSuper = true)
 public class DimSchemaResp extends DimensionResp {
 
     private Long useCnt = 0L;
+    private Set<String> fields = Sets.newHashSet();
 
     @Override
     public boolean equals(Object o) {

headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/DimExpressionConverter.java

@@ -0,0 +1,73 @@
+package com.tencent.supersonic.headless.core.translator.converter;
+
+import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
+import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
+import com.tencent.supersonic.headless.api.pojo.Dimension;
+import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
+import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
+import com.tencent.supersonic.headless.core.pojo.OntologyQuery;
+import com.tencent.supersonic.headless.core.pojo.QueryStatement;
+import com.tencent.supersonic.headless.core.pojo.SqlQuery;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Component;
+import org.springframework.util.CollectionUtils;
+
+import java.util.*;
+
+/**
+ * This converter replaces dimension bizName in the S2SQL with calculation expression (if
+ * configured).
+ */
+@Component("DimExpressionConverter")
+@Slf4j
+public class DimExpressionConverter implements QueryConverter {
+    @Override
+    public boolean accept(QueryStatement queryStatement) {
+        return Objects.nonNull(queryStatement.getSqlQuery())
+                && StringUtils.isNotBlank(queryStatement.getSqlQuery().getSql());
+    }
+
+    @Override
+    public void convert(QueryStatement queryStatement) throws Exception {
+
+        SemanticSchemaResp semanticSchema = queryStatement.getSemanticSchema();
+        SqlQuery sqlQuery = queryStatement.getSqlQuery();
+        OntologyQuery ontologyQuery = queryStatement.getOntologyQuery();
+
+        Map<String, String> bizName2Expr = getDimensionExpressions(semanticSchema, ontologyQuery);
+        if (!CollectionUtils.isEmpty(bizName2Expr)) {
+            String sql = SqlReplaceHelper.replaceSqlByExpression(sqlQuery.getSql(), bizName2Expr);
+            sqlQuery.setSql(sql);
+        }
+    }
+
+    private Map<String, String> getDimensionExpressions(SemanticSchemaResp semanticSchema,
+            OntologyQuery ontologyQuery) {
+
+        Set<DimSchemaResp> queryDimensions = ontologyQuery.getDimensions();
+        Set<String> queryFields = ontologyQuery.getFields();
+        log.debug("begin to generateDerivedMetric {} [{}]", queryDimensions);
+
+        Set<String> allFields = new HashSet<>();
+        Map<String, Dimension> dimensionMap = new HashMap<>();
+        semanticSchema.getModelResps().forEach(modelResp -> {
+            allFields.addAll(modelResp.getFieldList());
+            if (modelResp.getModelDetail().getDimensions() != null) {
+                modelResp.getModelDetail().getDimensions()
+                        .forEach(dimension -> dimensionMap.put(dimension.getBizName(), dimension));
+            }
+        });
+
+
+        Map<String, String> dim2Expr = new HashMap<>();
+        for (DimSchemaResp queryDim : queryDimensions) {
+            queryDim.getFields().addAll(SqlSelectHelper.getFieldsFromExpr(queryDim.getExpr()));
+            dim2Expr.put(queryDim.getBizName(), queryDim.getExpr());
+            queryFields.addAll(queryDim.getFields());
+        }
+
+        return dim2Expr;
+    }
+
+}

headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/MetricExpressionConverter.java

@@ -2,11 +2,8 @@ package com.tencent.supersonic.headless.core.translator.converter;
 
 import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
 import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
-import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
 import com.tencent.supersonic.headless.api.pojo.Measure;
-import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
 import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
-import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
 import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
 import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
 import com.tencent.supersonic.headless.core.pojo.OntologyQuery;
@@ -20,7 +17,7 @@ import org.springframework.util.CollectionUtils;
 import java.util.*;
 
 /**
- * This converter replaces metric fields in the S2SQL with calculation expressions (if configured).
+ * This converter replaces metric bizName in the S2SQL with calculation expression (if configured).
  */
 @Component("MetricExpressionConverter")
 @Slf4j
@@ -38,11 +35,10 @@ public class MetricExpressionConverter implements QueryConverter {
         SqlQuery sqlQuery = queryStatement.getSqlQuery();
         OntologyQuery ontologyQuery = queryStatement.getOntologyQuery();
 
-        Map<String, String> metric2Expr = getMetricExpressions(semanticSchema, ontologyQuery);
+        Map<String, String> bizName2Expr = getMetricExpressions(semanticSchema, ontologyQuery);
-        if (!CollectionUtils.isEmpty(metric2Expr)) {
+        if (!CollectionUtils.isEmpty(bizName2Expr)) {
-            String sql = SqlReplaceHelper.replaceSqlByExpression(sqlQuery.getSql(), metric2Expr);
+            String sql = SqlReplaceHelper.replaceSqlByExpression(sqlQuery.getSql(), bizName2Expr);
             sqlQuery.setSql(sql);
-            ontologyQuery.setAggOption(AggOption.NATIVE);
         }
     }
 
@@ -50,12 +46,9 @@ public class MetricExpressionConverter implements QueryConverter {
             OntologyQuery ontologyQuery) {
 
         List<MetricSchemaResp> allMetrics = semanticSchema.getMetrics();
-        List<DimSchemaResp> allDimensions = semanticSchema.getDimensions();
-        AggOption aggOption = ontologyQuery.getAggOption();
         Set<MetricSchemaResp> queryMetrics = ontologyQuery.getMetrics();
-        Set<DimSchemaResp> queryDimensions = ontologyQuery.getDimensions();
         Set<String> queryFields = ontologyQuery.getFields();
-        log.debug("begin to generateDerivedMetric {} [{}]", aggOption, queryMetrics);
+        log.debug("begin to generateDerivedMetric {} [{}]", queryMetrics);
 
         Set<String> allFields = new HashSet<>();
         Map<String, Measure> allMeasures = new HashMap<>();
@@ -70,13 +63,12 @@ public class MetricExpressionConverter implements QueryConverter {
         Map<String, String> visitedMetrics = new HashMap<>();
         Map<String, String> metric2Expr = new HashMap<>();
         for (MetricSchemaResp queryMetric : queryMetrics) {
-            String fieldExpr = buildFieldExpr(allMetrics, allFields, allMeasures, allDimensions,
+            String fieldExpr = buildFieldExpr(allMetrics, allMeasures, queryMetric.getExpr(),
-                    queryMetric.getExpr(), queryMetric.getMetricDefineType(), aggOption,
+                    queryMetric.getMetricDefineType(), visitedMetrics, queryFields);
-                    visitedMetrics, queryDimensions, queryFields);
             // add all fields referenced in the expression
             queryMetric.getFields().addAll(SqlSelectHelper.getFieldsFromExpr(fieldExpr));
             log.debug("derived metric {}->{}", queryMetric.getBizName(), fieldExpr);
-            if (queryMetric.isDerived()) {
+            if (!queryMetric.getBizName().equals(fieldExpr)) {
                 metric2Expr.put(queryMetric.getBizName(), fieldExpr);
             }
         }
@@ -85,18 +77,16 @@ public class MetricExpressionConverter implements QueryConverter {
     }
 
     private String buildFieldExpr(final List<MetricSchemaResp> metricResps,
-            final Set<String> allFields, final Map<String, Measure> allMeasures,
+            final Map<String, Measure> allMeasures, final String metricExpr,
-            final List<DimSchemaResp> dimensionResps, final String expression,
+            final MetricDefineType metricDefineType, Map<String, String> visitedMetric,
-            final MetricDefineType metricDefineType, AggOption aggOption,
-            Map<String, String> visitedMetric, Set<DimSchemaResp> queryDimensions,
             Set<String> queryFields) {
-        Set<String> fields = SqlSelectHelper.getFieldsFromExpr(expression);
+        Set<String> fields = SqlSelectHelper.getFieldsFromExpr(metricExpr);
         if (!CollectionUtils.isEmpty(fields)) {
             Map<String, String> replace = new HashMap<>();
             for (String field : fields) {
-                queryFields.add(field);
                 switch (metricDefineType) {
                     case METRIC:
+                        // if defineType=METRIC, field should be the bizName of its parent metric
                         Optional<MetricSchemaResp> metricItem = metricResps.stream()
                                 .filter(m -> m.getBizName().equalsIgnoreCase(field)).findFirst();
                         if (metricItem.isPresent()) {
@@ -105,49 +95,39 @@ public class MetricExpressionConverter implements QueryConverter {
                                 break;
                             }
                             replace.put(field,
-                                    buildFieldExpr(metricResps, allFields, allMeasures,
+                                    buildFieldExpr(metricResps, allMeasures,
-                                            dimensionResps, metricItem.get().getExpr(),
+                                            metricItem.get().getExpr(),
-                                            metricItem.get().getMetricDefineType(), aggOption,
+                                            metricItem.get().getMetricDefineType(), visitedMetric,
-                                            visitedMetric, queryDimensions, queryFields));
+                                            queryFields));
                             visitedMetric.put(field, replace.get(field));
                         }
                         break;
                     case MEASURE:
+                        // if defineType=MEASURE, field should be the bizName of its measure
                         if (allMeasures.containsKey(field)) {
                             Measure measure = allMeasures.get(field);
-                            if (AggOperatorEnum.COUNT_DISTINCT.getOperator()
+                            String expr = metricExpr;
-                                    .equalsIgnoreCase(measure.getAgg())) {
+                            if (Objects.nonNull(measure.getAgg())) {
-                                return AggOption.NATIVE.equals(aggOption) ? measure.getExpr()
+                                expr = String.format("%s (%s)", measure.getAgg(), metricExpr);
-                                        : AggOperatorEnum.COUNT.getOperator() + " ( "
-                                                + AggOperatorEnum.DISTINCT + " " + measure.getExpr()
-                                                + " ) ";
                             }
-                            String expr = AggOption.NATIVE.equals(aggOption) ? measure.getExpr()
-                                    : measure.getAgg() + " ( " + measure.getExpr() + " ) ";
-
                             replace.put(field, expr);
+                            queryFields.add(field);
                         }
                         break;
                     case FIELD:
-                        if (allFields.contains(field)) {
+                        queryFields.add(field);
-                            Optional<DimSchemaResp> dimensionItem = dimensionResps.stream()
-                                    .filter(d -> d.getBizName().equals(field)).findFirst();
-                            if (dimensionItem.isPresent()) {
-                                queryDimensions.add(dimensionItem.get());
-                            }
-                        }
                         break;
                     default:
                         break;
                 }
             }
             if (!CollectionUtils.isEmpty(replace)) {
-                String expr = SqlReplaceHelper.replaceExpression(expression, replace);
+                String expr = SqlReplaceHelper.replaceExpression(metricExpr, replace);
-                log.debug("derived measure {}->{}", expression, expr);
+                log.debug("derived measure {}->{}", metricExpr, expr);
                 return expr;
             }
         }
-        return expression;
+        return metricExpr;
     }
 
 }

headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlQueryConverter.java

@@ -65,13 +65,7 @@ public class SqlQueryConverter implements QueryConverter {
         ontologyQuery.getDimensions().addAll(queryDimensions);
 
         AggOption sqlQueryAggOption = getAggOption(sqlQuery.getSql(), queryMetrics);
-        // if sql query itself has aggregation, ontology query just returns detail
+        ontologyQuery.setAggOption(sqlQueryAggOption);
-        if (sqlQueryAggOption.equals(AggOption.AGGREGATION)) {
-            ontologyQuery.setAggOption(AggOption.NATIVE);
-        } else if (sqlQueryAggOption.equals(AggOption.NATIVE) && !queryMetrics.isEmpty()) {
-            ontologyQuery.setAggOption(AggOption.DEFAULT);
-        }
-
         queryStatement.setOntologyQuery(ontologyQuery);
         log.info("parse sqlQuery [{}] ", sqlQuery);
     }

headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/SqlBuilder.java

@@ -251,7 +251,7 @@ public class SqlBuilder {
         EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType());
         Set<String> queryFields = tableView.getFields();
         queryMetrics.stream().forEach(m -> queryFields.addAll(m.getFields()));
-        queryDimensions.stream().forEach(m -> queryFields.add(m.getBizName()));
+        queryDimensions.stream().forEach(d -> queryFields.addAll(d.getFields()));
 
         try {
             for (String field : queryFields) {

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java

@@ -28,7 +28,8 @@ public class ModelConverter {
 
     public static ModelDO convert(ModelReq modelReq, User user) {
         ModelDO modelDO = new ModelDO();
-        ModelDetail modelDetail = createModelDetail(modelReq);
+        // ModelDetail modelDetail = createModelDetail(modelReq);
+        ModelDetail modelDetail = modelReq.getModelDetail();
         modelReq.createdBy(user.getName());
         BeanMapper.mapper(modelReq, modelDO);
         modelDO.setStatus(StatusEnum.ONLINE.getCode());
@@ -107,7 +108,7 @@ public class ModelConverter {
             dimensionReq.setSemanticType(SemanticType.CATEGORY.name());
         }
         dimensionReq.setModelId(modelDO.getId());
-        dimensionReq.setExpr(dim.getBizName());
+        dimensionReq.setExpr(dim.getExpr());
         dimensionReq.setType(dim.getType().name());
         dimensionReq
                 .setDescription(Objects.isNull(dim.getDescription()) ? "" : dim.getDescription());
@@ -118,11 +119,11 @@ public class ModelConverter {
     public static MetricReq convert(Measure measure, ModelDO modelDO) {
         MetricReq metricReq = new MetricReq();
         metricReq.setName(measure.getName());
-        metricReq.setBizName(measure.getExpr());
+        metricReq.setBizName(measure.getBizName());
         metricReq.setDescription(measure.getName());
         metricReq.setModelId(modelDO.getId());
         MetricDefineByMeasureParams exprTypeParams = new MetricDefineByMeasureParams();
-        exprTypeParams.setExpr(measure.getBizName());
+        exprTypeParams.setExpr(measure.getExpr());
         exprTypeParams.setMeasures(Lists.newArrayList(measure));
         metricReq.setMetricDefineByMeasureParams(exprTypeParams);
         metricReq.setMetricDefineType(MetricDefineType.MEASURE);
@@ -163,11 +164,14 @@ public class ModelConverter {
                         getIdentifyType(fieldType).name(), columnSchema.getColumnName(), 1);
                 modelDetail.getIdentifiers().add(identify);
             } else if (FieldType.measure.equals(fieldType)) {
-                Measure measure = new Measure(columnSchema.getName(), columnSchema.getColumnName(),
+                Measure measure = new Measure(columnSchema.getName(),
-                        columnSchema.getAgg().getOperator(), 1);
+                        modelReq.getBizName() + "_" + columnSchema.getColumnName(),
+                        columnSchema.getColumnName(), columnSchema.getAgg().getOperator(), 1);
                 modelDetail.getMeasures().add(measure);
             } else {
-                Dimension dim = new Dimension(columnSchema.getName(), columnSchema.getColumnName(),
+                Dimension dim = new Dimension(columnSchema.getName(),
+                        modelReq.getBizName() + "_" + columnSchema.getColumnName(),
+                        columnSchema.getColumnName(),
                         DimensionType.valueOf(columnSchema.getFiledType().name()), 1);
                 modelDetail.getDimensions().add(dim);
             }

launchers/standalone/src/main/resources/META-INF/spring.factories

@@ -30,6 +30,7 @@ com.tencent.supersonic.headless.core.translator.converter.QueryConverter=\
     com.tencent.supersonic.headless.core.translator.converter.StructQueryConverter,\
     com.tencent.supersonic.headless.core.translator.converter.SqlQueryConverter,\
     com.tencent.supersonic.headless.core.translator.converter.DefaultDimValueConverter,\
+    com.tencent.supersonic.headless.core.translator.converter.DimExpressionConverter,\
     com.tencent.supersonic.headless.core.translator.converter.MetricExpressionConverter,\
     com.tencent.supersonic.headless.core.translator.converter.MetricRatioConverter