(improvement)(headless) adapter for derived metrics (#646)

headless/api/src/main/java/com/tencent/supersonic/headless/api/enums/MetricType.java

@@ -1,9 +1,44 @@
 package com.tencent.supersonic.headless.api.enums;
 
 
+import com.tencent.supersonic.headless.api.pojo.MeasureParam;
+import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
+import java.util.List;
+import java.util.Objects;
+
 public enum MetricType {
 
     ATOMIC,
-    DERIVED
+    DERIVED;
 
+    public static MetricType of(String src) {
+        for (MetricType metricType : MetricType.values()) {
+            if (Objects.nonNull(src) && src.equalsIgnoreCase(metricType.name())) {
+                return metricType;
+            }
+        }
+        return null;
+    }
+
+    public static Boolean isDerived(String src) {
+        MetricType metricType = of(src);
+        return Objects.nonNull(metricType) && metricType.equals(DERIVED);
+    }
+
+    public static Boolean isDerived(MetricDefineType metricDefineType, MetricDefineByMeasureParams typeParams) {
+        if (MetricDefineType.METRIC.equals(metricDefineType)) {
+            return true;
+        }
+        if (MetricDefineType.MEASURE.equals(metricDefineType)) {
+            List<MeasureParam> measures = typeParams.getMeasures();
+            if (measures.size() > 1) {
+                return true;
+            }
+            if (measures.size() == 1 && measures.get(0).getBizName()
+                    .equalsIgnoreCase(typeParams.getExpr())) {
+                return false;
+            }
+        }
+        return false;
+    }
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/request/MetricReq.java

@@ -3,12 +3,10 @@ package com.tencent.supersonic.headless.api.request;
 import com.alibaba.fastjson.JSONObject;
 import com.tencent.supersonic.headless.api.enums.MetricDefineType;
 import com.tencent.supersonic.headless.api.enums.MetricType;
-import com.tencent.supersonic.headless.api.pojo.MeasureParam;
 import com.tencent.supersonic.headless.api.pojo.MetricDefineByFieldParams;
 import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
 import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams;
 import lombok.Data;
-import java.util.List;
 
 @Data
 public class MetricReq extends MetricBaseReq {
@@ -30,20 +28,7 @@ public class MetricReq extends MetricBaseReq {
     }
 
     public MetricType getMetricType() {
-        if (MetricDefineType.METRIC.equals(metricDefineType)) {
-            return MetricType.DERIVED;
-        }
-        if (MetricDefineType.MEASURE.equals(metricDefineType)) {
-            List<MeasureParam> measures = typeParams.getMeasures();
-            if (measures.size() > 1) {
-                return MetricType.DERIVED;
-            }
-            if (measures.size() == 1 && measures.get(0).getBizName()
-                    .equalsIgnoreCase(typeParams.getExpr())) {
-                return MetricType.ATOMIC;
-            }
-        }
-        return MetricType.ATOMIC;
+        return MetricType.isDerived(metricDefineType, typeParams) ? MetricType.DERIVED : MetricType.ATOMIC;
     }
 
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/response/MetricResp.java

@@ -10,16 +10,15 @@ import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
 import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams;
 import com.tencent.supersonic.headless.api.pojo.RelateDimension;
 import com.tencent.supersonic.headless.api.pojo.SchemaItem;
-import lombok.Data;
-import lombok.ToString;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
-
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
+import lombok.Data;
+import lombok.ToString;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
 
 
 @Data

headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java

@@ -1,5 +1,12 @@
 package com.tencent.supersonic.headless.core.utils;
 
+import static com.tencent.supersonic.common.pojo.Constants.DAY;
+import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
+import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
+import static com.tencent.supersonic.common.pojo.Constants.MONTH;
+import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
+import static com.tencent.supersonic.common.pojo.Constants.WEEK;
+
 import com.tencent.supersonic.common.pojo.Aggregator;
 import com.tencent.supersonic.common.pojo.DateConf;
 import com.tencent.supersonic.common.pojo.ItemDateResp;
@@ -8,8 +15,26 @@ import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
 import com.tencent.supersonic.common.util.DateModeUtils;
 import com.tencent.supersonic.common.util.SqlFilterUtils;
 import com.tencent.supersonic.common.util.StringUtil;
+import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
+import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
 import com.tencent.supersonic.headless.api.enums.EngineType;
+import com.tencent.supersonic.headless.api.enums.MetricDefineType;
+import com.tencent.supersonic.headless.api.enums.MetricType;
+import com.tencent.supersonic.headless.api.pojo.Measure;
 import com.tencent.supersonic.headless.api.request.QueryStructReq;
+import com.tencent.supersonic.headless.api.response.DimensionResp;
+import com.tencent.supersonic.headless.api.response.MetricResp;
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.tuple.ImmutablePair;
@@ -19,22 +44,6 @@ import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 import org.springframework.util.CollectionUtils;
 
-import java.time.LocalDate;
-import java.time.format.DateTimeFormatter;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.stream.Collectors;
-
-import static com.tencent.supersonic.common.pojo.Constants.DAY;
-import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
-import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
-import static com.tencent.supersonic.common.pojo.Constants.MONTH;
-import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
-import static com.tencent.supersonic.common.pojo.Constants.WEEK;
-
 /**
  * tools functions to analyze queryStructReq
  */
@@ -54,7 +63,7 @@ public class SqlGenerateUtils {
     private String internalMetricNameSuffix;
 
     public SqlGenerateUtils(SqlFilterUtils sqlFilterUtils,
-                            DateModeUtils dateModeUtils) {
+            DateModeUtils dateModeUtils) {
         this.sqlFilterUtils = sqlFilterUtils;
         this.dateModeUtils = dateModeUtils;
     }
@@ -97,6 +106,13 @@ public class SqlGenerateUtils {
                 : String.join(",", queryStructCmd.getGroups()) + "," + aggStr;
     }
 
+    public String getSelect(QueryStructReq queryStructCmd, Map<String, String> deriveMetrics) {
+        String aggStr = queryStructCmd.getAggregators().stream().map(a -> getSelectField(a, deriveMetrics))
+                .collect(Collectors.joining(","));
+        return CollectionUtils.isEmpty(queryStructCmd.getGroups()) ? aggStr
+                : String.join(",", queryStructCmd.getGroups()) + "," + aggStr;
+    }
+
     public String getSelectField(final Aggregator agg) {
         if (AggOperatorEnum.COUNT_DISTINCT.equals(agg.getFunc())) {
             return "count(distinct " + agg.getColumn() + " ) AS " + agg.getColumn() + " ";
@@ -109,6 +125,13 @@ public class SqlGenerateUtils {
         ).collect(Collectors.joining(",")) + " ) AS " + agg.getColumn() + " ";
     }
 
+    public String getSelectField(final Aggregator agg, Map<String, String> deriveMetrics) {
+        if (!deriveMetrics.containsKey(agg.getColumn())) {
+            return getSelectField(agg);
+        }
+        return deriveMetrics.get(agg.getColumn());
+    }
+
     public String getGroupBy(QueryStructReq queryStructCmd) {
         if (CollectionUtils.isEmpty(queryStructCmd.getGroups())) {
             return "";
@@ -125,6 +148,19 @@ public class SqlGenerateUtils {
                 .collect(Collectors.joining(","));
     }
 
+    public String getOrderBy(QueryStructReq queryStructCmd, Map<String, String> deriveMetrics) {
+        if (CollectionUtils.isEmpty(queryStructCmd.getOrders())) {
+            return "";
+        }
+        if (!queryStructCmd.getOrders().stream().anyMatch(o -> deriveMetrics.containsKey(o.getColumn()))) {
+            return getOrderBy(queryStructCmd);
+        }
+        return "order by " + queryStructCmd.getOrders().stream()
+                .map(order -> " " + (deriveMetrics.containsKey(order.getColumn()) ? deriveMetrics.get(order.getColumn())
+                        : order.getColumn()) + " " + order.getDirection() + " ")
+                .collect(Collectors.joining(","));
+    }
+
     public String generateWhere(QueryStructReq queryStructReq, ItemDateResp itemDateResp) {
         String whereClauseFromFilter = sqlFilterUtils.getWhereClause(queryStructReq.getOriginalFilter());
         String whereFromDate = getDateWhereClause(queryStructReq.getDateInfo(), itemDateResp);
@@ -132,7 +168,7 @@ public class SqlGenerateUtils {
     }
 
     private String mergeDateWhereClause(QueryStructReq queryStructCmd, String whereClauseFromFilter,
-                                       String whereFromDate) {
+            String whereFromDate) {
         if (Strings.isNotEmpty(whereFromDate) && Strings.isNotEmpty(whereClauseFromFilter)) {
             return String.format("%s AND (%s)", whereFromDate, whereClauseFromFilter);
         } else if (Strings.isEmpty(whereFromDate) && Strings.isNotEmpty(whereClauseFromFilter)) {
@@ -233,4 +269,83 @@ public class SqlGenerateUtils {
         return modelBizName + UNDERLINE + internalMetricNameSuffix;
     }
 
+    public String generateDerivedMetric(final List<MetricResp> metricResps, final Set<String> allFields,
+            final Map<String, Measure> allMeasures, final List<DimensionResp> dimensionResps,
+            final String expression, final MetricDefineType metricDefineType, Set<String> visitedMetric,
+            Set<String> measures,
+            Set<String> dimensions) {
+        Set<String> fields = SqlParserSelectHelper.getColumnFromExpr(expression);
+        if (!CollectionUtils.isEmpty(fields)) {
+            Map<String, String> replace = new HashMap<>();
+            for (String field : fields) {
+                switch (metricDefineType) {
+                    case METRIC:
+                        Optional<MetricResp> metricItem = metricResps.stream()
+                                .filter(m -> m.getBizName().equalsIgnoreCase(field)).findFirst();
+                        if (metricItem.isPresent()) {
+                            if (MetricType.isDerived(metricItem.get().getMetricDefineType(),
+                                    metricItem.get().getTypeParams())) {
+                                if (visitedMetric.contains(field)) {
+                                    break;
+                                }
+                                replace.put(field,
+                                        generateDerivedMetric(metricResps, allFields, allMeasures, dimensionResps,
+                                                getExpr(metricItem.get()), metricItem.get().getMetricDefineType(),
+                                                visitedMetric, measures, dimensions));
+                                visitedMetric.add(field);
+                            } else {
+                                replace.put(field, getExpr(metricItem.get()));
+                            }
+                        }
+                        break;
+                    case MEASURE:
+                        if (allMeasures.containsKey(field)) {
+                            measures.add(field);
+                            replace.put(field, getExpr(allMeasures.get(field)));
+                        }
+                        break;
+                    case FIELD:
+                        if (allFields.contains(field)) {
+                            Optional<DimensionResp> dimensionItem = dimensionResps.stream()
+                                    .filter(d -> d.getBizName().equals(field)).findFirst();
+                            if (dimensionItem.isPresent()) {
+                                dimensions.add(field);
+                            } else {
+                                measures.add(field);
+                            }
+                        }
+                        break;
+                    default:
+                        break;
+
+                }
+            }
+            if (!CollectionUtils.isEmpty(replace)) {
+                return SqlParserReplaceHelper.replaceExpression(expression, replace);
+            }
+        }
+        return expression;
+    }
+
+    public String getExpr(Measure measure) {
+        if (AggOperatorEnum.COUNT_DISTINCT.getOperator().equalsIgnoreCase(measure.getAgg())) {
+            return AggOperatorEnum.COUNT.getOperator() + " ( " + AggOperatorEnum.DISTINCT + " " + measure.getBizName()
+                    + " ) ";
+        }
+        return measure.getAgg() + " ( " + measure.getBizName() + " ) ";
+    }
+
+    public String getExpr(MetricResp metricResp) {
+        if (Objects.isNull(metricResp.getMetricDefineType())) {
+            return metricResp.getTypeParams().getExpr();
+        }
+        if (metricResp.getMetricDefineType().equals(MetricDefineType.METRIC)) {
+            return metricResp.getMetricDefineByMetricParams().getExpr();
+        }
+        if (metricResp.getMetricDefineType().equals(MetricDefineType.FIELD)) {
+            return metricResp.getMetricDefineByFieldParams().getExpr();
+        }
+        // measure add agg function
+        return metricResp.getTypeParams().getExpr();
+    }
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/HeadlessQueryEngineImpl.java

@@ -6,21 +6,21 @@ import com.tencent.supersonic.headless.api.request.ParseSqlReq;
 import com.tencent.supersonic.headless.api.request.QueryStructReq;
 import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
 import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
-import com.tencent.supersonic.headless.core.executor.QueryExecutor;
 import com.tencent.supersonic.headless.core.optimizer.QueryOptimizer;
 import com.tencent.supersonic.headless.core.parser.QueryParser;
 import com.tencent.supersonic.headless.core.parser.calcite.s2sql.HeadlessModel;
 import com.tencent.supersonic.headless.core.pojo.QueryStatement;
 import com.tencent.supersonic.headless.core.utils.ComponentFactory;
+import com.tencent.supersonic.headless.core.executor.QueryExecutor;
 import com.tencent.supersonic.headless.server.manager.HeadlessSchemaManager;
 import com.tencent.supersonic.headless.server.service.Catalog;
 import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine;
 import com.tencent.supersonic.headless.server.utils.QueryStructUtils;
 import com.tencent.supersonic.headless.server.utils.QueryUtils;
-import java.util.List;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Component;
 import org.springframework.util.CollectionUtils;
+import java.util.List;
 
 @Slf4j
 @Component
@@ -82,22 +82,20 @@ public class HeadlessQueryEngineImpl implements HeadlessQueryEngine {
     }
 
     @Override
-    public QueryStatement physicalSql(QueryStructReq queryStructCmd, ParseSqlReq sqlCommend) throws Exception {
+    public QueryStatement physicalSql(QueryStructReq queryStructCmd, ParseSqlReq sqlCommend) {
         QueryStatement queryStatement = new QueryStatement();
-        queryStatement.setSql(sqlCommend.getSql());
         queryStatement.setQueryStructReq(queryStructCmd);
         queryStatement.setParseSqlReq(sqlCommend);
+        queryStatement.setSql(sqlCommend.getSql());
         queryStatement.setIsS2SQL(true);
-        queryStatement.setHeadlessModel(getHeadLessModel(queryStatement));
         return optimize(queryStructCmd, queryParser.parser(sqlCommend, queryStatement));
     }
 
-    public QueryStatement physicalSql(QueryStructReq queryStructCmd, MetricQueryReq metricCommand) throws Exception {
+    public QueryStatement physicalSql(QueryStructReq queryStructCmd, MetricQueryReq metricCommand) {
         QueryStatement queryStatement = new QueryStatement();
         queryStatement.setQueryStructReq(queryStructCmd);
         queryStatement.setMetricReq(metricCommand);
         queryStatement.setIsS2SQL(false);
-        queryStatement.setHeadlessModel(getHeadLessModel(queryStatement));
         return queryParser.parser(queryStatement);
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java

@@ -11,6 +11,8 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelp
 import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
 import com.tencent.supersonic.headless.api.enums.AggOption;
 import com.tencent.supersonic.headless.api.enums.EngineType;
+import com.tencent.supersonic.headless.api.enums.MetricType;
+import com.tencent.supersonic.headless.api.pojo.Measure;
 import com.tencent.supersonic.headless.api.pojo.MetricTable;
 import com.tencent.supersonic.headless.api.pojo.SchemaItem;
 import com.tencent.supersonic.headless.api.request.ParseSqlReq;
@@ -18,14 +20,18 @@ import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
 import com.tencent.supersonic.headless.api.request.QueryStructReq;
 import com.tencent.supersonic.headless.api.request.SqlExecuteReq;
 import com.tencent.supersonic.headless.api.response.DatabaseResp;
+import com.tencent.supersonic.headless.api.response.DimensionResp;
+import com.tencent.supersonic.headless.api.response.MetricResp;
 import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
 import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor;
 import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory;
 import com.tencent.supersonic.headless.core.pojo.QueryStatement;
 import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
+import com.tencent.supersonic.headless.server.pojo.MetaFilter;
 import com.tencent.supersonic.headless.server.service.Catalog;
 import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -119,7 +125,9 @@ public class QueryReqConverter {
             result.setSupportWith(false);
             result.setWithAlias(false);
         }
-        //5.physicalSql by ParseSqlReq
+        //5. do deriveMetric
+        generateDerivedMetric(queryS2SQLReq.getModelIds(), modelSchemaResps, result);
+        //6.physicalSql by ParseSqlReq
         queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(queryS2SQLReq.getSql()));
         queryStructReq.setModelIds(new HashSet<>(queryS2SQLReq.getModelIds()));
         queryStructReq.setQueryType(getQueryType(aggOption));
@@ -240,4 +248,70 @@ public class QueryReqConverter {
         return queryType;
     }
 
+    private void generateDerivedMetric(List<Long> modelIds, List<ModelSchemaResp> modelSchemaResps,
+            ParseSqlReq parseSqlReq) {
+        String sql = parseSqlReq.getSql();
+        for (MetricTable metricTable : parseSqlReq.getTables()) {
+            List<String> measures = new ArrayList<>();
+            Map<String, String> replaces = new HashMap<>();
+            generateDerivedMetric(modelIds, modelSchemaResps, metricTable.getMetrics(), metricTable.getDimensions(),
+                    measures, replaces);
+            if (!CollectionUtils.isEmpty(replaces)) {
+                // metricTable sql use measures replace metric
+                sql = SqlParserReplaceHelper.replaceSqlByExpression(sql, replaces);
+                metricTable.setAggOption(AggOption.NATIVE);
+            }
+            // metricTable  use measures replace metric
+            if (!CollectionUtils.isEmpty(measures)) {
+                metricTable.setMetrics(measures);
+            }
+        }
+        parseSqlReq.setSql(sql);
+    }
+
+    private void generateDerivedMetric(List<Long> modelIds, List<ModelSchemaResp> modelSchemaResps,
+            List<String> metrics, List<String> dimensions,
+            List<String> measures, Map<String, String> replaces) {
+        MetaFilter metaFilter = new MetaFilter();
+        metaFilter.setModelIds(modelIds);
+        List<MetricResp> metricResps = catalog.getMetrics(metaFilter);
+        List<DimensionResp> dimensionResps = catalog.getDimensions(metaFilter);
+        // check metrics has derived
+        if (!metricResps.stream()
+                .anyMatch(m -> metrics.contains(m.getBizName()) && MetricType.isDerived(m.getMetricDefineType(),
+                        m.getTypeParams()))) {
+            return;
+        }
+        Set<String> allFields = new HashSet<>();
+        Map<String, Measure> allMeasures = new HashMap<>();
+        modelSchemaResps.stream().forEach(modelSchemaResp -> {
+            allFields.addAll(modelSchemaResp.getFieldList());
+            if (Objects.nonNull(modelSchemaResp.getModelDetail().getMeasures())) {
+                modelSchemaResp.getModelDetail().getMeasures().stream()
+                        .forEach(mm -> allMeasures.put(mm.getBizName(), mm));
+            }
+        });
+
+        Set<String> deriveDimension = new HashSet<>();
+        Set<String> deriveMetric = new HashSet<>();
+        Set<String> visitedMetric = new HashSet<>();
+        if (!CollectionUtils.isEmpty(metricResps)) {
+            for (MetricResp metricResp : metricResps) {
+                if (metrics.contains(metricResp.getBizName())) {
+                    if (MetricType.isDerived(metricResp.getMetricDefineType(), metricResp.getTypeParams())) {
+                        String expr = sqlGenerateUtils.generateDerivedMetric(metricResps, allFields, allMeasures,
+                                dimensionResps,
+                                sqlGenerateUtils.getExpr(metricResp), metricResp.getMetricDefineType(), visitedMetric,
+                                deriveMetric, deriveDimension);
+                        replaces.put(metricResp.getBizName(), expr);
+                    } else {
+                        measures.add(metricResp.getBizName());
+                    }
+                }
+            }
+        }
+        measures.addAll(deriveMetric);
+        dimensions.addAll(deriveDimension);
+    }
+
 }