(improvement)(headless) adapter for derived metrics (#646)

common/src/main/java/com/tencent/supersonic/common/pojo/enums/AggOperatorEnum.java

@@ -10,6 +10,7 @@ public enum AggOperatorEnum {
 
     SUM("SUM"),
 
+    COUNT("COUNT"),
     COUNT_DISTINCT("COUNT_DISTINCT"),
     DISTINCT("DISTINCT"),
 

common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/ExpressionReplaceVisitor.java

@@ -0,0 +1,79 @@
+package com.tencent.supersonic.common.util.jsqlparser;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import net.sf.jsqlparser.expression.BinaryExpression;
+import net.sf.jsqlparser.expression.Expression;
+import net.sf.jsqlparser.expression.ExpressionVisitorAdapter;
+import net.sf.jsqlparser.expression.Function;
+import net.sf.jsqlparser.schema.Column;
+
+public class ExpressionReplaceVisitor extends ExpressionVisitorAdapter {
+
+    private Map<String, String> fieldExprMap;
+
+
+    public ExpressionReplaceVisitor(Map<String, String> fieldExprMap) {
+        this.fieldExprMap = fieldExprMap;
+    }
+
+    protected void visitBinaryExpression(BinaryExpression expr) {
+        Expression left = expr.getLeftExpression();
+        Expression right = expr.getRightExpression();
+        Boolean leftVisited = false;
+        Boolean rightVisited = false;
+        if (left instanceof Function) {
+            Function leftFunc = (Function) left;
+            if (visitFunction(leftFunc)) {
+                leftVisited = true;
+            }
+        }
+        if (right instanceof Function) {
+            Function function = (Function) right;
+            if (visitFunction(function)) {
+                rightVisited = true;
+            }
+        }
+        if (left instanceof Column) {
+            Expression expression = QueryExpressionReplaceVisitor.getExpression(
+                    QueryExpressionReplaceVisitor.getReplaceExpr((Column) left, fieldExprMap));
+            if (Objects.nonNull(expression)) {
+                expr.setLeftExpression(expression);
+                leftVisited = true;
+            }
+        }
+        if (right instanceof Column) {
+            Expression expression = QueryExpressionReplaceVisitor.getExpression(
+                    QueryExpressionReplaceVisitor.getReplaceExpr((Column) right, fieldExprMap));
+            if (Objects.nonNull(expression)) {
+                expr.setRightExpression(expression);
+                rightVisited = true;
+            }
+        }
+        if (!leftVisited) {
+            expr.getLeftExpression().accept(this);
+        }
+        if (!rightVisited) {
+            expr.getRightExpression().accept(this);
+        }
+    }
+
+    private boolean visitFunction(Function function) {
+        if (function.getParameters().getExpressions().get(0) instanceof Column) {
+            Expression expression = QueryExpressionReplaceVisitor.getExpression(
+                    QueryExpressionReplaceVisitor.getReplaceExpr(function, fieldExprMap));
+            if (Objects.nonNull(expression)) {
+                List<Expression> expressions = new ArrayList<>();
+                expressions.add(expression);
+                for (int i = 1; i < function.getParameters().getExpressions().size(); i++) {
+                    expressions.add(function.getParameters().getExpressions().get(i));
+                }
+                function.getParameters().setExpressions(expressions);
+                return true;
+            }
+        }
+        return false;
+    }
+}

common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/QueryExpressionReplaceVisitor.java

@@ -0,0 +1,106 @@
+package com.tencent.supersonic.common.util.jsqlparser;
+
+import java.util.Map;
+import java.util.Objects;
+import net.sf.jsqlparser.expression.BinaryExpression;
+import net.sf.jsqlparser.expression.Expression;
+import net.sf.jsqlparser.expression.ExpressionVisitorAdapter;
+import net.sf.jsqlparser.expression.Function;
+import net.sf.jsqlparser.parser.CCJSqlParserUtil;
+import net.sf.jsqlparser.schema.Column;
+import net.sf.jsqlparser.statement.select.SelectExpressionItem;
+
+public class QueryExpressionReplaceVisitor extends ExpressionVisitorAdapter {
+
+    private Map<String, String> fieldExprMap;
+
+    public QueryExpressionReplaceVisitor(Map<String, String> fieldExprMap) {
+        this.fieldExprMap = fieldExprMap;
+    }
+
+    protected void visitBinaryExpression(BinaryExpression expr) {
+        Expression left = expr.getLeftExpression();
+        String toReplace = "";
+        if (left instanceof Function) {
+            Function leftFunc = (Function) left;
+            if (leftFunc.getParameters().getExpressions().get(0) instanceof Column) {
+                toReplace = getReplaceExpr(leftFunc, fieldExprMap);
+            }
+        }
+        if (left instanceof Column) {
+            toReplace = getReplaceExpr((Column) left, fieldExprMap);
+        }
+        if (!toReplace.isEmpty()) {
+            Expression expression = getExpression(toReplace);
+            if (Objects.nonNull(expression)) {
+                expr.setLeftExpression(expression);
+                return;
+            }
+        }
+        expr.getLeftExpression().accept(this);
+        expr.getRightExpression().accept(this);
+
+    }
+
+    public void visit(SelectExpressionItem selectExpressionItem) {
+
+        Expression expression = selectExpressionItem.getExpression();
+        String toReplace = "";
+        if (expression instanceof Function) {
+            Function leftFunc = (Function) expression;
+            if (leftFunc.getParameters().getExpressions().get(0) instanceof Column) {
+                toReplace = getReplaceExpr(leftFunc, fieldExprMap);
+            }
+
+        }
+        if (expression instanceof Column) {
+            toReplace = getReplaceExpr((Column) expression, fieldExprMap);
+        }
+        if (!toReplace.isEmpty()) {
+            Expression toReplaceExpr = getExpression(toReplace);
+            if (Objects.nonNull(toReplaceExpr)) {
+                selectExpressionItem.setExpression(toReplaceExpr);
+            }
+        }
+        //selectExpressionItem.getExpression().accept(this);
+    }
+
+    public static Expression replace(Expression expression, Map<String, String> fieldExprMap) {
+        String toReplace = "";
+        if (expression instanceof Function) {
+            toReplace = getReplaceExpr((Function) expression, fieldExprMap);
+        }
+        if (expression instanceof Column) {
+            toReplace = getReplaceExpr((Column) expression, fieldExprMap);
+        }
+        if (!toReplace.isEmpty()) {
+            Expression replace = getExpression(toReplace);
+            if (Objects.nonNull(replace)) {
+                return replace;
+            }
+        }
+        return expression;
+    }
+
+    public static Expression getExpression(String expr) {
+        if (expr.isEmpty()) {
+            return null;
+        }
+        try {
+            Expression expression = CCJSqlParserUtil.parseExpression(expr);
+            return expression;
+        } catch (Exception e) {
+            return null;
+        }
+    }
+
+    public static String getReplaceExpr(Column column, Map<String, String> fieldExprMap) {
+        return fieldExprMap.containsKey(column.getColumnName()) ? fieldExprMap.get(column.getColumnName()) : "";
+    }
+
+    public static String getReplaceExpr(Function function, Map<String, String> fieldExprMap) {
+        Column column = (Column) function.getParameters().getExpressions().get(0);
+        return getReplaceExpr(column, fieldExprMap);
+    }
+
+}

common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserReplaceHelper.java

@@ -2,6 +2,11 @@ package com.tencent.supersonic.common.util.jsqlparser;
 
 import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
 import com.tencent.supersonic.common.util.StringUtil;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
 import lombok.extern.slf4j.Slf4j;
 import net.sf.jsqlparser.JSQLParserException;
 import net.sf.jsqlparser.expression.Alias;
@@ -32,11 +37,6 @@ import net.sf.jsqlparser.statement.select.SubSelect;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.tuple.Pair;
 import org.springframework.util.CollectionUtils;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
 
 /**
  * Sql Parser replace Helper
@@ -120,7 +120,7 @@ public class SqlParserReplaceHelper {
     }
 
     public static String replaceValue(String sql, Map<String, Map<String, String>> filedNameToValueMap,
-                                      boolean exactReplace) {
+            boolean exactReplace) {
         Select selectStatement = SqlParserSelectHelper.getSelect(sql);
         SelectBody selectBody = selectStatement.getSelectBody();
         if (!(selectBody instanceof PlainSelect)) {
@@ -196,7 +196,7 @@ public class SqlParserReplaceHelper {
     }
 
     private static void replaceFieldsInPlainOneSelect(Map<String, String> fieldNameMap, boolean exactReplace,
-                                                      PlainSelect plainSelect) {
+            PlainSelect plainSelect) {
         //1. replace where fields
         Expression where = plainSelect.getWhere();
         FieldReplaceVisitor visitor = new FieldReplaceVisitor(fieldNameMap, exactReplace);
@@ -351,7 +351,7 @@ public class SqlParserReplaceHelper {
     }
 
     private static void replaceOrderByFunction(Map<String, String> functionMap,
-                                               List<OrderByElement> orderByElementList) {
+            List<OrderByElement> orderByElementList) {
         if (Objects.isNull(orderByElementList)) {
             return;
         }
@@ -499,5 +499,66 @@ public class SqlParserReplaceHelper {
             return expression;
         }
     }
+
+    public static String replaceExpression(String expr, Map<String, String> replace) {
+        Expression expression = QueryExpressionReplaceVisitor.getExpression(expr);
+        if (Objects.nonNull(expression)) {
+            if (expression instanceof Column && replace.containsKey(expr)) {
+                return replace.get(expr);
+            }
+            ExpressionReplaceVisitor expressionReplaceVisitor = new ExpressionReplaceVisitor(replace);
+            expression.accept(expressionReplaceVisitor);
+            return expression.toString();
+        }
+        return expr;
+    }
+
+    public static String replaceSqlByExpression(String sql, Map<String, String> replace) {
+        Select selectStatement = SqlParserSelectHelper.getSelect(sql);
+        SelectBody selectBody = selectStatement.getSelectBody();
+        List<PlainSelect> plainSelectList = new ArrayList<>();
+        if (selectBody instanceof PlainSelect) {
+            plainSelectList.add((PlainSelect) selectBody);
+        } else if (selectBody instanceof SetOperationList) {
+            SetOperationList setOperationList = (SetOperationList) selectBody;
+            if (!CollectionUtils.isEmpty(setOperationList.getSelects())) {
+                setOperationList.getSelects().forEach(subSelectBody -> {
+                    PlainSelect subPlainSelect = (PlainSelect) subSelectBody;
+                    plainSelectList.add(subPlainSelect);
+                });
+            }
+        } else {
+            return sql;
+        }
+        List<PlainSelect> plainSelects = SqlParserSelectHelper.getPlainSelects(plainSelectList);
+        for (PlainSelect plainSelect : plainSelects) {
+            replacePlainSelectByExpr(plainSelect, replace);
+        }
+        return selectStatement.toString();
+    }
+
+    private static void replacePlainSelectByExpr(PlainSelect plainSelect, Map<String, String> replace) {
+        QueryExpressionReplaceVisitor expressionReplaceVisitor = new QueryExpressionReplaceVisitor(replace);
+        for (SelectItem selectItem : plainSelect.getSelectItems()) {
+            selectItem.accept(expressionReplaceVisitor);
+        }
+        Expression having = plainSelect.getHaving();
+        if (Objects.nonNull(having)) {
+            having.accept(expressionReplaceVisitor);
+        }
+
+        Expression where = plainSelect.getWhere();
+        if (Objects.nonNull(where)) {
+            where.accept(expressionReplaceVisitor);
+        }
+
+        List<OrderByElement> orderByElements = plainSelect.getOrderByElements();
+        if (!CollectionUtils.isEmpty(orderByElements)) {
+            for (OrderByElement orderByElement : orderByElements) {
+                orderByElement.setExpression(
+                        QueryExpressionReplaceVisitor.replace(orderByElement.getExpression(), replace));
+            }
+        }
+    }
 }
 

common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelper.java

@@ -6,9 +6,9 @@ import java.util.List;
 import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
-
 import lombok.extern.slf4j.Slf4j;
 import net.sf.jsqlparser.JSQLParserException;
+import net.sf.jsqlparser.expression.BinaryExpression;
 import net.sf.jsqlparser.expression.Expression;
 import net.sf.jsqlparser.expression.ExpressionVisitorAdapter;
 import net.sf.jsqlparser.expression.Function;
@@ -32,8 +32,8 @@ import net.sf.jsqlparser.statement.select.SelectBody;
 import net.sf.jsqlparser.statement.select.SelectExpressionItem;
 import net.sf.jsqlparser.statement.select.SelectItem;
 import net.sf.jsqlparser.statement.select.SelectVisitorAdapter;
-import net.sf.jsqlparser.statement.select.SubSelect;
 import net.sf.jsqlparser.statement.select.SetOperationList;
+import net.sf.jsqlparser.statement.select.SubSelect;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.tuple.ImmutablePair;
 import org.springframework.util.CollectionUtils;
@@ -480,5 +480,31 @@ public class SqlParserSelectHelper {
         }
         return selectStatement.toString();
     }
+
+    public static Set<String> getColumnFromExpr(String expr) {
+        Expression expression = QueryExpressionReplaceVisitor.getExpression(expr);
+        Set<String> columns = new HashSet<>();
+        if (Objects.nonNull(expression)) {
+            getColumnFromExpr(expression, columns);
+        }
+        return columns;
+    }
+
+    public static void getColumnFromExpr(Expression expression, Set<String> columns) {
+        if (expression instanceof Column) {
+            columns.add(((Column) expression).getColumnName());
+        }
+        if (expression instanceof Function) {
+            List<Expression> expressionList = ((Function) expression).getParameters().getExpressions();
+            for (Expression expr : expressionList) {
+                getColumnFromExpr(expr, columns);
+            }
+        }
+        if (expression instanceof BinaryExpression) {
+            BinaryExpression expr = (BinaryExpression) expression;
+            getColumnFromExpr(expr.getLeftExpression(), columns);
+            getColumnFromExpr(expr.getRightExpression(), columns);
+        }
+    }
 }
 

headless/api/src/main/java/com/tencent/supersonic/headless/api/enums/MetricType.java

@@ -1,9 +1,44 @@
 package com.tencent.supersonic.headless.api.enums;
 
 
+import com.tencent.supersonic.headless.api.pojo.MeasureParam;
+import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
+import java.util.List;
+import java.util.Objects;
+
 public enum MetricType {
 
     ATOMIC,
-    DERIVED
+    DERIVED;
 
+    public static MetricType of(String src) {
+        for (MetricType metricType : MetricType.values()) {
+            if (Objects.nonNull(src) && src.equalsIgnoreCase(metricType.name())) {
+                return metricType;
+            }
+        }
+        return null;
+    }
+
+    public static Boolean isDerived(String src) {
+        MetricType metricType = of(src);
+        return Objects.nonNull(metricType) && metricType.equals(DERIVED);
+    }
+
+    public static Boolean isDerived(MetricDefineType metricDefineType, MetricDefineByMeasureParams typeParams) {
+        if (MetricDefineType.METRIC.equals(metricDefineType)) {
+            return true;
+        }
+        if (MetricDefineType.MEASURE.equals(metricDefineType)) {
+            List<MeasureParam> measures = typeParams.getMeasures();
+            if (measures.size() > 1) {
+                return true;
+            }
+            if (measures.size() == 1 && measures.get(0).getBizName()
+                    .equalsIgnoreCase(typeParams.getExpr())) {
+                return false;
+            }
+        }
+        return false;
+    }
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/request/MetricReq.java

@@ -3,12 +3,10 @@ package com.tencent.supersonic.headless.api.request;
 import com.alibaba.fastjson.JSONObject;
 import com.tencent.supersonic.headless.api.enums.MetricDefineType;
 import com.tencent.supersonic.headless.api.enums.MetricType;
-import com.tencent.supersonic.headless.api.pojo.MeasureParam;
 import com.tencent.supersonic.headless.api.pojo.MetricDefineByFieldParams;
 import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
 import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams;
 import lombok.Data;
-import java.util.List;
 
 @Data
 public class MetricReq extends MetricBaseReq {
@@ -30,20 +28,7 @@ public class MetricReq extends MetricBaseReq {
     }
 
     public MetricType getMetricType() {
-        if (MetricDefineType.METRIC.equals(metricDefineType)) {
+        return MetricType.isDerived(metricDefineType, typeParams) ? MetricType.DERIVED : MetricType.ATOMIC;
-            return MetricType.DERIVED;
-        }
-        if (MetricDefineType.MEASURE.equals(metricDefineType)) {
-            List<MeasureParam> measures = typeParams.getMeasures();
-            if (measures.size() > 1) {
-                return MetricType.DERIVED;
-            }
-            if (measures.size() == 1 && measures.get(0).getBizName()
-                    .equalsIgnoreCase(typeParams.getExpr())) {
-                return MetricType.ATOMIC;
-            }
-        }
-        return MetricType.ATOMIC;
     }
 
 }

headless/api/src/main/java/com/tencent/supersonic/headless/api/response/MetricResp.java

@@ -10,16 +10,15 @@ import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
 import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams;
 import com.tencent.supersonic.headless.api.pojo.RelateDimension;
 import com.tencent.supersonic.headless.api.pojo.SchemaItem;
-import lombok.Data;
-import lombok.ToString;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
-
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
+import lombok.Data;
+import lombok.ToString;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
 
 
 @Data

headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java

@@ -1,5 +1,12 @@
 package com.tencent.supersonic.headless.core.utils;
 
+import static com.tencent.supersonic.common.pojo.Constants.DAY;
+import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
+import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
+import static com.tencent.supersonic.common.pojo.Constants.MONTH;
+import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
+import static com.tencent.supersonic.common.pojo.Constants.WEEK;
+
 import com.tencent.supersonic.common.pojo.Aggregator;
 import com.tencent.supersonic.common.pojo.DateConf;
 import com.tencent.supersonic.common.pojo.ItemDateResp;
@@ -8,8 +15,26 @@ import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum;
 import com.tencent.supersonic.common.util.DateModeUtils;
 import com.tencent.supersonic.common.util.SqlFilterUtils;
 import com.tencent.supersonic.common.util.StringUtil;
+import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
+import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
 import com.tencent.supersonic.headless.api.enums.EngineType;
+import com.tencent.supersonic.headless.api.enums.MetricDefineType;
+import com.tencent.supersonic.headless.api.enums.MetricType;
+import com.tencent.supersonic.headless.api.pojo.Measure;
 import com.tencent.supersonic.headless.api.request.QueryStructReq;
+import com.tencent.supersonic.headless.api.response.DimensionResp;
+import com.tencent.supersonic.headless.api.response.MetricResp;
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.tuple.ImmutablePair;
@@ -19,22 +44,6 @@ import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 import org.springframework.util.CollectionUtils;
 
-import java.time.LocalDate;
-import java.time.format.DateTimeFormatter;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.stream.Collectors;
-
-import static com.tencent.supersonic.common.pojo.Constants.DAY;
-import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT;
-import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE;
-import static com.tencent.supersonic.common.pojo.Constants.MONTH;
-import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE;
-import static com.tencent.supersonic.common.pojo.Constants.WEEK;
-
 /**
  * tools functions to analyze queryStructReq
  */
@@ -54,7 +63,7 @@ public class SqlGenerateUtils {
     private String internalMetricNameSuffix;
 
     public SqlGenerateUtils(SqlFilterUtils sqlFilterUtils,
-                            DateModeUtils dateModeUtils) {
+            DateModeUtils dateModeUtils) {
         this.sqlFilterUtils = sqlFilterUtils;
         this.dateModeUtils = dateModeUtils;
     }
@@ -97,6 +106,13 @@ public class SqlGenerateUtils {
                 : String.join(",", queryStructCmd.getGroups()) + "," + aggStr;
     }
 
+    public String getSelect(QueryStructReq queryStructCmd, Map<String, String> deriveMetrics) {
+        String aggStr = queryStructCmd.getAggregators().stream().map(a -> getSelectField(a, deriveMetrics))
+                .collect(Collectors.joining(","));
+        return CollectionUtils.isEmpty(queryStructCmd.getGroups()) ? aggStr
+                : String.join(",", queryStructCmd.getGroups()) + "," + aggStr;
+    }
+
     public String getSelectField(final Aggregator agg) {
         if (AggOperatorEnum.COUNT_DISTINCT.equals(agg.getFunc())) {
             return "count(distinct " + agg.getColumn() + " ) AS " + agg.getColumn() + " ";
@@ -109,6 +125,13 @@ public class SqlGenerateUtils {
         ).collect(Collectors.joining(",")) + " ) AS " + agg.getColumn() + " ";
     }
 
+    public String getSelectField(final Aggregator agg, Map<String, String> deriveMetrics) {
+        if (!deriveMetrics.containsKey(agg.getColumn())) {
+            return getSelectField(agg);
+        }
+        return deriveMetrics.get(agg.getColumn());
+    }
+
     public String getGroupBy(QueryStructReq queryStructCmd) {
         if (CollectionUtils.isEmpty(queryStructCmd.getGroups())) {
             return "";
@@ -125,6 +148,19 @@ public class SqlGenerateUtils {
                 .collect(Collectors.joining(","));
     }
 
+    public String getOrderBy(QueryStructReq queryStructCmd, Map<String, String> deriveMetrics) {
+        if (CollectionUtils.isEmpty(queryStructCmd.getOrders())) {
+            return "";
+        }
+        if (!queryStructCmd.getOrders().stream().anyMatch(o -> deriveMetrics.containsKey(o.getColumn()))) {
+            return getOrderBy(queryStructCmd);
+        }
+        return "order by " + queryStructCmd.getOrders().stream()
+                .map(order -> " " + (deriveMetrics.containsKey(order.getColumn()) ? deriveMetrics.get(order.getColumn())
+                        : order.getColumn()) + " " + order.getDirection() + " ")
+                .collect(Collectors.joining(","));
+    }
+
     public String generateWhere(QueryStructReq queryStructReq, ItemDateResp itemDateResp) {
         String whereClauseFromFilter = sqlFilterUtils.getWhereClause(queryStructReq.getOriginalFilter());
         String whereFromDate = getDateWhereClause(queryStructReq.getDateInfo(), itemDateResp);
@@ -132,7 +168,7 @@ public class SqlGenerateUtils {
     }
 
     private String mergeDateWhereClause(QueryStructReq queryStructCmd, String whereClauseFromFilter,
-                                       String whereFromDate) {
+            String whereFromDate) {
         if (Strings.isNotEmpty(whereFromDate) && Strings.isNotEmpty(whereClauseFromFilter)) {
             return String.format("%s AND (%s)", whereFromDate, whereClauseFromFilter);
         } else if (Strings.isEmpty(whereFromDate) && Strings.isNotEmpty(whereClauseFromFilter)) {
@@ -233,4 +269,83 @@ public class SqlGenerateUtils {
         return modelBizName + UNDERLINE + internalMetricNameSuffix;
     }
 
+    public String generateDerivedMetric(final List<MetricResp> metricResps, final Set<String> allFields,
+            final Map<String, Measure> allMeasures, final List<DimensionResp> dimensionResps,
+            final String expression, final MetricDefineType metricDefineType, Set<String> visitedMetric,
+            Set<String> measures,
+            Set<String> dimensions) {
+        Set<String> fields = SqlParserSelectHelper.getColumnFromExpr(expression);
+        if (!CollectionUtils.isEmpty(fields)) {
+            Map<String, String> replace = new HashMap<>();
+            for (String field : fields) {
+                switch (metricDefineType) {
+                    case METRIC:
+                        Optional<MetricResp> metricItem = metricResps.stream()
+                                .filter(m -> m.getBizName().equalsIgnoreCase(field)).findFirst();
+                        if (metricItem.isPresent()) {
+                            if (MetricType.isDerived(metricItem.get().getMetricDefineType(),
+                                    metricItem.get().getTypeParams())) {
+                                if (visitedMetric.contains(field)) {
+                                    break;
+                                }
+                                replace.put(field,
+                                        generateDerivedMetric(metricResps, allFields, allMeasures, dimensionResps,
+                                                getExpr(metricItem.get()), metricItem.get().getMetricDefineType(),
+                                                visitedMetric, measures, dimensions));
+                                visitedMetric.add(field);
+                            } else {
+                                replace.put(field, getExpr(metricItem.get()));
+                            }
+                        }
+                        break;
+                    case MEASURE:
+                        if (allMeasures.containsKey(field)) {
+                            measures.add(field);
+                            replace.put(field, getExpr(allMeasures.get(field)));
+                        }
+                        break;
+                    case FIELD:
+                        if (allFields.contains(field)) {
+                            Optional<DimensionResp> dimensionItem = dimensionResps.stream()
+                                    .filter(d -> d.getBizName().equals(field)).findFirst();
+                            if (dimensionItem.isPresent()) {
+                                dimensions.add(field);
+                            } else {
+                                measures.add(field);
+                            }
+                        }
+                        break;
+                    default:
+                        break;
+
+                }
+            }
+            if (!CollectionUtils.isEmpty(replace)) {
+                return SqlParserReplaceHelper.replaceExpression(expression, replace);
+            }
+        }
+        return expression;
+    }
+
+    public String getExpr(Measure measure) {
+        if (AggOperatorEnum.COUNT_DISTINCT.getOperator().equalsIgnoreCase(measure.getAgg())) {
+            return AggOperatorEnum.COUNT.getOperator() + " ( " + AggOperatorEnum.DISTINCT + " " + measure.getBizName()
+                    + " ) ";
+        }
+        return measure.getAgg() + " ( " + measure.getBizName() + " ) ";
+    }
+
+    public String getExpr(MetricResp metricResp) {
+        if (Objects.isNull(metricResp.getMetricDefineType())) {
+            return metricResp.getTypeParams().getExpr();
+        }
+        if (metricResp.getMetricDefineType().equals(MetricDefineType.METRIC)) {
+            return metricResp.getMetricDefineByMetricParams().getExpr();
+        }
+        if (metricResp.getMetricDefineType().equals(MetricDefineType.FIELD)) {
+            return metricResp.getMetricDefineByFieldParams().getExpr();
+        }
+        // measure add agg function
+        return metricResp.getTypeParams().getExpr();
+    }
 }

headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/HeadlessQueryEngineImpl.java

@@ -6,21 +6,21 @@ import com.tencent.supersonic.headless.api.request.ParseSqlReq;
 import com.tencent.supersonic.headless.api.request.QueryStructReq;
 import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
 import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
-import com.tencent.supersonic.headless.core.executor.QueryExecutor;
 import com.tencent.supersonic.headless.core.optimizer.QueryOptimizer;
 import com.tencent.supersonic.headless.core.parser.QueryParser;
 import com.tencent.supersonic.headless.core.parser.calcite.s2sql.HeadlessModel;
 import com.tencent.supersonic.headless.core.pojo.QueryStatement;
 import com.tencent.supersonic.headless.core.utils.ComponentFactory;
+import com.tencent.supersonic.headless.core.executor.QueryExecutor;
 import com.tencent.supersonic.headless.server.manager.HeadlessSchemaManager;
 import com.tencent.supersonic.headless.server.service.Catalog;
 import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine;
 import com.tencent.supersonic.headless.server.utils.QueryStructUtils;
 import com.tencent.supersonic.headless.server.utils.QueryUtils;
-import java.util.List;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Component;
 import org.springframework.util.CollectionUtils;
+import java.util.List;
 
 @Slf4j
 @Component
@@ -82,22 +82,20 @@ public class HeadlessQueryEngineImpl implements HeadlessQueryEngine {
     }
 
     @Override
-    public QueryStatement physicalSql(QueryStructReq queryStructCmd, ParseSqlReq sqlCommend) throws Exception {
+    public QueryStatement physicalSql(QueryStructReq queryStructCmd, ParseSqlReq sqlCommend) {
         QueryStatement queryStatement = new QueryStatement();
-        queryStatement.setSql(sqlCommend.getSql());
         queryStatement.setQueryStructReq(queryStructCmd);
         queryStatement.setParseSqlReq(sqlCommend);
+        queryStatement.setSql(sqlCommend.getSql());
         queryStatement.setIsS2SQL(true);
-        queryStatement.setHeadlessModel(getHeadLessModel(queryStatement));
         return optimize(queryStructCmd, queryParser.parser(sqlCommend, queryStatement));
     }
 
-    public QueryStatement physicalSql(QueryStructReq queryStructCmd, MetricQueryReq metricCommand) throws Exception {
+    public QueryStatement physicalSql(QueryStructReq queryStructCmd, MetricQueryReq metricCommand) {
         QueryStatement queryStatement = new QueryStatement();
         queryStatement.setQueryStructReq(queryStructCmd);
         queryStatement.setMetricReq(metricCommand);
         queryStatement.setIsS2SQL(false);
-        queryStatement.setHeadlessModel(getHeadLessModel(queryStatement));
         return queryParser.parser(queryStatement);
     }
 

headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java

@@ -11,6 +11,8 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelp
 import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
 import com.tencent.supersonic.headless.api.enums.AggOption;
 import com.tencent.supersonic.headless.api.enums.EngineType;
+import com.tencent.supersonic.headless.api.enums.MetricType;
+import com.tencent.supersonic.headless.api.pojo.Measure;
 import com.tencent.supersonic.headless.api.pojo.MetricTable;
 import com.tencent.supersonic.headless.api.pojo.SchemaItem;
 import com.tencent.supersonic.headless.api.request.ParseSqlReq;
@@ -18,14 +20,18 @@ import com.tencent.supersonic.headless.api.request.QueryS2SQLReq;
 import com.tencent.supersonic.headless.api.request.QueryStructReq;
 import com.tencent.supersonic.headless.api.request.SqlExecuteReq;
 import com.tencent.supersonic.headless.api.response.DatabaseResp;
+import com.tencent.supersonic.headless.api.response.DimensionResp;
+import com.tencent.supersonic.headless.api.response.MetricResp;
 import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
 import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor;
 import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory;
 import com.tencent.supersonic.headless.core.pojo.QueryStatement;
 import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
+import com.tencent.supersonic.headless.server.pojo.MetaFilter;
 import com.tencent.supersonic.headless.server.service.Catalog;
 import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -119,7 +125,9 @@ public class QueryReqConverter {
             result.setSupportWith(false);
             result.setWithAlias(false);
         }
-        //5.physicalSql by ParseSqlReq
+        //5. do deriveMetric
+        generateDerivedMetric(queryS2SQLReq.getModelIds(), modelSchemaResps, result);
+        //6.physicalSql by ParseSqlReq
         queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(queryS2SQLReq.getSql()));
         queryStructReq.setModelIds(new HashSet<>(queryS2SQLReq.getModelIds()));
         queryStructReq.setQueryType(getQueryType(aggOption));
@@ -240,4 +248,70 @@ public class QueryReqConverter {
         return queryType;
     }
 
+    private void generateDerivedMetric(List<Long> modelIds, List<ModelSchemaResp> modelSchemaResps,
+            ParseSqlReq parseSqlReq) {
+        String sql = parseSqlReq.getSql();
+        for (MetricTable metricTable : parseSqlReq.getTables()) {
+            List<String> measures = new ArrayList<>();
+            Map<String, String> replaces = new HashMap<>();
+            generateDerivedMetric(modelIds, modelSchemaResps, metricTable.getMetrics(), metricTable.getDimensions(),
+                    measures, replaces);
+            if (!CollectionUtils.isEmpty(replaces)) {
+                // metricTable sql use measures replace metric
+                sql = SqlParserReplaceHelper.replaceSqlByExpression(sql, replaces);
+                metricTable.setAggOption(AggOption.NATIVE);
+            }
+            // metricTable  use measures replace metric
+            if (!CollectionUtils.isEmpty(measures)) {
+                metricTable.setMetrics(measures);
+            }
+        }
+        parseSqlReq.setSql(sql);
+    }
+
+    private void generateDerivedMetric(List<Long> modelIds, List<ModelSchemaResp> modelSchemaResps,
+            List<String> metrics, List<String> dimensions,
+            List<String> measures, Map<String, String> replaces) {
+        MetaFilter metaFilter = new MetaFilter();
+        metaFilter.setModelIds(modelIds);
+        List<MetricResp> metricResps = catalog.getMetrics(metaFilter);
+        List<DimensionResp> dimensionResps = catalog.getDimensions(metaFilter);
+        // check metrics has derived
+        if (!metricResps.stream()
+                .anyMatch(m -> metrics.contains(m.getBizName()) && MetricType.isDerived(m.getMetricDefineType(),
+                        m.getTypeParams()))) {
+            return;
+        }
+        Set<String> allFields = new HashSet<>();
+        Map<String, Measure> allMeasures = new HashMap<>();
+        modelSchemaResps.stream().forEach(modelSchemaResp -> {
+            allFields.addAll(modelSchemaResp.getFieldList());
+            if (Objects.nonNull(modelSchemaResp.getModelDetail().getMeasures())) {
+                modelSchemaResp.getModelDetail().getMeasures().stream()
+                        .forEach(mm -> allMeasures.put(mm.getBizName(), mm));
+            }
+        });
+
+        Set<String> deriveDimension = new HashSet<>();
+        Set<String> deriveMetric = new HashSet<>();
+        Set<String> visitedMetric = new HashSet<>();
+        if (!CollectionUtils.isEmpty(metricResps)) {
+            for (MetricResp metricResp : metricResps) {
+                if (metrics.contains(metricResp.getBizName())) {
+                    if (MetricType.isDerived(metricResp.getMetricDefineType(), metricResp.getTypeParams())) {
+                        String expr = sqlGenerateUtils.generateDerivedMetric(metricResps, allFields, allMeasures,
+                                dimensionResps,
+                                sqlGenerateUtils.getExpr(metricResp), metricResp.getMetricDefineType(), visitedMetric,
+                                deriveMetric, deriveDimension);
+                        replaces.put(metricResp.getBizName(), expr);
+                    } else {
+                        measures.add(metricResp.getBizName());
+                    }
+                }
+            }
+        }
+        measures.addAll(deriveMetric);
+        dimensions.addAll(deriveDimension);
+    }
+
 }