From d4eecc1bf8a0355c700ad5efc77128c23aa341aa Mon Sep 17 00:00:00 2001 From: jipeli <54889677+jipeli@users.noreply.github.com> Date: Thu, 18 Jan 2024 11:11:56 +0800 Subject: [PATCH] (improvement)(headless) adapter for derived metrics (#646) --- .../common/pojo/enums/AggOperatorEnum.java | 1 + .../jsqlparser/ExpressionReplaceVisitor.java | 79 +++++++++ .../QueryExpressionReplaceVisitor.java | 106 ++++++++++++ .../jsqlparser/SqlParserReplaceHelper.java | 77 ++++++++- .../jsqlparser/SqlParserSelectHelper.java | 30 +++- .../headless/api/enums/MetricType.java | 37 ++++- .../headless/api/request/MetricReq.java | 17 +- .../headless/api/response/MetricResp.java | 9 +- .../headless/core/utils/SqlGenerateUtils.java | 151 +++++++++++++++--- .../service/impl/HeadlessQueryEngineImpl.java | 12 +- .../server/utils/QueryReqConverter.java | 76 ++++++++- 11 files changed, 537 insertions(+), 58 deletions(-) create mode 100644 common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/ExpressionReplaceVisitor.java create mode 100644 common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/QueryExpressionReplaceVisitor.java diff --git a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/AggOperatorEnum.java b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/AggOperatorEnum.java index 412dcf1f0..e438380ae 100644 --- a/common/src/main/java/com/tencent/supersonic/common/pojo/enums/AggOperatorEnum.java +++ b/common/src/main/java/com/tencent/supersonic/common/pojo/enums/AggOperatorEnum.java @@ -10,6 +10,7 @@ public enum AggOperatorEnum { SUM("SUM"), + COUNT("COUNT"), COUNT_DISTINCT("COUNT_DISTINCT"), DISTINCT("DISTINCT"), diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/ExpressionReplaceVisitor.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/ExpressionReplaceVisitor.java new file mode 100644 index 000000000..950fa5ad7 --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/ExpressionReplaceVisitor.java @@ -0,0 +1,79 @@ +package com.tencent.supersonic.common.util.jsqlparser; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import net.sf.jsqlparser.expression.BinaryExpression; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.ExpressionVisitorAdapter; +import net.sf.jsqlparser.expression.Function; +import net.sf.jsqlparser.schema.Column; + +public class ExpressionReplaceVisitor extends ExpressionVisitorAdapter { + + private Map fieldExprMap; + + + public ExpressionReplaceVisitor(Map fieldExprMap) { + this.fieldExprMap = fieldExprMap; + } + + protected void visitBinaryExpression(BinaryExpression expr) { + Expression left = expr.getLeftExpression(); + Expression right = expr.getRightExpression(); + Boolean leftVisited = false; + Boolean rightVisited = false; + if (left instanceof Function) { + Function leftFunc = (Function) left; + if (visitFunction(leftFunc)) { + leftVisited = true; + } + } + if (right instanceof Function) { + Function function = (Function) right; + if (visitFunction(function)) { + rightVisited = true; + } + } + if (left instanceof Column) { + Expression expression = QueryExpressionReplaceVisitor.getExpression( + QueryExpressionReplaceVisitor.getReplaceExpr((Column) left, fieldExprMap)); + if (Objects.nonNull(expression)) { + expr.setLeftExpression(expression); + leftVisited = true; + } + } + if (right instanceof Column) { + Expression expression = QueryExpressionReplaceVisitor.getExpression( + QueryExpressionReplaceVisitor.getReplaceExpr((Column) right, fieldExprMap)); + if (Objects.nonNull(expression)) { + expr.setRightExpression(expression); + rightVisited = true; + } + } + if (!leftVisited) { + expr.getLeftExpression().accept(this); + } + if (!rightVisited) { + expr.getRightExpression().accept(this); + } + } + + private boolean visitFunction(Function function) { + if (function.getParameters().getExpressions().get(0) instanceof Column) { + Expression expression = QueryExpressionReplaceVisitor.getExpression( + QueryExpressionReplaceVisitor.getReplaceExpr(function, fieldExprMap)); + if (Objects.nonNull(expression)) { + List expressions = new ArrayList<>(); + expressions.add(expression); + for (int i = 1; i < function.getParameters().getExpressions().size(); i++) { + expressions.add(function.getParameters().getExpressions().get(i)); + } + function.getParameters().setExpressions(expressions); + return true; + } + } + return false; + } +} diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/QueryExpressionReplaceVisitor.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/QueryExpressionReplaceVisitor.java new file mode 100644 index 000000000..a6afc3dbc --- /dev/null +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/QueryExpressionReplaceVisitor.java @@ -0,0 +1,106 @@ +package com.tencent.supersonic.common.util.jsqlparser; + +import java.util.Map; +import java.util.Objects; +import net.sf.jsqlparser.expression.BinaryExpression; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.ExpressionVisitorAdapter; +import net.sf.jsqlparser.expression.Function; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.schema.Column; +import net.sf.jsqlparser.statement.select.SelectExpressionItem; + +public class QueryExpressionReplaceVisitor extends ExpressionVisitorAdapter { + + private Map fieldExprMap; + + public QueryExpressionReplaceVisitor(Map fieldExprMap) { + this.fieldExprMap = fieldExprMap; + } + + protected void visitBinaryExpression(BinaryExpression expr) { + Expression left = expr.getLeftExpression(); + String toReplace = ""; + if (left instanceof Function) { + Function leftFunc = (Function) left; + if (leftFunc.getParameters().getExpressions().get(0) instanceof Column) { + toReplace = getReplaceExpr(leftFunc, fieldExprMap); + } + } + if (left instanceof Column) { + toReplace = getReplaceExpr((Column) left, fieldExprMap); + } + if (!toReplace.isEmpty()) { + Expression expression = getExpression(toReplace); + if (Objects.nonNull(expression)) { + expr.setLeftExpression(expression); + return; + } + } + expr.getLeftExpression().accept(this); + expr.getRightExpression().accept(this); + + } + + public void visit(SelectExpressionItem selectExpressionItem) { + + Expression expression = selectExpressionItem.getExpression(); + String toReplace = ""; + if (expression instanceof Function) { + Function leftFunc = (Function) expression; + if (leftFunc.getParameters().getExpressions().get(0) instanceof Column) { + toReplace = getReplaceExpr(leftFunc, fieldExprMap); + } + + } + if (expression instanceof Column) { + toReplace = getReplaceExpr((Column) expression, fieldExprMap); + } + if (!toReplace.isEmpty()) { + Expression toReplaceExpr = getExpression(toReplace); + if (Objects.nonNull(toReplaceExpr)) { + selectExpressionItem.setExpression(toReplaceExpr); + } + } + //selectExpressionItem.getExpression().accept(this); + } + + public static Expression replace(Expression expression, Map fieldExprMap) { + String toReplace = ""; + if (expression instanceof Function) { + toReplace = getReplaceExpr((Function) expression, fieldExprMap); + } + if (expression instanceof Column) { + toReplace = getReplaceExpr((Column) expression, fieldExprMap); + } + if (!toReplace.isEmpty()) { + Expression replace = getExpression(toReplace); + if (Objects.nonNull(replace)) { + return replace; + } + } + return expression; + } + + public static Expression getExpression(String expr) { + if (expr.isEmpty()) { + return null; + } + try { + Expression expression = CCJSqlParserUtil.parseExpression(expr); + return expression; + } catch (Exception e) { + return null; + } + } + + public static String getReplaceExpr(Column column, Map fieldExprMap) { + return fieldExprMap.containsKey(column.getColumnName()) ? fieldExprMap.get(column.getColumnName()) : ""; + } + + public static String getReplaceExpr(Function function, Map fieldExprMap) { + Column column = (Column) function.getParameters().getExpressions().get(0); + return getReplaceExpr(column, fieldExprMap); + } + +} diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserReplaceHelper.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserReplaceHelper.java index 8f094c6be..6782c8208 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserReplaceHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserReplaceHelper.java @@ -2,6 +2,11 @@ package com.tencent.supersonic.common.util.jsqlparser; import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum; import com.tencent.supersonic.common.util.StringUtil; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.expression.Alias; @@ -32,11 +37,6 @@ import net.sf.jsqlparser.statement.select.SubSelect; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.springframework.util.CollectionUtils; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; /** * Sql Parser replace Helper @@ -120,7 +120,7 @@ public class SqlParserReplaceHelper { } public static String replaceValue(String sql, Map> filedNameToValueMap, - boolean exactReplace) { + boolean exactReplace) { Select selectStatement = SqlParserSelectHelper.getSelect(sql); SelectBody selectBody = selectStatement.getSelectBody(); if (!(selectBody instanceof PlainSelect)) { @@ -196,7 +196,7 @@ public class SqlParserReplaceHelper { } private static void replaceFieldsInPlainOneSelect(Map fieldNameMap, boolean exactReplace, - PlainSelect plainSelect) { + PlainSelect plainSelect) { //1. replace where fields Expression where = plainSelect.getWhere(); FieldReplaceVisitor visitor = new FieldReplaceVisitor(fieldNameMap, exactReplace); @@ -351,7 +351,7 @@ public class SqlParserReplaceHelper { } private static void replaceOrderByFunction(Map functionMap, - List orderByElementList) { + List orderByElementList) { if (Objects.isNull(orderByElementList)) { return; } @@ -499,5 +499,66 @@ public class SqlParserReplaceHelper { return expression; } } + + public static String replaceExpression(String expr, Map replace) { + Expression expression = QueryExpressionReplaceVisitor.getExpression(expr); + if (Objects.nonNull(expression)) { + if (expression instanceof Column && replace.containsKey(expr)) { + return replace.get(expr); + } + ExpressionReplaceVisitor expressionReplaceVisitor = new ExpressionReplaceVisitor(replace); + expression.accept(expressionReplaceVisitor); + return expression.toString(); + } + return expr; + } + + public static String replaceSqlByExpression(String sql, Map replace) { + Select selectStatement = SqlParserSelectHelper.getSelect(sql); + SelectBody selectBody = selectStatement.getSelectBody(); + List plainSelectList = new ArrayList<>(); + if (selectBody instanceof PlainSelect) { + plainSelectList.add((PlainSelect) selectBody); + } else if (selectBody instanceof SetOperationList) { + SetOperationList setOperationList = (SetOperationList) selectBody; + if (!CollectionUtils.isEmpty(setOperationList.getSelects())) { + setOperationList.getSelects().forEach(subSelectBody -> { + PlainSelect subPlainSelect = (PlainSelect) subSelectBody; + plainSelectList.add(subPlainSelect); + }); + } + } else { + return sql; + } + List plainSelects = SqlParserSelectHelper.getPlainSelects(plainSelectList); + for (PlainSelect plainSelect : plainSelects) { + replacePlainSelectByExpr(plainSelect, replace); + } + return selectStatement.toString(); + } + + private static void replacePlainSelectByExpr(PlainSelect plainSelect, Map replace) { + QueryExpressionReplaceVisitor expressionReplaceVisitor = new QueryExpressionReplaceVisitor(replace); + for (SelectItem selectItem : plainSelect.getSelectItems()) { + selectItem.accept(expressionReplaceVisitor); + } + Expression having = plainSelect.getHaving(); + if (Objects.nonNull(having)) { + having.accept(expressionReplaceVisitor); + } + + Expression where = plainSelect.getWhere(); + if (Objects.nonNull(where)) { + where.accept(expressionReplaceVisitor); + } + + List orderByElements = plainSelect.getOrderByElements(); + if (!CollectionUtils.isEmpty(orderByElements)) { + for (OrderByElement orderByElement : orderByElements) { + orderByElement.setExpression( + QueryExpressionReplaceVisitor.replace(orderByElement.getExpression(), replace)); + } + } + } } diff --git a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelper.java b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelper.java index 7cf1ccf78..af38bd9b2 100644 --- a/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelper.java +++ b/common/src/main/java/com/tencent/supersonic/common/util/jsqlparser/SqlParserSelectHelper.java @@ -6,9 +6,9 @@ import java.util.List; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; - import lombok.extern.slf4j.Slf4j; import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.expression.BinaryExpression; import net.sf.jsqlparser.expression.Expression; import net.sf.jsqlparser.expression.ExpressionVisitorAdapter; import net.sf.jsqlparser.expression.Function; @@ -32,8 +32,8 @@ import net.sf.jsqlparser.statement.select.SelectBody; import net.sf.jsqlparser.statement.select.SelectExpressionItem; import net.sf.jsqlparser.statement.select.SelectItem; import net.sf.jsqlparser.statement.select.SelectVisitorAdapter; -import net.sf.jsqlparser.statement.select.SubSelect; import net.sf.jsqlparser.statement.select.SetOperationList; +import net.sf.jsqlparser.statement.select.SubSelect; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.ImmutablePair; import org.springframework.util.CollectionUtils; @@ -480,5 +480,31 @@ public class SqlParserSelectHelper { } return selectStatement.toString(); } + + public static Set getColumnFromExpr(String expr) { + Expression expression = QueryExpressionReplaceVisitor.getExpression(expr); + Set columns = new HashSet<>(); + if (Objects.nonNull(expression)) { + getColumnFromExpr(expression, columns); + } + return columns; + } + + public static void getColumnFromExpr(Expression expression, Set columns) { + if (expression instanceof Column) { + columns.add(((Column) expression).getColumnName()); + } + if (expression instanceof Function) { + List expressionList = ((Function) expression).getParameters().getExpressions(); + for (Expression expr : expressionList) { + getColumnFromExpr(expr, columns); + } + } + if (expression instanceof BinaryExpression) { + BinaryExpression expr = (BinaryExpression) expression; + getColumnFromExpr(expr.getLeftExpression(), columns); + getColumnFromExpr(expr.getRightExpression(), columns); + } + } } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/enums/MetricType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/enums/MetricType.java index 47d38b941..ec5735236 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/enums/MetricType.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/enums/MetricType.java @@ -1,9 +1,44 @@ package com.tencent.supersonic.headless.api.enums; +import com.tencent.supersonic.headless.api.pojo.MeasureParam; +import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams; +import java.util.List; +import java.util.Objects; + public enum MetricType { ATOMIC, - DERIVED + DERIVED; + public static MetricType of(String src) { + for (MetricType metricType : MetricType.values()) { + if (Objects.nonNull(src) && src.equalsIgnoreCase(metricType.name())) { + return metricType; + } + } + return null; + } + + public static Boolean isDerived(String src) { + MetricType metricType = of(src); + return Objects.nonNull(metricType) && metricType.equals(DERIVED); + } + + public static Boolean isDerived(MetricDefineType metricDefineType, MetricDefineByMeasureParams typeParams) { + if (MetricDefineType.METRIC.equals(metricDefineType)) { + return true; + } + if (MetricDefineType.MEASURE.equals(metricDefineType)) { + List measures = typeParams.getMeasures(); + if (measures.size() > 1) { + return true; + } + if (measures.size() == 1 && measures.get(0).getBizName() + .equalsIgnoreCase(typeParams.getExpr())) { + return false; + } + } + return false; + } } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/MetricReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/MetricReq.java index 50cd6f8b9..834bee10d 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/MetricReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/request/MetricReq.java @@ -3,12 +3,10 @@ package com.tencent.supersonic.headless.api.request; import com.alibaba.fastjson.JSONObject; import com.tencent.supersonic.headless.api.enums.MetricDefineType; import com.tencent.supersonic.headless.api.enums.MetricType; -import com.tencent.supersonic.headless.api.pojo.MeasureParam; import com.tencent.supersonic.headless.api.pojo.MetricDefineByFieldParams; import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams; import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams; import lombok.Data; -import java.util.List; @Data public class MetricReq extends MetricBaseReq { @@ -30,20 +28,7 @@ public class MetricReq extends MetricBaseReq { } public MetricType getMetricType() { - if (MetricDefineType.METRIC.equals(metricDefineType)) { - return MetricType.DERIVED; - } - if (MetricDefineType.MEASURE.equals(metricDefineType)) { - List measures = typeParams.getMeasures(); - if (measures.size() > 1) { - return MetricType.DERIVED; - } - if (measures.size() == 1 && measures.get(0).getBizName() - .equalsIgnoreCase(typeParams.getExpr())) { - return MetricType.ATOMIC; - } - } - return MetricType.ATOMIC; + return MetricType.isDerived(metricDefineType, typeParams) ? MetricType.DERIVED : MetricType.ATOMIC; } } diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/response/MetricResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/response/MetricResp.java index 145a93a81..4b50abd59 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/response/MetricResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/response/MetricResp.java @@ -10,16 +10,15 @@ import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams; import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams; import com.tencent.supersonic.headless.api.pojo.RelateDimension; import com.tencent.supersonic.headless.api.pojo.SchemaItem; -import lombok.Data; -import lombok.ToString; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import lombok.Data; +import lombok.ToString; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; @Data diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java index 017abadf6..191b83376 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java @@ -1,5 +1,12 @@ package com.tencent.supersonic.headless.core.utils; +import static com.tencent.supersonic.common.pojo.Constants.DAY; +import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT; +import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE; +import static com.tencent.supersonic.common.pojo.Constants.MONTH; +import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE; +import static com.tencent.supersonic.common.pojo.Constants.WEEK; + import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.DateConf; import com.tencent.supersonic.common.pojo.ItemDateResp; @@ -8,8 +15,26 @@ import com.tencent.supersonic.common.pojo.enums.TimeDimensionEnum; import com.tencent.supersonic.common.util.DateModeUtils; import com.tencent.supersonic.common.util.SqlFilterUtils; import com.tencent.supersonic.common.util.StringUtil; +import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper; +import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; import com.tencent.supersonic.headless.api.enums.EngineType; +import com.tencent.supersonic.headless.api.enums.MetricDefineType; +import com.tencent.supersonic.headless.api.enums.MetricType; +import com.tencent.supersonic.headless.api.pojo.Measure; import com.tencent.supersonic.headless.api.request.QueryStructReq; +import com.tencent.supersonic.headless.api.response.DimensionResp; +import com.tencent.supersonic.headless.api.response.MetricResp; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.ImmutablePair; @@ -19,22 +44,6 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; -import java.time.LocalDate; -import java.time.format.DateTimeFormatter; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.stream.Collectors; - -import static com.tencent.supersonic.common.pojo.Constants.DAY; -import static com.tencent.supersonic.common.pojo.Constants.DAY_FORMAT; -import static com.tencent.supersonic.common.pojo.Constants.JOIN_UNDERLINE; -import static com.tencent.supersonic.common.pojo.Constants.MONTH; -import static com.tencent.supersonic.common.pojo.Constants.UNDERLINE; -import static com.tencent.supersonic.common.pojo.Constants.WEEK; - /** * tools functions to analyze queryStructReq */ @@ -54,7 +63,7 @@ public class SqlGenerateUtils { private String internalMetricNameSuffix; public SqlGenerateUtils(SqlFilterUtils sqlFilterUtils, - DateModeUtils dateModeUtils) { + DateModeUtils dateModeUtils) { this.sqlFilterUtils = sqlFilterUtils; this.dateModeUtils = dateModeUtils; } @@ -97,6 +106,13 @@ public class SqlGenerateUtils { : String.join(",", queryStructCmd.getGroups()) + "," + aggStr; } + public String getSelect(QueryStructReq queryStructCmd, Map deriveMetrics) { + String aggStr = queryStructCmd.getAggregators().stream().map(a -> getSelectField(a, deriveMetrics)) + .collect(Collectors.joining(",")); + return CollectionUtils.isEmpty(queryStructCmd.getGroups()) ? aggStr + : String.join(",", queryStructCmd.getGroups()) + "," + aggStr; + } + public String getSelectField(final Aggregator agg) { if (AggOperatorEnum.COUNT_DISTINCT.equals(agg.getFunc())) { return "count(distinct " + agg.getColumn() + " ) AS " + agg.getColumn() + " "; @@ -109,6 +125,13 @@ public class SqlGenerateUtils { ).collect(Collectors.joining(",")) + " ) AS " + agg.getColumn() + " "; } + public String getSelectField(final Aggregator agg, Map deriveMetrics) { + if (!deriveMetrics.containsKey(agg.getColumn())) { + return getSelectField(agg); + } + return deriveMetrics.get(agg.getColumn()); + } + public String getGroupBy(QueryStructReq queryStructCmd) { if (CollectionUtils.isEmpty(queryStructCmd.getGroups())) { return ""; @@ -125,6 +148,19 @@ public class SqlGenerateUtils { .collect(Collectors.joining(",")); } + public String getOrderBy(QueryStructReq queryStructCmd, Map deriveMetrics) { + if (CollectionUtils.isEmpty(queryStructCmd.getOrders())) { + return ""; + } + if (!queryStructCmd.getOrders().stream().anyMatch(o -> deriveMetrics.containsKey(o.getColumn()))) { + return getOrderBy(queryStructCmd); + } + return "order by " + queryStructCmd.getOrders().stream() + .map(order -> " " + (deriveMetrics.containsKey(order.getColumn()) ? deriveMetrics.get(order.getColumn()) + : order.getColumn()) + " " + order.getDirection() + " ") + .collect(Collectors.joining(",")); + } + public String generateWhere(QueryStructReq queryStructReq, ItemDateResp itemDateResp) { String whereClauseFromFilter = sqlFilterUtils.getWhereClause(queryStructReq.getOriginalFilter()); String whereFromDate = getDateWhereClause(queryStructReq.getDateInfo(), itemDateResp); @@ -132,7 +168,7 @@ public class SqlGenerateUtils { } private String mergeDateWhereClause(QueryStructReq queryStructCmd, String whereClauseFromFilter, - String whereFromDate) { + String whereFromDate) { if (Strings.isNotEmpty(whereFromDate) && Strings.isNotEmpty(whereClauseFromFilter)) { return String.format("%s AND (%s)", whereFromDate, whereClauseFromFilter); } else if (Strings.isEmpty(whereFromDate) && Strings.isNotEmpty(whereClauseFromFilter)) { @@ -233,4 +269,83 @@ public class SqlGenerateUtils { return modelBizName + UNDERLINE + internalMetricNameSuffix; } + public String generateDerivedMetric(final List metricResps, final Set allFields, + final Map allMeasures, final List dimensionResps, + final String expression, final MetricDefineType metricDefineType, Set visitedMetric, + Set measures, + Set dimensions) { + Set fields = SqlParserSelectHelper.getColumnFromExpr(expression); + if (!CollectionUtils.isEmpty(fields)) { + Map replace = new HashMap<>(); + for (String field : fields) { + switch (metricDefineType) { + case METRIC: + Optional metricItem = metricResps.stream() + .filter(m -> m.getBizName().equalsIgnoreCase(field)).findFirst(); + if (metricItem.isPresent()) { + if (MetricType.isDerived(metricItem.get().getMetricDefineType(), + metricItem.get().getTypeParams())) { + if (visitedMetric.contains(field)) { + break; + } + replace.put(field, + generateDerivedMetric(metricResps, allFields, allMeasures, dimensionResps, + getExpr(metricItem.get()), metricItem.get().getMetricDefineType(), + visitedMetric, measures, dimensions)); + visitedMetric.add(field); + } else { + replace.put(field, getExpr(metricItem.get())); + } + } + break; + case MEASURE: + if (allMeasures.containsKey(field)) { + measures.add(field); + replace.put(field, getExpr(allMeasures.get(field))); + } + break; + case FIELD: + if (allFields.contains(field)) { + Optional dimensionItem = dimensionResps.stream() + .filter(d -> d.getBizName().equals(field)).findFirst(); + if (dimensionItem.isPresent()) { + dimensions.add(field); + } else { + measures.add(field); + } + } + break; + default: + break; + + } + } + if (!CollectionUtils.isEmpty(replace)) { + return SqlParserReplaceHelper.replaceExpression(expression, replace); + } + } + return expression; + } + + public String getExpr(Measure measure) { + if (AggOperatorEnum.COUNT_DISTINCT.getOperator().equalsIgnoreCase(measure.getAgg())) { + return AggOperatorEnum.COUNT.getOperator() + " ( " + AggOperatorEnum.DISTINCT + " " + measure.getBizName() + + " ) "; + } + return measure.getAgg() + " ( " + measure.getBizName() + " ) "; + } + + public String getExpr(MetricResp metricResp) { + if (Objects.isNull(metricResp.getMetricDefineType())) { + return metricResp.getTypeParams().getExpr(); + } + if (metricResp.getMetricDefineType().equals(MetricDefineType.METRIC)) { + return metricResp.getMetricDefineByMetricParams().getExpr(); + } + if (metricResp.getMetricDefineType().equals(MetricDefineType.FIELD)) { + return metricResp.getMetricDefineByFieldParams().getExpr(); + } + // measure add agg function + return metricResp.getTypeParams().getExpr(); + } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/HeadlessQueryEngineImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/HeadlessQueryEngineImpl.java index 53340ec29..59875615c 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/HeadlessQueryEngineImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/HeadlessQueryEngineImpl.java @@ -6,21 +6,21 @@ import com.tencent.supersonic.headless.api.request.ParseSqlReq; import com.tencent.supersonic.headless.api.request.QueryStructReq; import com.tencent.supersonic.headless.api.response.ModelSchemaResp; import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp; -import com.tencent.supersonic.headless.core.executor.QueryExecutor; import com.tencent.supersonic.headless.core.optimizer.QueryOptimizer; import com.tencent.supersonic.headless.core.parser.QueryParser; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.HeadlessModel; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.utils.ComponentFactory; +import com.tencent.supersonic.headless.core.executor.QueryExecutor; import com.tencent.supersonic.headless.server.manager.HeadlessSchemaManager; import com.tencent.supersonic.headless.server.service.Catalog; import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine; import com.tencent.supersonic.headless.server.utils.QueryStructUtils; import com.tencent.supersonic.headless.server.utils.QueryUtils; -import java.util.List; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; +import java.util.List; @Slf4j @Component @@ -82,22 +82,20 @@ public class HeadlessQueryEngineImpl implements HeadlessQueryEngine { } @Override - public QueryStatement physicalSql(QueryStructReq queryStructCmd, ParseSqlReq sqlCommend) throws Exception { + public QueryStatement physicalSql(QueryStructReq queryStructCmd, ParseSqlReq sqlCommend) { QueryStatement queryStatement = new QueryStatement(); - queryStatement.setSql(sqlCommend.getSql()); queryStatement.setQueryStructReq(queryStructCmd); queryStatement.setParseSqlReq(sqlCommend); + queryStatement.setSql(sqlCommend.getSql()); queryStatement.setIsS2SQL(true); - queryStatement.setHeadlessModel(getHeadLessModel(queryStatement)); return optimize(queryStructCmd, queryParser.parser(sqlCommend, queryStatement)); } - public QueryStatement physicalSql(QueryStructReq queryStructCmd, MetricQueryReq metricCommand) throws Exception { + public QueryStatement physicalSql(QueryStructReq queryStructCmd, MetricQueryReq metricCommand) { QueryStatement queryStatement = new QueryStatement(); queryStatement.setQueryStructReq(queryStructCmd); queryStatement.setMetricReq(metricCommand); queryStatement.setIsS2SQL(false); - queryStatement.setHeadlessModel(getHeadLessModel(queryStatement)); return queryParser.parser(queryStatement); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java index 1ccd56f9c..21381388a 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java @@ -11,6 +11,8 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectFunctionHelp import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; import com.tencent.supersonic.headless.api.enums.AggOption; import com.tencent.supersonic.headless.api.enums.EngineType; +import com.tencent.supersonic.headless.api.enums.MetricType; +import com.tencent.supersonic.headless.api.pojo.Measure; import com.tencent.supersonic.headless.api.pojo.MetricTable; import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.request.ParseSqlReq; @@ -18,14 +20,18 @@ import com.tencent.supersonic.headless.api.request.QueryS2SQLReq; import com.tencent.supersonic.headless.api.request.QueryStructReq; import com.tencent.supersonic.headless.api.request.SqlExecuteReq; import com.tencent.supersonic.headless.api.response.DatabaseResp; +import com.tencent.supersonic.headless.api.response.DimensionResp; +import com.tencent.supersonic.headless.api.response.MetricResp; import com.tencent.supersonic.headless.api.response.ModelSchemaResp; import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor; import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory; import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; +import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.Catalog; import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -119,7 +125,9 @@ public class QueryReqConverter { result.setSupportWith(false); result.setWithAlias(false); } - //5.physicalSql by ParseSqlReq + //5. do deriveMetric + generateDerivedMetric(queryS2SQLReq.getModelIds(), modelSchemaResps, result); + //6.physicalSql by ParseSqlReq queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(queryS2SQLReq.getSql())); queryStructReq.setModelIds(new HashSet<>(queryS2SQLReq.getModelIds())); queryStructReq.setQueryType(getQueryType(aggOption)); @@ -240,4 +248,70 @@ public class QueryReqConverter { return queryType; } + private void generateDerivedMetric(List modelIds, List modelSchemaResps, + ParseSqlReq parseSqlReq) { + String sql = parseSqlReq.getSql(); + for (MetricTable metricTable : parseSqlReq.getTables()) { + List measures = new ArrayList<>(); + Map replaces = new HashMap<>(); + generateDerivedMetric(modelIds, modelSchemaResps, metricTable.getMetrics(), metricTable.getDimensions(), + measures, replaces); + if (!CollectionUtils.isEmpty(replaces)) { + // metricTable sql use measures replace metric + sql = SqlParserReplaceHelper.replaceSqlByExpression(sql, replaces); + metricTable.setAggOption(AggOption.NATIVE); + } + // metricTable use measures replace metric + if (!CollectionUtils.isEmpty(measures)) { + metricTable.setMetrics(measures); + } + } + parseSqlReq.setSql(sql); + } + + private void generateDerivedMetric(List modelIds, List modelSchemaResps, + List metrics, List dimensions, + List measures, Map replaces) { + MetaFilter metaFilter = new MetaFilter(); + metaFilter.setModelIds(modelIds); + List metricResps = catalog.getMetrics(metaFilter); + List dimensionResps = catalog.getDimensions(metaFilter); + // check metrics has derived + if (!metricResps.stream() + .anyMatch(m -> metrics.contains(m.getBizName()) && MetricType.isDerived(m.getMetricDefineType(), + m.getTypeParams()))) { + return; + } + Set allFields = new HashSet<>(); + Map allMeasures = new HashMap<>(); + modelSchemaResps.stream().forEach(modelSchemaResp -> { + allFields.addAll(modelSchemaResp.getFieldList()); + if (Objects.nonNull(modelSchemaResp.getModelDetail().getMeasures())) { + modelSchemaResp.getModelDetail().getMeasures().stream() + .forEach(mm -> allMeasures.put(mm.getBizName(), mm)); + } + }); + + Set deriveDimension = new HashSet<>(); + Set deriveMetric = new HashSet<>(); + Set visitedMetric = new HashSet<>(); + if (!CollectionUtils.isEmpty(metricResps)) { + for (MetricResp metricResp : metricResps) { + if (metrics.contains(metricResp.getBizName())) { + if (MetricType.isDerived(metricResp.getMetricDefineType(), metricResp.getTypeParams())) { + String expr = sqlGenerateUtils.generateDerivedMetric(metricResps, allFields, allMeasures, + dimensionResps, + sqlGenerateUtils.getExpr(metricResp), metricResp.getMetricDefineType(), visitedMetric, + deriveMetric, deriveDimension); + replaces.put(metricResp.getBizName(), expr); + } else { + measures.add(metricResp.getBizName()); + } + } + } + } + measures.addAll(deriveMetric); + dimensions.addAll(deriveDimension); + } + }