topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-10 11:07:06 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[improvement][headless]bizName of dimension and metric shall not be the same as the field name in the database table.

Browse Source
This commit is contained in:
jerryjzhang
2024-12-17 15:18:08 +08:00
parent 831fbfe475
commit b4c19533a4
10 changed files with 154 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
common/src/main/java/com/tencent/supersonic/common/jsqlparser/QueryExpressionReplaceVisitor.java
View File

@@ -1,13 +1,10 @@
package com.tencent.supersonic.common.jsqlparser;
import net.sf.jsqlparser.expression.Alias;
import net.sf.jsqlparser.expression.BinaryExpression;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.expression.ExpressionVisitorAdapter;
import net.sf.jsqlparser.expression.Function;
import net.sf.jsqlparser.expression.*;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import net.sf.jsqlparser.schema.Column;
import net.sf.jsqlparser.statement.select.SelectItem;
import org.apache.commons.lang3.StringUtils;
import java.util.Map;
import java.util.Objects;
@@ -50,7 +47,8 @@ public class QueryExpressionReplaceVisitor extends ExpressionVisitorAdapter {
String columnName = "";
if (expression instanceof Function) {
Function leftFunc = (Function) expression;
if (leftFunc.getParameters().getExpressions().get(0) instanceof Column) {
if (Objects.nonNull(leftFunc.getParameters())
&& leftFunc.getParameters().getExpressions().get(0) instanceof Column) {
Column column = (Column) leftFunc.getParameters().getExpressions().get(0);
columnName = column.getColumnName();
toReplace = getReplaceExpr(leftFunc, fieldExprMap);
@@ -75,7 +73,10 @@ public class QueryExpressionReplaceVisitor extends ExpressionVisitorAdapter {
public static Expression replace(Expression expression, Map<String, String> fieldExprMap) {
String toReplace = "";
if (expression instanceof Function) {
toReplace = getReplaceExpr((Function) expression, fieldExprMap);
Function function = (Function) expression;
if (function.getParameters().getExpressions().get(0) instanceof Column) {
toReplace = getReplaceExpr((Function) expression, fieldExprMap);
}
}
if (expression instanceof Column) {
toReplace = getReplaceExpr((Column) expression, fieldExprMap);
@@ -109,6 +110,16 @@ public class QueryExpressionReplaceVisitor extends ExpressionVisitorAdapter {
public static String getReplaceExpr(Function function, Map<String, String> fieldExprMap) {
Column column = (Column) function.getParameters().getExpressions().get(0);
return getReplaceExpr(column, fieldExprMap);
String expr = getReplaceExpr(column, fieldExprMap);
// if metric expr itself has agg function then replace original function in the SQL
if (StringUtils.isBlank(expr)) {
return expr;
} else if (!SqlSelectFunctionHelper.getAggregateFunctions(expr).isEmpty()) {
return expr;
} else {
String col = getReplaceExpr(column, fieldExprMap);
column.setColumnName(col);
return function.toString();
}
}
}

10
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Dimension.java
View File

@@ -32,6 +32,16 @@ public class Dimension {
this.type = type;
this.isCreateDimension = isCreateDimension;
this.bizName = bizName;
this.expr = bizName;
}
public Dimension(String name, String bizName, String expr, DimensionType type,
Integer isCreateDimension) {
this.name = name;
this.type = type;
this.isCreateDimension = isCreateDimension;
this.bizName = bizName;
this.expr = expr;
}
public Dimension(String name, String bizName, DimensionType type, Integer isCreateDimension,

9
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/Measure.java
View File

@@ -23,11 +23,20 @@ public class Measure {
private String alias;
public Measure(String name, String bizName, String expr, String agg, Integer isCreateMetric) {
this.name = name;
this.agg = agg;
this.isCreateMetric = isCreateMetric;
this.bizName = bizName;
this.expr = expr;
}
public Measure(String name, String bizName, String agg, Integer isCreateMetric) {
this.name = name;
this.agg = agg;
this.isCreateMetric = isCreateMetric;
this.bizName = bizName;
this.expr = bizName;
}
public Measure(String bizName, String constraint) {

4
headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/DimSchemaResp.java
View File

@@ -1,13 +1,17 @@
package com.tencent.supersonic.headless.api.pojo.response;
import com.google.common.collect.Sets;
import lombok.Data;
import lombok.ToString;
import java.util.Set;
@Data
@ToString(callSuper = true)
public class DimSchemaResp extends DimensionResp {
private Long useCnt = 0L;
private Set<String> fields = Sets.newHashSet();
@Override
public boolean equals(Object o) {

73
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/DimExpressionConverter.java Normal file
View File

@@ -0,0 +1,73 @@
package com.tencent.supersonic.headless.core.translator.converter;
import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.headless.api.pojo.Dimension;
import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.core.pojo.OntologyQuery;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.pojo.SqlQuery;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.*;
/**
* This converter replaces dimension bizName in the S2SQL with calculation expression (if
* configured).
*/
@Component("DimExpressionConverter")
@Slf4j
public class DimExpressionConverter implements QueryConverter {
@Override
public boolean accept(QueryStatement queryStatement) {
return Objects.nonNull(queryStatement.getSqlQuery())
&& StringUtils.isNotBlank(queryStatement.getSqlQuery().getSql());
}
@Override
public void convert(QueryStatement queryStatement) throws Exception {
SemanticSchemaResp semanticSchema = queryStatement.getSemanticSchema();
SqlQuery sqlQuery = queryStatement.getSqlQuery();
OntologyQuery ontologyQuery = queryStatement.getOntologyQuery();
Map<String, String> bizName2Expr = getDimensionExpressions(semanticSchema, ontologyQuery);
if (!CollectionUtils.isEmpty(bizName2Expr)) {
String sql = SqlReplaceHelper.replaceSqlByExpression(sqlQuery.getSql(), bizName2Expr);
sqlQuery.setSql(sql);
}
}
private Map<String, String> getDimensionExpressions(SemanticSchemaResp semanticSchema,
OntologyQuery ontologyQuery) {
Set<DimSchemaResp> queryDimensions = ontologyQuery.getDimensions();
Set<String> queryFields = ontologyQuery.getFields();
log.debug("begin to generateDerivedMetric {} [{}]", queryDimensions);
Set<String> allFields = new HashSet<>();
Map<String, Dimension> dimensionMap = new HashMap<>();
semanticSchema.getModelResps().forEach(modelResp -> {
allFields.addAll(modelResp.getFieldList());
if (modelResp.getModelDetail().getDimensions() != null) {
modelResp.getModelDetail().getDimensions()
.forEach(dimension -> dimensionMap.put(dimension.getBizName(), dimension));
}
});
Map<String, String> dim2Expr = new HashMap<>();
for (DimSchemaResp queryDim : queryDimensions) {
queryDim.getFields().addAll(SqlSelectHelper.getFieldsFromExpr(queryDim.getExpr()));
dim2Expr.put(queryDim.getBizName(), queryDim.getExpr());
queryFields.addAll(queryDim.getFields());
}
return dim2Expr;
}
}

70
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/MetricExpressionConverter.java
View File

@@ -2,11 +2,8 @@ package com.tencent.supersonic.headless.core.translator.converter;
import com.tencent.supersonic.common.jsqlparser.SqlReplaceHelper;
import com.tencent.supersonic.common.jsqlparser.SqlSelectHelper;
import com.tencent.supersonic.common.pojo.enums.AggOperatorEnum;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.pojo.response.DimSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricSchemaResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.core.pojo.OntologyQuery;
@@ -20,7 +17,7 @@ import org.springframework.util.CollectionUtils;
import java.util.*;
/**
* This converter replaces metric fields in the S2SQL with calculation expressions (if configured).
* This converter replaces metric bizName in the S2SQL with calculation expression (if configured).
*/
@Component("MetricExpressionConverter")
@Slf4j
@@ -38,11 +35,10 @@ public class MetricExpressionConverter implements QueryConverter {
SqlQuery sqlQuery = queryStatement.getSqlQuery();
OntologyQuery ontologyQuery = queryStatement.getOntologyQuery();
Map<String, String> metric2Expr = getMetricExpressions(semanticSchema, ontologyQuery);
if (!CollectionUtils.isEmpty(metric2Expr)) {
String sql = SqlReplaceHelper.replaceSqlByExpression(sqlQuery.getSql(), metric2Expr);
Map<String, String> bizName2Expr = getMetricExpressions(semanticSchema, ontologyQuery);
if (!CollectionUtils.isEmpty(bizName2Expr)) {
String sql = SqlReplaceHelper.replaceSqlByExpression(sqlQuery.getSql(), bizName2Expr);
sqlQuery.setSql(sql);
ontologyQuery.setAggOption(AggOption.NATIVE);
}
}
@@ -50,12 +46,9 @@ public class MetricExpressionConverter implements QueryConverter {
OntologyQuery ontologyQuery) {
List<MetricSchemaResp> allMetrics = semanticSchema.getMetrics();
List<DimSchemaResp> allDimensions = semanticSchema.getDimensions();
AggOption aggOption = ontologyQuery.getAggOption();
Set<MetricSchemaResp> queryMetrics = ontologyQuery.getMetrics();
Set<DimSchemaResp> queryDimensions = ontologyQuery.getDimensions();
Set<String> queryFields = ontologyQuery.getFields();
log.debug("begin to generateDerivedMetric {} [{}]", aggOption, queryMetrics);
log.debug("begin to generateDerivedMetric {} [{}]", queryMetrics);
Set<String> allFields = new HashSet<>();
Map<String, Measure> allMeasures = new HashMap<>();
@@ -70,13 +63,12 @@ public class MetricExpressionConverter implements QueryConverter {
Map<String, String> visitedMetrics = new HashMap<>();
Map<String, String> metric2Expr = new HashMap<>();
for (MetricSchemaResp queryMetric : queryMetrics) {
String fieldExpr = buildFieldExpr(allMetrics, allFields, allMeasures, allDimensions,
queryMetric.getExpr(), queryMetric.getMetricDefineType(), aggOption,
visitedMetrics, queryDimensions, queryFields);
String fieldExpr = buildFieldExpr(allMetrics, allMeasures, queryMetric.getExpr(),
queryMetric.getMetricDefineType(), visitedMetrics, queryFields);
// add all fields referenced in the expression
queryMetric.getFields().addAll(SqlSelectHelper.getFieldsFromExpr(fieldExpr));
log.debug("derived metric {}->{}", queryMetric.getBizName(), fieldExpr);
if (queryMetric.isDerived()) {
if (!queryMetric.getBizName().equals(fieldExpr)) {
metric2Expr.put(queryMetric.getBizName(), fieldExpr);
}
}
@@ -85,18 +77,16 @@ public class MetricExpressionConverter implements QueryConverter {
}
private String buildFieldExpr(final List<MetricSchemaResp> metricResps,
final Set<String> allFields, final Map<String, Measure> allMeasures,
final List<DimSchemaResp> dimensionResps, final String expression,
final MetricDefineType metricDefineType, AggOption aggOption,
Map<String, String> visitedMetric, Set<DimSchemaResp> queryDimensions,
final Map<String, Measure> allMeasures, final String metricExpr,
final MetricDefineType metricDefineType, Map<String, String> visitedMetric,
Set<String> queryFields) {
Set<String> fields = SqlSelectHelper.getFieldsFromExpr(expression);
Set<String> fields = SqlSelectHelper.getFieldsFromExpr(metricExpr);
if (!CollectionUtils.isEmpty(fields)) {
Map<String, String> replace = new HashMap<>();
for (String field : fields) {
queryFields.add(field);
switch (metricDefineType) {
case METRIC:
// if defineType=METRIC, field should be the bizName of its parent metric
Optional<MetricSchemaResp> metricItem = metricResps.stream()
.filter(m -> m.getBizName().equalsIgnoreCase(field)).findFirst();
if (metricItem.isPresent()) {
@@ -105,49 +95,39 @@ public class MetricExpressionConverter implements QueryConverter {
break;
}
replace.put(field,
buildFieldExpr(metricResps, allFields, allMeasures,
dimensionResps, metricItem.get().getExpr(),
metricItem.get().getMetricDefineType(), aggOption,
visitedMetric, queryDimensions, queryFields));
buildFieldExpr(metricResps, allMeasures,
metricItem.get().getExpr(),
metricItem.get().getMetricDefineType(), visitedMetric,
queryFields));
visitedMetric.put(field, replace.get(field));
}
break;
case MEASURE:
// if defineType=MEASURE, field should be the bizName of its measure
if (allMeasures.containsKey(field)) {
Measure measure = allMeasures.get(field);
if (AggOperatorEnum.COUNT_DISTINCT.getOperator()
.equalsIgnoreCase(measure.getAgg())) {
return AggOption.NATIVE.equals(aggOption) ? measure.getExpr()
: AggOperatorEnum.COUNT.getOperator() + " ( "
+ AggOperatorEnum.DISTINCT + " " + measure.getExpr()
+ " ) ";
String expr = metricExpr;
if (Objects.nonNull(measure.getAgg())) {
expr = String.format("%s (%s)", measure.getAgg(), metricExpr);
}
String expr = AggOption.NATIVE.equals(aggOption) ? measure.getExpr()
: measure.getAgg() + " ( " + measure.getExpr() + " ) ";
replace.put(field, expr);
queryFields.add(field);
}
break;
case FIELD:
if (allFields.contains(field)) {
Optional<DimSchemaResp> dimensionItem = dimensionResps.stream()
.filter(d -> d.getBizName().equals(field)).findFirst();
if (dimensionItem.isPresent()) {
queryDimensions.add(dimensionItem.get());
}
}
queryFields.add(field);
break;
default:
break;
}
}
if (!CollectionUtils.isEmpty(replace)) {
String expr = SqlReplaceHelper.replaceExpression(expression, replace);
log.debug("derived measure {}->{}", expression, expr);
String expr = SqlReplaceHelper.replaceExpression(metricExpr, replace);
log.debug("derived measure {}->{}", metricExpr, expr);
return expr;
}
}
return expression;
return metricExpr;
}
}

8
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/converter/SqlQueryConverter.java
View File

@@ -65,13 +65,7 @@ public class SqlQueryConverter implements QueryConverter {
ontologyQuery.getDimensions().addAll(queryDimensions);
AggOption sqlQueryAggOption = getAggOption(sqlQuery.getSql(), queryMetrics);
// if sql query itself has aggregation, ontology query just returns detail
if (sqlQueryAggOption.equals(AggOption.AGGREGATION)) {
ontologyQuery.setAggOption(AggOption.NATIVE);
} else if (sqlQueryAggOption.equals(AggOption.NATIVE) && !queryMetrics.isEmpty()) {
ontologyQuery.setAggOption(AggOption.DEFAULT);
}
ontologyQuery.setAggOption(sqlQueryAggOption);
queryStatement.setOntologyQuery(ontologyQuery);
log.info("parse sqlQuery [{}] ", sqlQuery);
}

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/translator/parser/calcite/SqlBuilder.java
View File

@@ -251,7 +251,7 @@ public class SqlBuilder {
EngineType engineType = EngineType.fromString(schema.getOntology().getDatabase().getType());
Set<String> queryFields = tableView.getFields();
queryMetrics.stream().forEach(m -> queryFields.addAll(m.getFields()));
queryDimensions.stream().forEach(m -> queryFields.add(m.getBizName()));
queryDimensions.stream().forEach(d -> queryFields.addAll(d.getFields()));
try {
for (String field : queryFields) {

18
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/ModelConverter.java
View File

@@ -28,7 +28,8 @@ public class ModelConverter {
public static ModelDO convert(ModelReq modelReq, User user) {
ModelDO modelDO = new ModelDO();
ModelDetail modelDetail = createModelDetail(modelReq);
// ModelDetail modelDetail = createModelDetail(modelReq);
ModelDetail modelDetail = modelReq.getModelDetail();
modelReq.createdBy(user.getName());
BeanMapper.mapper(modelReq, modelDO);
modelDO.setStatus(StatusEnum.ONLINE.getCode());
@@ -107,7 +108,7 @@ public class ModelConverter {
dimensionReq.setSemanticType(SemanticType.CATEGORY.name());
}
dimensionReq.setModelId(modelDO.getId());
dimensionReq.setExpr(dim.getBizName());
dimensionReq.setExpr(dim.getExpr());
dimensionReq.setType(dim.getType().name());
dimensionReq
.setDescription(Objects.isNull(dim.getDescription()) ? "" : dim.getDescription());
@@ -118,11 +119,11 @@ public class ModelConverter {
public static MetricReq convert(Measure measure, ModelDO modelDO) {
MetricReq metricReq = new MetricReq();
metricReq.setName(measure.getName());
metricReq.setBizName(measure.getExpr());
metricReq.setBizName(measure.getBizName());
metricReq.setDescription(measure.getName());
metricReq.setModelId(modelDO.getId());
MetricDefineByMeasureParams exprTypeParams = new MetricDefineByMeasureParams();
exprTypeParams.setExpr(measure.getBizName());
exprTypeParams.setExpr(measure.getExpr());
exprTypeParams.setMeasures(Lists.newArrayList(measure));
metricReq.setMetricDefineByMeasureParams(exprTypeParams);
metricReq.setMetricDefineType(MetricDefineType.MEASURE);
@@ -163,11 +164,14 @@ public class ModelConverter {
getIdentifyType(fieldType).name(), columnSchema.getColumnName(), 1);
modelDetail.getIdentifiers().add(identify);
} else if (FieldType.measure.equals(fieldType)) {
Measure measure = new Measure(columnSchema.getName(), columnSchema.getColumnName(),
columnSchema.getAgg().getOperator(), 1);
Measure measure = new Measure(columnSchema.getName(),
modelReq.getBizName() + "_" + columnSchema.getColumnName(),
columnSchema.getColumnName(), columnSchema.getAgg().getOperator(), 1);
modelDetail.getMeasures().add(measure);
} else {
Dimension dim = new Dimension(columnSchema.getName(), columnSchema.getColumnName(),
Dimension dim = new Dimension(columnSchema.getName(),
modelReq.getBizName() + "_" + columnSchema.getColumnName(),
columnSchema.getColumnName(),
DimensionType.valueOf(columnSchema.getFiledType().name()), 1);
modelDetail.getDimensions().add(dim);
}

1
launchers/standalone/src/main/resources/META-INF/spring.factories
View File

@@ -30,6 +30,7 @@ com.tencent.supersonic.headless.core.translator.converter.QueryConverter=\
com.tencent.supersonic.headless.core.translator.converter.StructQueryConverter,\
com.tencent.supersonic.headless.core.translator.converter.SqlQueryConverter,\
com.tencent.supersonic.headless.core.translator.converter.DefaultDimValueConverter,\
com.tencent.supersonic.headless.core.translator.converter.DimExpressionConverter,\
com.tencent.supersonic.headless.core.translator.converter.MetricExpressionConverter,\
com.tencent.supersonic.headless.core.translator.converter.MetricRatioConverter