mirror of
https://github.com/tencentmusic/supersonic.git
synced 2025-12-11 12:07:42 +00:00
(improvement)(headless) parser add model field (#650)
This commit is contained in:
@@ -29,6 +29,9 @@ public enum MetricType {
|
|||||||
if (MetricDefineType.METRIC.equals(metricDefineType)) {
|
if (MetricDefineType.METRIC.equals(metricDefineType)) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
if (MetricDefineType.FIELD.equals(metricDefineType)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
if (MetricDefineType.MEASURE.equals(metricDefineType)) {
|
if (MetricDefineType.MEASURE.equals(metricDefineType)) {
|
||||||
List<MeasureParam> measures = typeParams.getMeasures();
|
List<MeasureParam> measures = typeParams.getMeasures();
|
||||||
if (measures.size() > 1) {
|
if (measures.size() > 1) {
|
||||||
|
|||||||
@@ -1,10 +1,17 @@
|
|||||||
package com.tencent.supersonic.headless.core.manager;
|
package com.tencent.supersonic.headless.core.manager;
|
||||||
|
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
|
import com.tencent.supersonic.headless.api.enums.MetricDefineType;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.FieldParam;
|
||||||
import com.tencent.supersonic.headless.api.pojo.MeasureParam;
|
import com.tencent.supersonic.headless.api.pojo.MeasureParam;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.MetricDefineByFieldParams;
|
||||||
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
|
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.MetricParam;
|
||||||
import com.tencent.supersonic.headless.api.response.MetricResp;
|
import com.tencent.supersonic.headless.api.response.MetricResp;
|
||||||
|
import com.tencent.supersonic.headless.core.pojo.yaml.FieldParamYamlTpl;
|
||||||
import com.tencent.supersonic.headless.core.pojo.yaml.MeasureYamlTpl;
|
import com.tencent.supersonic.headless.core.pojo.yaml.MeasureYamlTpl;
|
||||||
|
import com.tencent.supersonic.headless.core.pojo.yaml.MetricParamYamlTpl;
|
||||||
import com.tencent.supersonic.headless.core.pojo.yaml.MetricTypeParamsYamlTpl;
|
import com.tencent.supersonic.headless.core.pojo.yaml.MetricTypeParamsYamlTpl;
|
||||||
import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
|
import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
@@ -37,12 +44,26 @@ public class MetricYamlManager {
|
|||||||
BeanUtils.copyProperties(metric, metricYamlTpl);
|
BeanUtils.copyProperties(metric, metricYamlTpl);
|
||||||
metricYamlTpl.setName(metric.getBizName());
|
metricYamlTpl.setName(metric.getBizName());
|
||||||
metricYamlTpl.setOwners(Lists.newArrayList(metric.getCreatedBy()));
|
metricYamlTpl.setOwners(Lists.newArrayList(metric.getCreatedBy()));
|
||||||
MetricDefineByMeasureParams metricDefineParams = metric.getTypeParams();
|
|
||||||
MetricTypeParamsYamlTpl metricTypeParamsYamlTpl = new MetricTypeParamsYamlTpl();
|
MetricTypeParamsYamlTpl metricTypeParamsYamlTpl = new MetricTypeParamsYamlTpl();
|
||||||
|
if (MetricDefineType.MEASURE.equals(metric.getMetricDefineType())) {
|
||||||
|
MetricDefineByMeasureParams metricDefineParams = metric.getTypeParams();
|
||||||
metricTypeParamsYamlTpl.setExpr(metricDefineParams.getExpr());
|
metricTypeParamsYamlTpl.setExpr(metricDefineParams.getExpr());
|
||||||
List<MeasureParam> measures = metricDefineParams.getMeasures();
|
List<MeasureParam> measures = metricDefineParams.getMeasures();
|
||||||
metricTypeParamsYamlTpl.setMeasures(
|
metricTypeParamsYamlTpl.setMeasures(
|
||||||
measures.stream().map(MetricYamlManager::convert).collect(Collectors.toList()));
|
measures.stream().map(MetricYamlManager::convert).collect(Collectors.toList()));
|
||||||
|
} else if (MetricDefineType.FIELD.equals(metric.getMetricDefineType())) {
|
||||||
|
MetricDefineByFieldParams metricDefineParams = metric.getMetricDefineByFieldParams();
|
||||||
|
metricTypeParamsYamlTpl.setExpr(metricDefineParams.getExpr());
|
||||||
|
List<FieldParam> fields = metricDefineParams.getFields();
|
||||||
|
metricTypeParamsYamlTpl.setFields(
|
||||||
|
fields.stream().map(MetricYamlManager::convert).collect(Collectors.toList()));
|
||||||
|
} else if (MetricDefineType.METRIC.equals(metric.getMetricDefineType())) {
|
||||||
|
MetricDefineByMetricParams metricDefineByMetricParams = metric.getMetricDefineByMetricParams();
|
||||||
|
metricTypeParamsYamlTpl.setExpr(metricDefineByMetricParams.getExpr());
|
||||||
|
List<MetricParam> metrics = metricDefineByMetricParams.getMetrics();
|
||||||
|
metricTypeParamsYamlTpl.setMetrics(
|
||||||
|
metrics.stream().map(MetricYamlManager::convert).collect(Collectors.toList()));
|
||||||
|
}
|
||||||
metricYamlTpl.setTypeParams(metricTypeParamsYamlTpl);
|
metricYamlTpl.setTypeParams(metricTypeParamsYamlTpl);
|
||||||
return metricYamlTpl;
|
return metricYamlTpl;
|
||||||
}
|
}
|
||||||
@@ -55,4 +76,17 @@ public class MetricYamlManager {
|
|||||||
return measureYamlTpl;
|
return measureYamlTpl;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static FieldParamYamlTpl convert(FieldParam fieldParam) {
|
||||||
|
FieldParamYamlTpl fieldParamYamlTpl = new FieldParamYamlTpl();
|
||||||
|
fieldParamYamlTpl.setFieldName(fieldParam.getFieldName());
|
||||||
|
return fieldParamYamlTpl;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static MetricParamYamlTpl convert(MetricParam metricParam) {
|
||||||
|
MetricParamYamlTpl metricParamYamlTpl = new MetricParamYamlTpl();
|
||||||
|
metricParamYamlTpl.setBizName(metricParam.getBizName());
|
||||||
|
metricParamYamlTpl.setId(metricParam.getId());
|
||||||
|
return metricParamYamlTpl;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -53,6 +53,7 @@ public class ModelYamlManager {
|
|||||||
} else {
|
} else {
|
||||||
dataModelYamlTpl.setTableQuery(modelDetail.getTableQuery());
|
dataModelYamlTpl.setTableQuery(modelDetail.getTableQuery());
|
||||||
}
|
}
|
||||||
|
dataModelYamlTpl.setFields(modelResp.getModelDetail().getFields());
|
||||||
return dataModelYamlTpl;
|
return dataModelYamlTpl;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -7,6 +7,8 @@ import lombok.Data;
|
|||||||
public class MetricTypeParams {
|
public class MetricTypeParams {
|
||||||
|
|
||||||
private List<Measure> measures;
|
private List<Measure> measures;
|
||||||
|
private List<Measure> metrics;
|
||||||
|
private List<Measure> fields;
|
||||||
|
|
||||||
private String expr;
|
private String expr;
|
||||||
|
|
||||||
|
|||||||
@@ -66,13 +66,13 @@ public class SchemaBuilder {
|
|||||||
String db = dbSrc.toLowerCase();
|
String db = dbSrc.toLowerCase();
|
||||||
DataSourceTable.Builder builder = DataSourceTable.newBuilder(tb);
|
DataSourceTable.Builder builder = DataSourceTable.newBuilder(tb);
|
||||||
for (String date : dates) {
|
for (String date : dates) {
|
||||||
builder.addField(date.toLowerCase(), SqlTypeName.VARCHAR);
|
builder.addField(date, SqlTypeName.VARCHAR);
|
||||||
}
|
}
|
||||||
for (String dim : dimensions) {
|
for (String dim : dimensions) {
|
||||||
builder.addField(dim.toLowerCase(), SqlTypeName.VARCHAR);
|
builder.addField(dim, SqlTypeName.VARCHAR);
|
||||||
}
|
}
|
||||||
for (String metric : metrics) {
|
for (String metric : metrics) {
|
||||||
builder.addField(metric.toLowerCase(), SqlTypeName.BIGINT);
|
builder.addField(metric, SqlTypeName.BIGINT);
|
||||||
}
|
}
|
||||||
DataSourceTable srcTable = builder
|
DataSourceTable srcTable = builder
|
||||||
.withRowCount(1)
|
.withRowCount(1)
|
||||||
|
|||||||
@@ -1,11 +1,15 @@
|
|||||||
package com.tencent.supersonic.headless.core.parser.calcite.sql.node;
|
package com.tencent.supersonic.headless.core.parser.calcite.sql.node;
|
||||||
|
|
||||||
|
import java.util.Objects;
|
||||||
import org.apache.calcite.sql.SqlNode;
|
import org.apache.calcite.sql.SqlNode;
|
||||||
import org.apache.calcite.sql.validate.SqlValidatorScope;
|
import org.apache.calcite.sql.validate.SqlValidatorScope;
|
||||||
|
|
||||||
public class AggFunctionNode extends SemanticNode {
|
public class AggFunctionNode extends SemanticNode {
|
||||||
|
|
||||||
public static SqlNode build(String agg, String name, SqlValidatorScope scope) throws Exception {
|
public static SqlNode build(String agg, String name, SqlValidatorScope scope) throws Exception {
|
||||||
|
if (Objects.isNull(agg) || agg.isEmpty()) {
|
||||||
|
return parse(name, scope);
|
||||||
|
}
|
||||||
if (AggFunction.COUNT_DISTINCT.name().equalsIgnoreCase(agg)) {
|
if (AggFunction.COUNT_DISTINCT.name().equalsIgnoreCase(agg)) {
|
||||||
return parse(AggFunction.COUNT.name() + " ( " + AggFunction.DISTINCT.name() + " " + name + " ) ", scope);
|
return parse(AggFunction.COUNT.name() + " ( " + AggFunction.DISTINCT.name() + " " + name + " ) ", scope);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,8 +11,8 @@ import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension;
|
|||||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Identify;
|
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Identify;
|
||||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.JoinRelation;
|
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.JoinRelation;
|
||||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Measure;
|
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Measure;
|
||||||
import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema;
|
|
||||||
import com.tencent.supersonic.headless.core.parser.calcite.schema.SchemaBuilder;
|
import com.tencent.supersonic.headless.core.parser.calcite.schema.SchemaBuilder;
|
||||||
|
import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema;
|
||||||
import com.tencent.supersonic.headless.core.parser.calcite.sql.node.extend.LateralViewExplodeNode;
|
import com.tencent.supersonic.headless.core.parser.calcite.sql.node.extend.LateralViewExplodeNode;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
@@ -72,7 +72,10 @@ public class DataSourceNode extends SemanticNode {
|
|||||||
String tb = dbTable.length > 1 ? dbTable[1] : dbTable[0];
|
String tb = dbTable.length > 1 ? dbTable[1] : dbTable[0];
|
||||||
String db = dbTable.length > 1 ? dbTable[0] : "";
|
String db = dbTable.length > 1 ? dbTable[0] : "";
|
||||||
addSchemaTable(scope, datasource, db, tb,
|
addSchemaTable(scope, datasource, db, tb,
|
||||||
fields.containsKey(entry.getKey()) ? fields.get(entry.getKey()) : new HashSet<>());
|
fields.containsKey(entry.getKey()) ? fields.get(entry.getKey())
|
||||||
|
: dbTbs.size() == 1 && fields.size() == 1 && fields.containsKey("")
|
||||||
|
? fields.get("")
|
||||||
|
: new HashSet<>());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -47,6 +47,7 @@ import org.apache.calcite.sql.pretty.SqlPrettyWriter;
|
|||||||
import org.apache.calcite.sql.validate.SqlValidator;
|
import org.apache.calcite.sql.validate.SqlValidator;
|
||||||
import org.apache.calcite.sql.validate.SqlValidatorScope;
|
import org.apache.calcite.sql.validate.SqlValidatorScope;
|
||||||
import org.apache.calcite.sql2rel.SqlToRelConverter;
|
import org.apache.calcite.sql2rel.SqlToRelConverter;
|
||||||
|
import org.apache.commons.collections4.CollectionUtils;
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -57,6 +58,7 @@ public abstract class SemanticNode {
|
|||||||
|
|
||||||
public static Set<SqlKind> AGGREGATION_KIND = new HashSet<>();
|
public static Set<SqlKind> AGGREGATION_KIND = new HashSet<>();
|
||||||
public static Set<String> AGGREGATION_FUNC = new HashSet<>();
|
public static Set<String> AGGREGATION_FUNC = new HashSet<>();
|
||||||
|
public static List<String> groupHints = new ArrayList<>(Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8", "9"));
|
||||||
|
|
||||||
static {
|
static {
|
||||||
AGGREGATION_KIND.add(SqlKind.AVG);
|
AGGREGATION_KIND.add(SqlKind.AVG);
|
||||||
@@ -212,6 +214,59 @@ public abstract class SemanticNode {
|
|||||||
fieldVisit(list, parseInfo, "");
|
fieldVisit(list, parseInfo, "");
|
||||||
});
|
});
|
||||||
fromVisit(sqlSelect.getFrom(), parseInfo);
|
fromVisit(sqlSelect.getFrom(), parseInfo);
|
||||||
|
if (sqlSelect.hasWhere()) {
|
||||||
|
whereVisit((SqlBasicCall) sqlSelect.getWhere(), parseInfo);
|
||||||
|
}
|
||||||
|
if (sqlSelect.hasOrderBy()) {
|
||||||
|
fieldVisit(sqlSelect.getOrderList(), parseInfo, "");
|
||||||
|
}
|
||||||
|
SqlNodeList group = sqlSelect.getGroup();
|
||||||
|
if (group != null) {
|
||||||
|
group.forEach(groupField -> {
|
||||||
|
if (groupHints.contains(groupField.toString())) {
|
||||||
|
int groupIdx = Integer.valueOf(groupField.toString()) - 1;
|
||||||
|
if (selectList.getList().size() > groupIdx) {
|
||||||
|
fieldVisit(selectList.get(groupIdx), parseInfo, "");
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
fieldVisit(groupField, parseInfo, "");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void whereVisit(SqlBasicCall where, Map<String, Object> parseInfo) {
|
||||||
|
if (where == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (where.operandCount() == 2 && where.operand(0).getKind().equals(SqlKind.IDENTIFIER)
|
||||||
|
&& where.operand(1).getKind().equals(SqlKind.LITERAL)) {
|
||||||
|
fieldVisit(where.operand(0), parseInfo, "");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// 子查询
|
||||||
|
if (where.operandCount() == 2
|
||||||
|
&& (where.operand(0).getKind().equals(SqlKind.IDENTIFIER)
|
||||||
|
&& (where.operand(1).getKind().equals(SqlKind.SELECT)
|
||||||
|
|| where.operand(1).getKind().equals(SqlKind.ORDER_BY)))
|
||||||
|
) {
|
||||||
|
fieldVisit(where.operand(0), parseInfo, "");
|
||||||
|
sqlVisit((SqlNode) (where.operand(1)), parseInfo);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (CollectionUtils.isNotEmpty(where.getOperandList()) && where.operand(0).getKind()
|
||||||
|
.equals(SqlKind.IDENTIFIER)) {
|
||||||
|
fieldVisit(where.operand(0), parseInfo, "");
|
||||||
|
}
|
||||||
|
if (where.operandCount() >= 2 && where.operand(1).getKind().equals(SqlKind.IDENTIFIER)) {
|
||||||
|
fieldVisit(where.operand(1), parseInfo, "");
|
||||||
|
}
|
||||||
|
if (CollectionUtils.isNotEmpty(where.getOperandList()) && where.operand(0) instanceof SqlBasicCall) {
|
||||||
|
whereVisit(where.operand(0), parseInfo);
|
||||||
|
}
|
||||||
|
if (where.operandCount() >= 2 && where.operand(1) instanceof SqlBasicCall) {
|
||||||
|
whereVisit(where.operand(1), parseInfo);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void fieldVisit(SqlNode field, Map<String, Object> parseInfo, String func) {
|
private static void fieldVisit(SqlNode field, Map<String, Object> parseInfo, String func) {
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package com.tencent.supersonic.headless.core.pojo.yaml;
|
package com.tencent.supersonic.headless.core.pojo.yaml;
|
||||||
|
|
||||||
import com.tencent.supersonic.headless.api.enums.ModelSourceType;
|
import com.tencent.supersonic.headless.api.enums.ModelSourceType;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.Field;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
@@ -27,6 +28,8 @@ public class DataModelYamlTpl {
|
|||||||
|
|
||||||
private List<MeasureYamlTpl> measures;
|
private List<MeasureYamlTpl> measures;
|
||||||
|
|
||||||
|
private List<Field> fields;
|
||||||
|
|
||||||
private ModelSourceType modelSourceTypeEnum;
|
private ModelSourceType modelSourceTypeEnum;
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,10 @@
|
|||||||
|
package com.tencent.supersonic.headless.core.pojo.yaml;
|
||||||
|
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
public class FieldParamYamlTpl {
|
||||||
|
|
||||||
|
private String fieldName;
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
package com.tencent.supersonic.headless.core.pojo.yaml;
|
||||||
|
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
@Data
|
||||||
|
public class MetricParamYamlTpl {
|
||||||
|
|
||||||
|
private Long id;
|
||||||
|
|
||||||
|
private String bizName;
|
||||||
|
|
||||||
|
}
|
||||||
@@ -9,7 +9,10 @@ public class MetricTypeParamsYamlTpl {
|
|||||||
|
|
||||||
private List<MeasureYamlTpl> measures;
|
private List<MeasureYamlTpl> measures;
|
||||||
|
|
||||||
|
private List<MetricParamYamlTpl> metrics;
|
||||||
|
|
||||||
|
private List<FieldParamYamlTpl> fields;
|
||||||
|
|
||||||
private String expr;
|
private String expr;
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,7 +19,6 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper;
|
|||||||
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper;
|
||||||
import com.tencent.supersonic.headless.api.enums.EngineType;
|
import com.tencent.supersonic.headless.api.enums.EngineType;
|
||||||
import com.tencent.supersonic.headless.api.enums.MetricDefineType;
|
import com.tencent.supersonic.headless.api.enums.MetricDefineType;
|
||||||
import com.tencent.supersonic.headless.api.enums.MetricType;
|
|
||||||
import com.tencent.supersonic.headless.api.pojo.Measure;
|
import com.tencent.supersonic.headless.api.pojo.Measure;
|
||||||
import com.tencent.supersonic.headless.api.request.QueryStructReq;
|
import com.tencent.supersonic.headless.api.request.QueryStructReq;
|
||||||
import com.tencent.supersonic.headless.api.response.DimensionResp;
|
import com.tencent.supersonic.headless.api.response.DimensionResp;
|
||||||
@@ -283,8 +282,6 @@ public class SqlGenerateUtils {
|
|||||||
Optional<MetricResp> metricItem = metricResps.stream()
|
Optional<MetricResp> metricItem = metricResps.stream()
|
||||||
.filter(m -> m.getBizName().equalsIgnoreCase(field)).findFirst();
|
.filter(m -> m.getBizName().equalsIgnoreCase(field)).findFirst();
|
||||||
if (metricItem.isPresent()) {
|
if (metricItem.isPresent()) {
|
||||||
if (MetricType.isDerived(metricItem.get().getMetricDefineType(),
|
|
||||||
metricItem.get().getTypeParams())) {
|
|
||||||
if (visitedMetric.contains(field)) {
|
if (visitedMetric.contains(field)) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -293,9 +290,6 @@ public class SqlGenerateUtils {
|
|||||||
getExpr(metricItem.get()), metricItem.get().getMetricDefineType(),
|
getExpr(metricItem.get()), metricItem.get().getMetricDefineType(),
|
||||||
visitedMetric, measures, dimensions));
|
visitedMetric, measures, dimensions));
|
||||||
visitedMetric.add(field);
|
visitedMetric.add(field);
|
||||||
} else {
|
|
||||||
replace.put(field, getExpr(metricItem.get()));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
case MEASURE:
|
case MEASURE:
|
||||||
@@ -321,7 +315,9 @@ public class SqlGenerateUtils {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (!CollectionUtils.isEmpty(replace)) {
|
if (!CollectionUtils.isEmpty(replace)) {
|
||||||
return SqlParserReplaceHelper.replaceExpression(expression, replace);
|
String expr = SqlParserReplaceHelper.replaceExpression(expression, replace);
|
||||||
|
log.info("derived measure {}->{}", expression, expr);
|
||||||
|
return expr;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return expression;
|
return expression;
|
||||||
|
|||||||
@@ -5,39 +5,32 @@ import com.google.common.cache.CacheLoader;
|
|||||||
import com.google.common.cache.LoadingCache;
|
import com.google.common.cache.LoadingCache;
|
||||||
import com.tencent.supersonic.common.pojo.ModelRela;
|
import com.tencent.supersonic.common.pojo.ModelRela;
|
||||||
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
|
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
|
||||||
|
import com.tencent.supersonic.headless.api.pojo.Field;
|
||||||
import com.tencent.supersonic.headless.api.response.DatabaseResp;
|
import com.tencent.supersonic.headless.api.response.DatabaseResp;
|
||||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
|
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
|
||||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
|
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
|
||||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataType;
|
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataType;
|
||||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension;
|
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension;
|
||||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DimensionTimeTypeParams;
|
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DimensionTimeTypeParams;
|
||||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel;
|
|
||||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Identify;
|
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Identify;
|
||||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.JoinRelation;
|
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.JoinRelation;
|
||||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Materialization.TimePartType;
|
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Materialization.TimePartType;
|
||||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Measure;
|
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Measure;
|
||||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Metric;
|
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Metric;
|
||||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.MetricTypeParams;
|
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.MetricTypeParams;
|
||||||
|
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel;
|
||||||
import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema;
|
import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema;
|
||||||
import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl;
|
import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl;
|
||||||
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionTimeTypeParamsTpl;
|
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionTimeTypeParamsTpl;
|
||||||
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
|
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
|
||||||
|
import com.tencent.supersonic.headless.core.pojo.yaml.FieldParamYamlTpl;
|
||||||
import com.tencent.supersonic.headless.core.pojo.yaml.IdentifyYamlTpl;
|
import com.tencent.supersonic.headless.core.pojo.yaml.IdentifyYamlTpl;
|
||||||
import com.tencent.supersonic.headless.core.pojo.yaml.MeasureYamlTpl;
|
import com.tencent.supersonic.headless.core.pojo.yaml.MeasureYamlTpl;
|
||||||
|
import com.tencent.supersonic.headless.core.pojo.yaml.MetricParamYamlTpl;
|
||||||
import com.tencent.supersonic.headless.core.pojo.yaml.MetricTypeParamsYamlTpl;
|
import com.tencent.supersonic.headless.core.pojo.yaml.MetricTypeParamsYamlTpl;
|
||||||
import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
|
import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
|
||||||
import com.tencent.supersonic.headless.server.service.Catalog;
|
import com.tencent.supersonic.headless.server.service.Catalog;
|
||||||
import com.tencent.supersonic.headless.server.utils.DatabaseConverter;
|
import com.tencent.supersonic.headless.server.utils.DatabaseConverter;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
import org.apache.commons.lang3.tuple.Triple;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
|
||||||
import org.springframework.cache.annotation.EnableCaching;
|
|
||||||
import org.springframework.context.annotation.Bean;
|
|
||||||
import org.springframework.context.annotation.Configuration;
|
|
||||||
import org.springframework.stereotype.Service;
|
|
||||||
import org.springframework.util.CollectionUtils;
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
@@ -50,6 +43,16 @@ import java.util.Optional;
|
|||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.apache.commons.lang3.tuple.Triple;
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
|
import org.springframework.cache.annotation.EnableCaching;
|
||||||
|
import org.springframework.context.annotation.Bean;
|
||||||
|
import org.springframework.context.annotation.Configuration;
|
||||||
|
import org.springframework.stereotype.Service;
|
||||||
|
import org.springframework.util.CollectionUtils;
|
||||||
|
|
||||||
|
|
||||||
@Slf4j
|
@Slf4j
|
||||||
@Service
|
@Service
|
||||||
@@ -122,12 +125,27 @@ public class HeadlessSchemaManager {
|
|||||||
public static DataSource getDatasource(final DataModelYamlTpl d) {
|
public static DataSource getDatasource(final DataModelYamlTpl d) {
|
||||||
DataSource datasource = DataSource.builder().id(d.getId()).sourceId(d.getSourceId())
|
DataSource datasource = DataSource.builder().id(d.getId()).sourceId(d.getSourceId())
|
||||||
.type(d.getType()).sqlQuery(d.getSqlQuery()).name(d.getName()).tableQuery(d.getTableQuery())
|
.type(d.getType()).sqlQuery(d.getSqlQuery()).name(d.getName()).tableQuery(d.getTableQuery())
|
||||||
.identifiers(getIdentify(d.getIdentifiers())).measures(getMeasures(d.getMeasures()))
|
.identifiers(getIdentify(d.getIdentifiers())).measures(getMeasureParams(d.getMeasures()))
|
||||||
.dimensions(getDimensions(d.getDimensions())).build();
|
.dimensions(getDimensions(d.getDimensions())).build();
|
||||||
datasource.setAggTime(getDataSourceAggTime(datasource.getDimensions()));
|
datasource.setAggTime(getDataSourceAggTime(datasource.getDimensions()));
|
||||||
if (Objects.nonNull(d.getModelSourceTypeEnum())) {
|
if (Objects.nonNull(d.getModelSourceTypeEnum())) {
|
||||||
datasource.setTimePartType(TimePartType.of(d.getModelSourceTypeEnum().name()));
|
datasource.setTimePartType(TimePartType.of(d.getModelSourceTypeEnum().name()));
|
||||||
}
|
}
|
||||||
|
if (Objects.nonNull(d.getFields()) && !CollectionUtils.isEmpty(d.getFields())) {
|
||||||
|
Set<String> dimensions = datasource.getDimensions().stream().map(dd -> dd.getBizName())
|
||||||
|
.collect(Collectors.toSet());
|
||||||
|
Set<String> measures = datasource.getMeasures().stream().map(mm -> mm.getName())
|
||||||
|
.collect(Collectors.toSet());
|
||||||
|
Set<String> identifiers = datasource.getIdentifiers().stream().map(ii -> ii.getName())
|
||||||
|
.collect(Collectors.toSet());
|
||||||
|
for (Field f : d.getFields()) {
|
||||||
|
if (dimensions.contains(f.getFieldName()) || measures.contains(f.getFieldName())
|
||||||
|
|| identifiers.contains(f.getFieldName())) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
datasource.getMeasures().add(Measure.builder().name(f.getFieldName()).agg("").build());
|
||||||
|
}
|
||||||
|
}
|
||||||
return datasource;
|
return datasource;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -156,11 +174,42 @@ public class HeadlessSchemaManager {
|
|||||||
private static MetricTypeParams getMetricTypeParams(MetricTypeParamsYamlTpl metricTypeParamsYamlTpl) {
|
private static MetricTypeParams getMetricTypeParams(MetricTypeParamsYamlTpl metricTypeParamsYamlTpl) {
|
||||||
MetricTypeParams metricTypeParams = new MetricTypeParams();
|
MetricTypeParams metricTypeParams = new MetricTypeParams();
|
||||||
metricTypeParams.setExpr(metricTypeParamsYamlTpl.getExpr());
|
metricTypeParams.setExpr(metricTypeParamsYamlTpl.getExpr());
|
||||||
metricTypeParams.setMeasures(getMeasures(metricTypeParamsYamlTpl.getMeasures()));
|
if (!CollectionUtils.isEmpty(metricTypeParamsYamlTpl.getMeasures())) {
|
||||||
|
metricTypeParams.setMeasures(getMeasureParams(metricTypeParamsYamlTpl.getMeasures()));
|
||||||
|
}
|
||||||
|
if (!CollectionUtils.isEmpty(metricTypeParamsYamlTpl.getMetrics())) {
|
||||||
|
metricTypeParams.setMeasures(getMetricParams(metricTypeParamsYamlTpl.getMetrics()));
|
||||||
|
}
|
||||||
|
if (!CollectionUtils.isEmpty(metricTypeParamsYamlTpl.getFields())) {
|
||||||
|
metricTypeParams.setMeasures(getFieldParams(metricTypeParamsYamlTpl.getFields()));
|
||||||
|
}
|
||||||
|
|
||||||
return metricTypeParams;
|
return metricTypeParams;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static List<Measure> getMeasures(List<MeasureYamlTpl> measureYamlTpls) {
|
private static List<Measure> getFieldParams(List<FieldParamYamlTpl> fieldParamYamlTpls) {
|
||||||
|
List<Measure> measures = new ArrayList<>();
|
||||||
|
for (FieldParamYamlTpl fieldParamYamlTpl : fieldParamYamlTpls) {
|
||||||
|
Measure measure = new Measure();
|
||||||
|
measure.setName(fieldParamYamlTpl.getFieldName());
|
||||||
|
measure.setExpr(fieldParamYamlTpl.getFieldName());
|
||||||
|
measures.add(measure);
|
||||||
|
}
|
||||||
|
return measures;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static List<Measure> getMetricParams(List<MetricParamYamlTpl> metricParamYamlTpls) {
|
||||||
|
List<Measure> measures = new ArrayList<>();
|
||||||
|
for (MetricParamYamlTpl metricParamYamlTpl : metricParamYamlTpls) {
|
||||||
|
Measure measure = new Measure();
|
||||||
|
measure.setName(metricParamYamlTpl.getBizName());
|
||||||
|
measure.setExpr(metricParamYamlTpl.getBizName());
|
||||||
|
measures.add(measure);
|
||||||
|
}
|
||||||
|
return measures;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static List<Measure> getMeasureParams(List<MeasureYamlTpl> measureYamlTpls) {
|
||||||
List<Measure> measures = new ArrayList<>();
|
List<Measure> measures = new ArrayList<>();
|
||||||
for (MeasureYamlTpl measureYamlTpl : measureYamlTpls) {
|
for (MeasureYamlTpl measureYamlTpl : measureYamlTpls) {
|
||||||
Measure measure = new Measure();
|
Measure measure = new Measure();
|
||||||
@@ -227,7 +276,8 @@ public class HeadlessSchemaManager {
|
|||||||
List<Triple<String, String, String>> conditions = new ArrayList<>();
|
List<Triple<String, String, String>> conditions = new ArrayList<>();
|
||||||
r.getJoinConditions().stream().forEach(rr -> {
|
r.getJoinConditions().stream().forEach(rr -> {
|
||||||
if (FilterOperatorEnum.isValueCompare(rr.getOperator())) {
|
if (FilterOperatorEnum.isValueCompare(rr.getOperator())) {
|
||||||
conditions.add(Triple.of(rr.getLeftField(), rr.getOperator().getValue(), rr.getRightField()));
|
conditions.add(
|
||||||
|
Triple.of(rr.getLeftField(), rr.getOperator().getValue(), rr.getRightField()));
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
joinRelation.setId(r.getId());
|
joinRelation.setId(r.getId());
|
||||||
|
|||||||
@@ -304,6 +304,7 @@ public class QueryReqConverter {
|
|||||||
sqlGenerateUtils.getExpr(metricResp), metricResp.getMetricDefineType(), visitedMetric,
|
sqlGenerateUtils.getExpr(metricResp), metricResp.getMetricDefineType(), visitedMetric,
|
||||||
deriveMetric, deriveDimension);
|
deriveMetric, deriveDimension);
|
||||||
replaces.put(metricResp.getBizName(), expr);
|
replaces.put(metricResp.getBizName(), expr);
|
||||||
|
log.info("derived metric {}->{}", metricResp.getBizName(), expr);
|
||||||
} else {
|
} else {
|
||||||
measures.add(metricResp.getBizName());
|
measures.add(metricResp.getBizName());
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ public class QueryUtils {
|
|||||||
@Value("${query.cache.enable:true}")
|
@Value("${query.cache.enable:true}")
|
||||||
private Boolean cacheEnable;
|
private Boolean cacheEnable;
|
||||||
|
|
||||||
@Value("${query.optimizer.enable:false}")
|
@Value("${query.optimizer.enable:true}")
|
||||||
private Boolean optimizeEnable;
|
private Boolean optimizeEnable;
|
||||||
|
|
||||||
private final CacheUtils cacheUtils;
|
private final CacheUtils cacheUtils;
|
||||||
|
|||||||
Reference in New Issue
Block a user