diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/enums/MetricType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/enums/MetricType.java index ec5735236..52d0ffbc7 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/enums/MetricType.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/enums/MetricType.java @@ -29,6 +29,9 @@ public enum MetricType { if (MetricDefineType.METRIC.equals(metricDefineType)) { return true; } + if (MetricDefineType.FIELD.equals(metricDefineType)) { + return true; + } if (MetricDefineType.MEASURE.equals(metricDefineType)) { List measures = typeParams.getMeasures(); if (measures.size() > 1) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/manager/MetricYamlManager.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/manager/MetricYamlManager.java index 4a753be7a..b5c5bb76e 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/manager/MetricYamlManager.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/manager/MetricYamlManager.java @@ -1,10 +1,17 @@ package com.tencent.supersonic.headless.core.manager; import com.google.common.collect.Lists; +import com.tencent.supersonic.headless.api.enums.MetricDefineType; +import com.tencent.supersonic.headless.api.pojo.FieldParam; import com.tencent.supersonic.headless.api.pojo.MeasureParam; +import com.tencent.supersonic.headless.api.pojo.MetricDefineByFieldParams; import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams; +import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams; +import com.tencent.supersonic.headless.api.pojo.MetricParam; import com.tencent.supersonic.headless.api.response.MetricResp; +import com.tencent.supersonic.headless.core.pojo.yaml.FieldParamYamlTpl; import com.tencent.supersonic.headless.core.pojo.yaml.MeasureYamlTpl; +import com.tencent.supersonic.headless.core.pojo.yaml.MetricParamYamlTpl; import com.tencent.supersonic.headless.core.pojo.yaml.MetricTypeParamsYamlTpl; import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl; import lombok.extern.slf4j.Slf4j; @@ -37,12 +44,26 @@ public class MetricYamlManager { BeanUtils.copyProperties(metric, metricYamlTpl); metricYamlTpl.setName(metric.getBizName()); metricYamlTpl.setOwners(Lists.newArrayList(metric.getCreatedBy())); - MetricDefineByMeasureParams metricDefineParams = metric.getTypeParams(); MetricTypeParamsYamlTpl metricTypeParamsYamlTpl = new MetricTypeParamsYamlTpl(); - metricTypeParamsYamlTpl.setExpr(metricDefineParams.getExpr()); - List measures = metricDefineParams.getMeasures(); - metricTypeParamsYamlTpl.setMeasures( - measures.stream().map(MetricYamlManager::convert).collect(Collectors.toList())); + if (MetricDefineType.MEASURE.equals(metric.getMetricDefineType())) { + MetricDefineByMeasureParams metricDefineParams = metric.getTypeParams(); + metricTypeParamsYamlTpl.setExpr(metricDefineParams.getExpr()); + List measures = metricDefineParams.getMeasures(); + metricTypeParamsYamlTpl.setMeasures( + measures.stream().map(MetricYamlManager::convert).collect(Collectors.toList())); + } else if (MetricDefineType.FIELD.equals(metric.getMetricDefineType())) { + MetricDefineByFieldParams metricDefineParams = metric.getMetricDefineByFieldParams(); + metricTypeParamsYamlTpl.setExpr(metricDefineParams.getExpr()); + List fields = metricDefineParams.getFields(); + metricTypeParamsYamlTpl.setFields( + fields.stream().map(MetricYamlManager::convert).collect(Collectors.toList())); + } else if (MetricDefineType.METRIC.equals(metric.getMetricDefineType())) { + MetricDefineByMetricParams metricDefineByMetricParams = metric.getMetricDefineByMetricParams(); + metricTypeParamsYamlTpl.setExpr(metricDefineByMetricParams.getExpr()); + List metrics = metricDefineByMetricParams.getMetrics(); + metricTypeParamsYamlTpl.setMetrics( + metrics.stream().map(MetricYamlManager::convert).collect(Collectors.toList())); + } metricYamlTpl.setTypeParams(metricTypeParamsYamlTpl); return metricYamlTpl; } @@ -55,4 +76,17 @@ public class MetricYamlManager { return measureYamlTpl; } + public static FieldParamYamlTpl convert(FieldParam fieldParam) { + FieldParamYamlTpl fieldParamYamlTpl = new FieldParamYamlTpl(); + fieldParamYamlTpl.setFieldName(fieldParam.getFieldName()); + return fieldParamYamlTpl; + } + + public static MetricParamYamlTpl convert(MetricParam metricParam) { + MetricParamYamlTpl metricParamYamlTpl = new MetricParamYamlTpl(); + metricParamYamlTpl.setBizName(metricParam.getBizName()); + metricParamYamlTpl.setId(metricParam.getId()); + return metricParamYamlTpl; + } + } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/manager/ModelYamlManager.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/manager/ModelYamlManager.java index 28c1b940d..91ae2590b 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/manager/ModelYamlManager.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/manager/ModelYamlManager.java @@ -53,6 +53,7 @@ public class ModelYamlManager { } else { dataModelYamlTpl.setTableQuery(modelDetail.getTableQuery()); } + dataModelYamlTpl.setFields(modelResp.getModelDetail().getFields()); return dataModelYamlTpl; } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/MetricTypeParams.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/MetricTypeParams.java index 6c250550d..3242ff1c6 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/MetricTypeParams.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/s2sql/MetricTypeParams.java @@ -7,6 +7,8 @@ import lombok.Data; public class MetricTypeParams { private List measures; + private List metrics; + private List fields; private String expr; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SchemaBuilder.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SchemaBuilder.java index 940bdbe0c..d4f11cef6 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SchemaBuilder.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/schema/SchemaBuilder.java @@ -66,13 +66,13 @@ public class SchemaBuilder { String db = dbSrc.toLowerCase(); DataSourceTable.Builder builder = DataSourceTable.newBuilder(tb); for (String date : dates) { - builder.addField(date.toLowerCase(), SqlTypeName.VARCHAR); + builder.addField(date, SqlTypeName.VARCHAR); } for (String dim : dimensions) { - builder.addField(dim.toLowerCase(), SqlTypeName.VARCHAR); + builder.addField(dim, SqlTypeName.VARCHAR); } for (String metric : metrics) { - builder.addField(metric.toLowerCase(), SqlTypeName.BIGINT); + builder.addField(metric, SqlTypeName.BIGINT); } DataSourceTable srcTable = builder .withRowCount(1) diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/AggFunctionNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/AggFunctionNode.java index c3204d2d6..d311150cf 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/AggFunctionNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/AggFunctionNode.java @@ -1,11 +1,15 @@ package com.tencent.supersonic.headless.core.parser.calcite.sql.node; +import java.util.Objects; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.validate.SqlValidatorScope; public class AggFunctionNode extends SemanticNode { public static SqlNode build(String agg, String name, SqlValidatorScope scope) throws Exception { + if (Objects.isNull(agg) || agg.isEmpty()) { + return parse(name, scope); + } if (AggFunction.COUNT_DISTINCT.name().equalsIgnoreCase(agg)) { return parse(AggFunction.COUNT.name() + " ( " + AggFunction.DISTINCT.name() + " " + name + " ) ", scope); } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DataSourceNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DataSourceNode.java index a815da7aa..98839ce9e 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DataSourceNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/DataSourceNode.java @@ -11,8 +11,8 @@ import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Identify; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.JoinRelation; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Measure; -import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.parser.calcite.schema.SchemaBuilder; +import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.parser.calcite.sql.node.extend.LateralViewExplodeNode; import java.util.ArrayList; import java.util.Arrays; @@ -72,7 +72,10 @@ public class DataSourceNode extends SemanticNode { String tb = dbTable.length > 1 ? dbTable[1] : dbTable[0]; String db = dbTable.length > 1 ? dbTable[0] : ""; addSchemaTable(scope, datasource, db, tb, - fields.containsKey(entry.getKey()) ? fields.get(entry.getKey()) : new HashSet<>()); + fields.containsKey(entry.getKey()) ? fields.get(entry.getKey()) + : dbTbs.size() == 1 && fields.size() == 1 && fields.containsKey("") + ? fields.get("") + : new HashSet<>()); } } } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/SemanticNode.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/SemanticNode.java index 630bfd215..f476a8214 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/SemanticNode.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/calcite/sql/node/SemanticNode.java @@ -47,6 +47,7 @@ import org.apache.calcite.sql.pretty.SqlPrettyWriter; import org.apache.calcite.sql.validate.SqlValidator; import org.apache.calcite.sql.validate.SqlValidatorScope; import org.apache.calcite.sql2rel.SqlToRelConverter; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; /** @@ -57,6 +58,7 @@ public abstract class SemanticNode { public static Set AGGREGATION_KIND = new HashSet<>(); public static Set AGGREGATION_FUNC = new HashSet<>(); + public static List groupHints = new ArrayList<>(Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8", "9")); static { AGGREGATION_KIND.add(SqlKind.AVG); @@ -212,6 +214,59 @@ public abstract class SemanticNode { fieldVisit(list, parseInfo, ""); }); fromVisit(sqlSelect.getFrom(), parseInfo); + if (sqlSelect.hasWhere()) { + whereVisit((SqlBasicCall) sqlSelect.getWhere(), parseInfo); + } + if (sqlSelect.hasOrderBy()) { + fieldVisit(sqlSelect.getOrderList(), parseInfo, ""); + } + SqlNodeList group = sqlSelect.getGroup(); + if (group != null) { + group.forEach(groupField -> { + if (groupHints.contains(groupField.toString())) { + int groupIdx = Integer.valueOf(groupField.toString()) - 1; + if (selectList.getList().size() > groupIdx) { + fieldVisit(selectList.get(groupIdx), parseInfo, ""); + } + } else { + fieldVisit(groupField, parseInfo, ""); + } + }); + } + } + + private static void whereVisit(SqlBasicCall where, Map parseInfo) { + if (where == null) { + return; + } + if (where.operandCount() == 2 && where.operand(0).getKind().equals(SqlKind.IDENTIFIER) + && where.operand(1).getKind().equals(SqlKind.LITERAL)) { + fieldVisit(where.operand(0), parseInfo, ""); + return; + } + // 子查询 + if (where.operandCount() == 2 + && (where.operand(0).getKind().equals(SqlKind.IDENTIFIER) + && (where.operand(1).getKind().equals(SqlKind.SELECT) + || where.operand(1).getKind().equals(SqlKind.ORDER_BY))) + ) { + fieldVisit(where.operand(0), parseInfo, ""); + sqlVisit((SqlNode) (where.operand(1)), parseInfo); + return; + } + if (CollectionUtils.isNotEmpty(where.getOperandList()) && where.operand(0).getKind() + .equals(SqlKind.IDENTIFIER)) { + fieldVisit(where.operand(0), parseInfo, ""); + } + if (where.operandCount() >= 2 && where.operand(1).getKind().equals(SqlKind.IDENTIFIER)) { + fieldVisit(where.operand(1), parseInfo, ""); + } + if (CollectionUtils.isNotEmpty(where.getOperandList()) && where.operand(0) instanceof SqlBasicCall) { + whereVisit(where.operand(0), parseInfo); + } + if (where.operandCount() >= 2 && where.operand(1) instanceof SqlBasicCall) { + whereVisit(where.operand(1), parseInfo); + } } private static void fieldVisit(SqlNode field, Map parseInfo, String func) { diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/DataModelYamlTpl.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/DataModelYamlTpl.java index 6db7e9743..2b00e52d3 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/DataModelYamlTpl.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/DataModelYamlTpl.java @@ -1,6 +1,7 @@ package com.tencent.supersonic.headless.core.pojo.yaml; import com.tencent.supersonic.headless.api.enums.ModelSourceType; +import com.tencent.supersonic.headless.api.pojo.Field; import lombok.Data; import java.util.List; @@ -27,6 +28,8 @@ public class DataModelYamlTpl { private List measures; + private List fields; + private ModelSourceType modelSourceTypeEnum; diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/FieldParamYamlTpl.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/FieldParamYamlTpl.java new file mode 100644 index 000000000..26a08bdbf --- /dev/null +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/FieldParamYamlTpl.java @@ -0,0 +1,10 @@ +package com.tencent.supersonic.headless.core.pojo.yaml; + +import lombok.Data; + +@Data +public class FieldParamYamlTpl { + + private String fieldName; + +} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/MetricParamYamlTpl.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/MetricParamYamlTpl.java new file mode 100644 index 000000000..9f84ae9c7 --- /dev/null +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/MetricParamYamlTpl.java @@ -0,0 +1,12 @@ +package com.tencent.supersonic.headless.core.pojo.yaml; + +import lombok.Data; + +@Data +public class MetricParamYamlTpl { + + private Long id; + + private String bizName; + +} diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/MetricTypeParamsYamlTpl.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/MetricTypeParamsYamlTpl.java index daf314e67..832947c99 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/MetricTypeParamsYamlTpl.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/MetricTypeParamsYamlTpl.java @@ -9,7 +9,10 @@ public class MetricTypeParamsYamlTpl { private List measures; + private List metrics; + + private List fields; + private String expr; - } diff --git a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java index 191b83376..264931a61 100644 --- a/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java +++ b/headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/SqlGenerateUtils.java @@ -19,7 +19,6 @@ import com.tencent.supersonic.common.util.jsqlparser.SqlParserReplaceHelper; import com.tencent.supersonic.common.util.jsqlparser.SqlParserSelectHelper; import com.tencent.supersonic.headless.api.enums.EngineType; import com.tencent.supersonic.headless.api.enums.MetricDefineType; -import com.tencent.supersonic.headless.api.enums.MetricType; import com.tencent.supersonic.headless.api.pojo.Measure; import com.tencent.supersonic.headless.api.request.QueryStructReq; import com.tencent.supersonic.headless.api.response.DimensionResp; @@ -283,19 +282,14 @@ public class SqlGenerateUtils { Optional metricItem = metricResps.stream() .filter(m -> m.getBizName().equalsIgnoreCase(field)).findFirst(); if (metricItem.isPresent()) { - if (MetricType.isDerived(metricItem.get().getMetricDefineType(), - metricItem.get().getTypeParams())) { - if (visitedMetric.contains(field)) { - break; - } - replace.put(field, - generateDerivedMetric(metricResps, allFields, allMeasures, dimensionResps, - getExpr(metricItem.get()), metricItem.get().getMetricDefineType(), - visitedMetric, measures, dimensions)); - visitedMetric.add(field); - } else { - replace.put(field, getExpr(metricItem.get())); + if (visitedMetric.contains(field)) { + break; } + replace.put(field, + generateDerivedMetric(metricResps, allFields, allMeasures, dimensionResps, + getExpr(metricItem.get()), metricItem.get().getMetricDefineType(), + visitedMetric, measures, dimensions)); + visitedMetric.add(field); } break; case MEASURE: @@ -321,7 +315,9 @@ public class SqlGenerateUtils { } } if (!CollectionUtils.isEmpty(replace)) { - return SqlParserReplaceHelper.replaceExpression(expression, replace); + String expr = SqlParserReplaceHelper.replaceExpression(expression, replace); + log.info("derived measure {}->{}", expression, expr); + return expr; } } return expression; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/HeadlessSchemaManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/HeadlessSchemaManager.java index d195aae7e..491e36f2f 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/HeadlessSchemaManager.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/HeadlessSchemaManager.java @@ -5,39 +5,32 @@ import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.tencent.supersonic.common.pojo.ModelRela; import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum; +import com.tencent.supersonic.headless.api.pojo.Field; import com.tencent.supersonic.headless.api.response.DatabaseResp; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataType; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DimensionTimeTypeParams; -import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Identify; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.JoinRelation; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Materialization.TimePartType; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Measure; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Metric; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.MetricTypeParams; +import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel; import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema; import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl; import com.tencent.supersonic.headless.core.pojo.yaml.DimensionTimeTypeParamsTpl; import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl; +import com.tencent.supersonic.headless.core.pojo.yaml.FieldParamYamlTpl; import com.tencent.supersonic.headless.core.pojo.yaml.IdentifyYamlTpl; import com.tencent.supersonic.headless.core.pojo.yaml.MeasureYamlTpl; +import com.tencent.supersonic.headless.core.pojo.yaml.MetricParamYamlTpl; import com.tencent.supersonic.headless.core.pojo.yaml.MetricTypeParamsYamlTpl; import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl; import com.tencent.supersonic.headless.server.service.Catalog; import com.tencent.supersonic.headless.server.utils.DatabaseConverter; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.tuple.Triple; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.cache.annotation.EnableCaching; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; - import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -50,6 +43,16 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Triple; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.cache.annotation.EnableCaching; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + @Slf4j @Service @@ -122,12 +125,27 @@ public class HeadlessSchemaManager { public static DataSource getDatasource(final DataModelYamlTpl d) { DataSource datasource = DataSource.builder().id(d.getId()).sourceId(d.getSourceId()) .type(d.getType()).sqlQuery(d.getSqlQuery()).name(d.getName()).tableQuery(d.getTableQuery()) - .identifiers(getIdentify(d.getIdentifiers())).measures(getMeasures(d.getMeasures())) + .identifiers(getIdentify(d.getIdentifiers())).measures(getMeasureParams(d.getMeasures())) .dimensions(getDimensions(d.getDimensions())).build(); datasource.setAggTime(getDataSourceAggTime(datasource.getDimensions())); if (Objects.nonNull(d.getModelSourceTypeEnum())) { datasource.setTimePartType(TimePartType.of(d.getModelSourceTypeEnum().name())); } + if (Objects.nonNull(d.getFields()) && !CollectionUtils.isEmpty(d.getFields())) { + Set dimensions = datasource.getDimensions().stream().map(dd -> dd.getBizName()) + .collect(Collectors.toSet()); + Set measures = datasource.getMeasures().stream().map(mm -> mm.getName()) + .collect(Collectors.toSet()); + Set identifiers = datasource.getIdentifiers().stream().map(ii -> ii.getName()) + .collect(Collectors.toSet()); + for (Field f : d.getFields()) { + if (dimensions.contains(f.getFieldName()) || measures.contains(f.getFieldName()) + || identifiers.contains(f.getFieldName())) { + continue; + } + datasource.getMeasures().add(Measure.builder().name(f.getFieldName()).agg("").build()); + } + } return datasource; } @@ -156,11 +174,42 @@ public class HeadlessSchemaManager { private static MetricTypeParams getMetricTypeParams(MetricTypeParamsYamlTpl metricTypeParamsYamlTpl) { MetricTypeParams metricTypeParams = new MetricTypeParams(); metricTypeParams.setExpr(metricTypeParamsYamlTpl.getExpr()); - metricTypeParams.setMeasures(getMeasures(metricTypeParamsYamlTpl.getMeasures())); + if (!CollectionUtils.isEmpty(metricTypeParamsYamlTpl.getMeasures())) { + metricTypeParams.setMeasures(getMeasureParams(metricTypeParamsYamlTpl.getMeasures())); + } + if (!CollectionUtils.isEmpty(metricTypeParamsYamlTpl.getMetrics())) { + metricTypeParams.setMeasures(getMetricParams(metricTypeParamsYamlTpl.getMetrics())); + } + if (!CollectionUtils.isEmpty(metricTypeParamsYamlTpl.getFields())) { + metricTypeParams.setMeasures(getFieldParams(metricTypeParamsYamlTpl.getFields())); + } + return metricTypeParams; } - private static List getMeasures(List measureYamlTpls) { + private static List getFieldParams(List fieldParamYamlTpls) { + List measures = new ArrayList<>(); + for (FieldParamYamlTpl fieldParamYamlTpl : fieldParamYamlTpls) { + Measure measure = new Measure(); + measure.setName(fieldParamYamlTpl.getFieldName()); + measure.setExpr(fieldParamYamlTpl.getFieldName()); + measures.add(measure); + } + return measures; + } + + private static List getMetricParams(List metricParamYamlTpls) { + List measures = new ArrayList<>(); + for (MetricParamYamlTpl metricParamYamlTpl : metricParamYamlTpls) { + Measure measure = new Measure(); + measure.setName(metricParamYamlTpl.getBizName()); + measure.setExpr(metricParamYamlTpl.getBizName()); + measures.add(measure); + } + return measures; + } + + private static List getMeasureParams(List measureYamlTpls) { List measures = new ArrayList<>(); for (MeasureYamlTpl measureYamlTpl : measureYamlTpls) { Measure measure = new Measure(); @@ -227,7 +276,8 @@ public class HeadlessSchemaManager { List> conditions = new ArrayList<>(); r.getJoinConditions().stream().forEach(rr -> { if (FilterOperatorEnum.isValueCompare(rr.getOperator())) { - conditions.add(Triple.of(rr.getLeftField(), rr.getOperator().getValue(), rr.getRightField())); + conditions.add( + Triple.of(rr.getLeftField(), rr.getOperator().getValue(), rr.getRightField())); } }); joinRelation.setId(r.getId()); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java index 21381388a..85db71323 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java @@ -304,6 +304,7 @@ public class QueryReqConverter { sqlGenerateUtils.getExpr(metricResp), metricResp.getMetricDefineType(), visitedMetric, deriveMetric, deriveDimension); replaces.put(metricResp.getBizName(), expr); + log.info("derived metric {}->{}", metricResp.getBizName(), expr); } else { measures.add(metricResp.getBizName()); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java index 29b1b1f1b..6e47dd672 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java @@ -49,7 +49,7 @@ public class QueryUtils { @Value("${query.cache.enable:true}") private Boolean cacheEnable; - @Value("${query.optimizer.enable:false}") + @Value("${query.optimizer.enable:true}") private Boolean optimizeEnable; private final CacheUtils cacheUtils;