topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-14 13:47:09 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

(improvement)(headless) parser add model field (#650)

Browse Source
This commit is contained in:
jipeli
2024-01-18 21:52:36 +08:00
committed by GitHub
parent 90f9da162e
commit b019f4d9bb
16 changed files with 218 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/HeadlessSchemaManager.java
View File

@@ -5,39 +5,32 @@ import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.tencent.supersonic.common.pojo.ModelRela;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.headless.api.pojo.Field;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataType;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DimensionTimeTypeParams;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Identify;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.JoinRelation;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Materialization.TimePartType;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Measure;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Metric;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.MetricTypeParams;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel;
import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema;
import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionTimeTypeParamsTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.FieldParamYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.IdentifyYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricParamYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
import com.tencent.supersonic.headless.server.service.Catalog;
import com.tencent.supersonic.headless.server.utils.DatabaseConverter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Triple;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -50,6 +43,16 @@ import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Triple;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
@Slf4j
@Service
@@ -122,12 +125,27 @@ public class HeadlessSchemaManager {
public static DataSource getDatasource(final DataModelYamlTpl d) {
DataSource datasource = DataSource.builder().id(d.getId()).sourceId(d.getSourceId())
.type(d.getType()).sqlQuery(d.getSqlQuery()).name(d.getName()).tableQuery(d.getTableQuery())
.identifiers(getIdentify(d.getIdentifiers())).measures(getMeasures(d.getMeasures()))
.identifiers(getIdentify(d.getIdentifiers())).measures(getMeasureParams(d.getMeasures()))
.dimensions(getDimensions(d.getDimensions())).build();
datasource.setAggTime(getDataSourceAggTime(datasource.getDimensions()));
if (Objects.nonNull(d.getModelSourceTypeEnum())) {
datasource.setTimePartType(TimePartType.of(d.getModelSourceTypeEnum().name()));
}
if (Objects.nonNull(d.getFields()) && !CollectionUtils.isEmpty(d.getFields())) {
Set<String> dimensions = datasource.getDimensions().stream().map(dd -> dd.getBizName())
.collect(Collectors.toSet());
Set<String> measures = datasource.getMeasures().stream().map(mm -> mm.getName())
.collect(Collectors.toSet());
Set<String> identifiers = datasource.getIdentifiers().stream().map(ii -> ii.getName())
.collect(Collectors.toSet());
for (Field f : d.getFields()) {
if (dimensions.contains(f.getFieldName()) || measures.contains(f.getFieldName())
|| identifiers.contains(f.getFieldName())) {
continue;
}
datasource.getMeasures().add(Measure.builder().name(f.getFieldName()).agg("").build());
}
}
return datasource;
}
@@ -156,11 +174,42 @@ public class HeadlessSchemaManager {
private static MetricTypeParams getMetricTypeParams(MetricTypeParamsYamlTpl metricTypeParamsYamlTpl) {
MetricTypeParams metricTypeParams = new MetricTypeParams();
metricTypeParams.setExpr(metricTypeParamsYamlTpl.getExpr());
metricTypeParams.setMeasures(getMeasures(metricTypeParamsYamlTpl.getMeasures()));
if (!CollectionUtils.isEmpty(metricTypeParamsYamlTpl.getMeasures())) {
metricTypeParams.setMeasures(getMeasureParams(metricTypeParamsYamlTpl.getMeasures()));
}
if (!CollectionUtils.isEmpty(metricTypeParamsYamlTpl.getMetrics())) {
metricTypeParams.setMeasures(getMetricParams(metricTypeParamsYamlTpl.getMetrics()));
}
if (!CollectionUtils.isEmpty(metricTypeParamsYamlTpl.getFields())) {
metricTypeParams.setMeasures(getFieldParams(metricTypeParamsYamlTpl.getFields()));
}
return metricTypeParams;
}
private static List<Measure> getMeasures(List<MeasureYamlTpl> measureYamlTpls) {
private static List<Measure> getFieldParams(List<FieldParamYamlTpl> fieldParamYamlTpls) {
List<Measure> measures = new ArrayList<>();
for (FieldParamYamlTpl fieldParamYamlTpl : fieldParamYamlTpls) {
Measure measure = new Measure();
measure.setName(fieldParamYamlTpl.getFieldName());
measure.setExpr(fieldParamYamlTpl.getFieldName());
measures.add(measure);
}
return measures;
}
private static List<Measure> getMetricParams(List<MetricParamYamlTpl> metricParamYamlTpls) {
List<Measure> measures = new ArrayList<>();
for (MetricParamYamlTpl metricParamYamlTpl : metricParamYamlTpls) {
Measure measure = new Measure();
measure.setName(metricParamYamlTpl.getBizName());
measure.setExpr(metricParamYamlTpl.getBizName());
measures.add(measure);
}
return measures;
}
private static List<Measure> getMeasureParams(List<MeasureYamlTpl> measureYamlTpls) {
List<Measure> measures = new ArrayList<>();
for (MeasureYamlTpl measureYamlTpl : measureYamlTpls) {
Measure measure = new Measure();
@@ -227,7 +276,8 @@ public class HeadlessSchemaManager {
List<Triple<String, String, String>> conditions = new ArrayList<>();
r.getJoinConditions().stream().forEach(rr -> {
if (FilterOperatorEnum.isValueCompare(rr.getOperator())) {
conditions.add(Triple.of(rr.getLeftField(), rr.getOperator().getValue(), rr.getRightField()));
conditions.add(
Triple.of(rr.getLeftField(), rr.getOperator().getValue(), rr.getRightField()));
}
});
joinRelation.setId(r.getId());

1
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryReqConverter.java
View File

@@ -304,6 +304,7 @@ public class QueryReqConverter {
sqlGenerateUtils.getExpr(metricResp), metricResp.getMetricDefineType(), visitedMetric,
deriveMetric, deriveDimension);
replaces.put(metricResp.getBizName(), expr);
log.info("derived metric {}->{}", metricResp.getBizName(), expr);
} else {
measures.add(metricResp.getBizName());
}

2
headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/QueryUtils.java
View File

@@ -49,7 +49,7 @@ public class QueryUtils {
@Value("${query.cache.enable:true}")
private Boolean cacheEnable;
@Value("${query.optimizer.enable:false}")
@Value("${query.optimizer.enable:true}")
private Boolean optimizeEnable;
private final CacheUtils cacheUtils;