topdog/supersonic
1
0
Fork 0
mirror of https://github.com/tencentmusic/supersonic.git synced 2025-12-14 13:47:09 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Make some refactoring in Headless (#672)

Browse Source 
* [improvement][headless]Move QueryCache from server to core and move yaml-related classes from core to server

* [improvement][headless]Declare QueryParser and QueryPlanner as interface instead of class.

---------

Co-authored-by: LXW
This commit is contained in:
Jun Zhang
2024-01-21 16:14:39 +08:00
committed by GitHub
parent 7afa42b4bc
commit 97c767a45b
37 changed files with 275 additions and 255 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/CacheCommonConfig.java vendored Normal file
View File

@@ -0,0 +1,24 @@
package com.tencent.supersonic.headless.core.cache;
import lombok.Data;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Configuration;
@Configuration
@Data
public class CacheCommonConfig {
@Value("${cache.common.app:supersonic}")
private String cacheCommonApp;
@Value("${cache.common.env:dev}")
private String cacheCommonEnv;
@Value("${cache.common.version:0}")
private Integer cacheCommonVersion;
@Value("${cache.common.expire.after.write:10}")
private Integer cacheCommonExpireAfterWrite;
}

14
headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/CacheManager.java vendored Normal file
View File

@@ -0,0 +1,14 @@
package com.tencent.supersonic.headless.core.cache;
public interface CacheManager {
Boolean put(String key, Object value);
Object get(String key);
String generateCacheKey(String prefix, String body);
Boolean removeCache(String key);
}

40
headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/CaffeineCacheConfig.java vendored Normal file
View File

@@ -0,0 +1,40 @@
package com.tencent.supersonic.headless.core.cache;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import java.util.concurrent.TimeUnit;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class CaffeineCacheConfig {
@Autowired
private CacheCommonConfig cacheCommonConfig;
@Value("${caffeine.initial.capacity:500}")
private Integer caffeineInitialCapacity;
@Value("${caffeine.max.size:5000}")
private Integer caffeineMaximumSize;
@Bean(name = "caffeineCache")
public Cache<String, Object> caffeineCache() {
return Caffeine.newBuilder()
.expireAfterWrite(cacheCommonConfig.getCacheCommonExpireAfterWrite(), TimeUnit.MINUTES)
.initialCapacity(caffeineInitialCapacity)
.maximumSize(caffeineMaximumSize)
.build();
}
@Bean(name = "searchCaffeineCache")
public Cache<Long, Object> searchCaffeineCache() {
return Caffeine.newBuilder()
.expireAfterWrite(10000, TimeUnit.MINUTES)
.initialCapacity(caffeineInitialCapacity)
.maximumSize(caffeineMaximumSize)
.build();
}
}

51
headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/CaffeineCacheManager.java vendored Normal file
View File

@@ -0,0 +1,51 @@
package com.tencent.supersonic.headless.core.cache;
import com.github.benmanes.caffeine.cache.Cache;
import com.google.common.base.Joiner;
import lombok.extern.slf4j.Slf4j;
import org.apache.logging.log4j.util.Strings;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;
@Component
@Slf4j
public class CaffeineCacheManager implements CacheManager {
@Autowired
private CacheCommonConfig cacheCommonConfig;
@Autowired
@Qualifier("caffeineCache")
private Cache<String, Object> caffeineCache;
@Override
public Boolean put(String key, Object value) {
log.info("[put caffeineCache] key:{}, value:{}", key, value);
caffeineCache.put(key, value);
return true;
}
@Override
public Object get(String key) {
Object value = caffeineCache.asMap().get(key);
log.info("[get caffeineCache] key:{}, value:{}", key, value);
return value;
}
@Override
public String generateCacheKey(String prefix, String body) {
if (Strings.isEmpty(prefix)) {
prefix = "-1";
}
return Joiner.on(":").join(cacheCommonConfig.getCacheCommonApp(), cacheCommonConfig.getCacheCommonEnv(),
cacheCommonConfig.getCacheCommonVersion(), prefix, body);
}
@Override
public Boolean removeCache(String key) {
caffeineCache.asMap().remove(key);
return true;
}
}

78
headless/core/src/main/java/com/tencent/supersonic/headless/core/cache/QueryCache.java vendored Normal file
View File

@@ -0,0 +1,78 @@
package com.tencent.supersonic.headless.core.cache;
import com.tencent.supersonic.headless.api.pojo.Cache;
import com.tencent.supersonic.headless.api.request.SemanticQueryReq;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@Component
@Slf4j
public class QueryCache {
@Value("${query.cache.enable:true}")
private Boolean cacheEnable;
@Autowired
private CacheManager cacheManager;
public Object query(SemanticQueryReq semanticQueryReq) {
String cacheKey = getCacheKey(semanticQueryReq);
handleGlobalCacheDisable(semanticQueryReq);
boolean isCache = isCache(semanticQueryReq);
if (isCache) {
Object result = cacheManager.get(cacheKey);
log.info("queryFromCache, key:{}, semanticQueryReq:{}", cacheKey, semanticQueryReq);
return result;
}
return null;
}
public Boolean put(SemanticQueryReq semanticQueryReq, Object value) {
if (cacheEnable && Objects.nonNull(value)) {
String key = getCacheKey(semanticQueryReq);
CompletableFuture.supplyAsync(() -> cacheManager.put(key, value))
.exceptionally(exception -> {
log.warn("exception:", exception);
return null;
});
log.info("add record to cache, key:{}", key);
return true;
}
return false;
}
public String getCacheKey(SemanticQueryReq semanticQueryReq) {
String commandMd5 = semanticQueryReq.generateCommandMd5();
String keyByModelIds = getKeyByModelIds(semanticQueryReq.getModelIds());
return cacheManager.generateCacheKey(keyByModelIds, commandMd5);
}
private void handleGlobalCacheDisable(SemanticQueryReq semanticQueryReq) {
if (!cacheEnable) {
Cache cacheInfo = new Cache();
cacheInfo.setCache(false);
semanticQueryReq.setCacheInfo(cacheInfo);
}
}
private String getKeyByModelIds(List<Long> modelIds) {
return String.join(",", modelIds.stream().map(Object::toString).collect(Collectors.toList()));
}
private boolean isCache(SemanticQueryReq semanticQueryReq) {
if (!cacheEnable) {
return false;
}
if (semanticQueryReq.getCacheInfo() != null) {
return semanticQueryReq.getCacheInfo().getCache();
}
return false;
}
}

40
headless/core/src/main/java/com/tencent/supersonic/headless/core/manager/DimensionYamlManager.java
View File

@@ -1,40 +0,0 @@
package com.tencent.supersonic.headless.core.manager;
import com.tencent.supersonic.headless.api.enums.IdentifyType;
import com.tencent.supersonic.headless.api.response.DimensionResp;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
/**
* manager to handle the dimension
*/
@Slf4j
@Service
public class DimensionYamlManager {
public static List<DimensionYamlTpl> convert2DimensionYaml(List<DimensionResp> dimensions) {
if (CollectionUtils.isEmpty(dimensions)) {
return new ArrayList<>();
}
return dimensions.stream()
.filter(dimension -> !dimension.getType().equalsIgnoreCase(IdentifyType.primary.name()))
.map(DimensionYamlManager::convert2DimensionYamlTpl).collect(Collectors.toList());
}
public static DimensionYamlTpl convert2DimensionYamlTpl(DimensionResp dimension) {
DimensionYamlTpl dimensionYamlTpl = new DimensionYamlTpl();
BeanUtils.copyProperties(dimension, dimensionYamlTpl);
dimensionYamlTpl.setName(dimension.getBizName());
dimensionYamlTpl.setOwners(dimension.getCreatedBy());
return dimensionYamlTpl;
}
}

92
headless/core/src/main/java/com/tencent/supersonic/headless/core/manager/MetricYamlManager.java
View File

@@ -1,92 +0,0 @@
package com.tencent.supersonic.headless.core.manager;
import com.google.common.collect.Lists;
import com.tencent.supersonic.headless.api.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.pojo.FieldParam;
import com.tencent.supersonic.headless.api.pojo.MeasureParam;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByFieldParams;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMeasureParams;
import com.tencent.supersonic.headless.api.pojo.MetricDefineByMetricParams;
import com.tencent.supersonic.headless.api.pojo.MetricParam;
import com.tencent.supersonic.headless.api.response.MetricResp;
import com.tencent.supersonic.headless.core.pojo.yaml.FieldParamYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricParamYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricTypeParamsYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MetricYamlTpl;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
/**
* manager to handle the metric
*/
@Slf4j
@Service
public class MetricYamlManager {
public static List<MetricYamlTpl> convert2YamlObj(List<MetricResp> metrics) {
List<MetricYamlTpl> metricYamlTpls = new ArrayList<>();
for (MetricResp metric : metrics) {
MetricYamlTpl metricYamlTpl = convert2MetricYamlTpl(metric);
metricYamlTpls.add(metricYamlTpl);
}
return metricYamlTpls;
}
public static MetricYamlTpl convert2MetricYamlTpl(MetricResp metric) {
MetricYamlTpl metricYamlTpl = new MetricYamlTpl();
BeanUtils.copyProperties(metric, metricYamlTpl);
metricYamlTpl.setName(metric.getBizName());
metricYamlTpl.setOwners(Lists.newArrayList(metric.getCreatedBy()));
MetricTypeParamsYamlTpl metricTypeParamsYamlTpl = new MetricTypeParamsYamlTpl();
if (MetricDefineType.MEASURE.equals(metric.getMetricDefineType())) {
MetricDefineByMeasureParams metricDefineParams = metric.getMetricDefineByMeasureParams();
metricTypeParamsYamlTpl.setExpr(metricDefineParams.getExpr());
List<MeasureParam> measures = metricDefineParams.getMeasures();
metricTypeParamsYamlTpl.setMeasures(
measures.stream().map(MetricYamlManager::convert).collect(Collectors.toList()));
} else if (MetricDefineType.FIELD.equals(metric.getMetricDefineType())) {
MetricDefineByFieldParams metricDefineParams = metric.getMetricDefineByFieldParams();
metricTypeParamsYamlTpl.setExpr(metricDefineParams.getExpr());
List<FieldParam> fields = metricDefineParams.getFields();
metricTypeParamsYamlTpl.setFields(
fields.stream().map(MetricYamlManager::convert).collect(Collectors.toList()));
} else if (MetricDefineType.METRIC.equals(metric.getMetricDefineType())) {
MetricDefineByMetricParams metricDefineByMetricParams = metric.getMetricDefineByMetricParams();
metricTypeParamsYamlTpl.setExpr(metricDefineByMetricParams.getExpr());
List<MetricParam> metrics = metricDefineByMetricParams.getMetrics();
metricTypeParamsYamlTpl.setMetrics(
metrics.stream().map(MetricYamlManager::convert).collect(Collectors.toList()));
}
metricYamlTpl.setTypeParams(metricTypeParamsYamlTpl);
return metricYamlTpl;
}
public static MeasureYamlTpl convert(MeasureParam measure) {
MeasureYamlTpl measureYamlTpl = new MeasureYamlTpl();
measureYamlTpl.setName(measure.getBizName());
measureYamlTpl.setConstraint(measure.getConstraint());
measureYamlTpl.setAgg(measure.getAgg());
return measureYamlTpl;
}
public static FieldParamYamlTpl convert(FieldParam fieldParam) {
FieldParamYamlTpl fieldParamYamlTpl = new FieldParamYamlTpl();
fieldParamYamlTpl.setFieldName(fieldParam.getFieldName());
return fieldParamYamlTpl;
}
public static MetricParamYamlTpl convert(MetricParam metricParam) {
MetricParamYamlTpl metricParamYamlTpl = new MetricParamYamlTpl();
metricParamYamlTpl.setBizName(metricParam.getBizName());
metricParamYamlTpl.setId(metricParam.getId());
return metricParamYamlTpl;
}
}

102
headless/core/src/main/java/com/tencent/supersonic/headless/core/manager/ModelYamlManager.java
View File

@@ -1,102 +0,0 @@
package com.tencent.supersonic.headless.core.manager;
import com.tencent.supersonic.headless.api.enums.DatasourceQuery;
import com.tencent.supersonic.headless.api.enums.ModelSourceType;
import com.tencent.supersonic.headless.api.pojo.Dim;
import com.tencent.supersonic.headless.api.pojo.Identify;
import com.tencent.supersonic.headless.api.pojo.Measure;
import com.tencent.supersonic.headless.api.pojo.ModelDetail;
import com.tencent.supersonic.headless.api.response.DatabaseResp;
import com.tencent.supersonic.headless.api.response.ModelResp;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptor;
import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory;
import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionTimeTypeParamsTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.IdentifyYamlTpl;
import com.tencent.supersonic.headless.core.pojo.yaml.MeasureYamlTpl;
import com.tencent.supersonic.headless.core.utils.SysTimeDimensionBuilder;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.Objects;
import java.util.stream.Collectors;
/**
* manager to handle the model
*/
@Service
@Slf4j
public class ModelYamlManager {
public static DataModelYamlTpl convert2YamlObj(ModelResp modelResp, DatabaseResp databaseResp) {
ModelDetail modelDetail = modelResp.getModelDetail();
DbAdaptor engineAdaptor = DbAdaptorFactory.getEngineAdaptor(databaseResp.getType());
SysTimeDimensionBuilder.addSysTimeDimension(modelDetail.getDimensions(), engineAdaptor);
addInterCntMetric(modelResp.getBizName(), modelDetail);
DataModelYamlTpl dataModelYamlTpl = new DataModelYamlTpl();
dataModelYamlTpl.setType(databaseResp.getType());
BeanUtils.copyProperties(modelDetail, dataModelYamlTpl);
dataModelYamlTpl.setIdentifiers(modelDetail.getIdentifiers().stream().map(ModelYamlManager::convert)
.collect(Collectors.toList()));
dataModelYamlTpl.setDimensions(modelDetail.getDimensions().stream().map(ModelYamlManager::convert)
.collect(Collectors.toList()));
dataModelYamlTpl.setMeasures(modelDetail.getMeasures().stream().map(ModelYamlManager::convert)
.collect(Collectors.toList()));
dataModelYamlTpl.setName(modelResp.getBizName());
dataModelYamlTpl.setSourceId(modelResp.getDatabaseId());
dataModelYamlTpl.setModelSourceTypeEnum(ModelSourceType.of(modelResp.getSourceType()));
if (modelDetail.getQueryType().equalsIgnoreCase(DatasourceQuery.SQL_QUERY.getName())) {
dataModelYamlTpl.setSqlQuery(modelDetail.getSqlQuery());
} else {
dataModelYamlTpl.setTableQuery(modelDetail.getTableQuery());
}
dataModelYamlTpl.setFields(modelResp.getModelDetail().getFields());
return dataModelYamlTpl;
}
public static DimensionYamlTpl convert(Dim dim) {
DimensionYamlTpl dimensionYamlTpl = new DimensionYamlTpl();
BeanUtils.copyProperties(dim, dimensionYamlTpl);
dimensionYamlTpl.setName(dim.getBizName());
if (Objects.isNull(dimensionYamlTpl.getExpr())) {
dimensionYamlTpl.setExpr(dim.getBizName());
}
if (dim.getTypeParams() != null) {
DimensionTimeTypeParamsTpl dimensionTimeTypeParamsTpl = new DimensionTimeTypeParamsTpl();
dimensionTimeTypeParamsTpl.setIsPrimary(dim.getTypeParams().getIsPrimary());
dimensionTimeTypeParamsTpl.setTimeGranularity(dim.getTypeParams().getTimeGranularity());
dimensionYamlTpl.setTypeParams(dimensionTimeTypeParamsTpl);
}
return dimensionYamlTpl;
}
public static MeasureYamlTpl convert(Measure measure) {
MeasureYamlTpl measureYamlTpl = new MeasureYamlTpl();
BeanUtils.copyProperties(measure, measureYamlTpl);
measureYamlTpl.setName(measure.getBizName());
return measureYamlTpl;
}
public static IdentifyYamlTpl convert(Identify identify) {
IdentifyYamlTpl identifyYamlTpl = new IdentifyYamlTpl();
identifyYamlTpl.setName(identify.getBizName());
identifyYamlTpl.setType(identify.getType());
return identifyYamlTpl;
}
private static void addInterCntMetric(String datasourceEnName, ModelDetail datasourceDetail) {
Measure measure = new Measure();
measure.setExpr("1");
if (!CollectionUtils.isEmpty(datasourceDetail.getIdentifiers())) {
measure.setExpr(datasourceDetail.getIdentifiers().get(0).getBizName());
}
measure.setAgg("count");
measure.setBizName(String.format("%s_%s", datasourceEnName, "internal_cnt"));
measure.setIsCreateMetric(1);
datasourceDetail.getMeasures().add(measure);
}
}

153
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/DefaultQueryParser.java Normal file
View File

@@ -0,0 +1,153 @@
package com.tencent.supersonic.headless.core.parser;
import com.google.common.base.Strings;
import com.tencent.supersonic.common.util.StringUtil;
import com.tencent.supersonic.headless.api.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.request.SqlExecuteReq;
import com.tencent.supersonic.headless.core.parser.converter.HeadlessConverter;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.ComponentFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
@Component
@Slf4j
@Primary
public class DefaultQueryParser implements QueryParser {
public void parse(QueryStatement queryStatement) throws Exception {
QueryStructReq queryStructReq = queryStatement.getQueryStructReq();
if (Objects.isNull(queryStatement.getParseSqlReq())) {
queryStatement.setParseSqlReq(new ParseSqlReq());
}
if (Objects.isNull(queryStatement.getMetricReq())) {
queryStatement.setMetricReq(new MetricQueryReq());
}
log.info("SemanticConverter before [{}]", queryStructReq);
for (HeadlessConverter headlessConverter : ComponentFactory.getSemanticConverters()) {
if (headlessConverter.accept(queryStatement)) {
log.info("SemanticConverter accept [{}]", headlessConverter.getClass().getName());
headlessConverter.convert(queryStatement);
}
}
log.info("SemanticConverter after {} {} {}", queryStructReq, queryStatement.getParseSqlReq(),
queryStatement.getMetricReq());
if (!queryStatement.getParseSqlReq().getSql().isEmpty()) {
queryStatement = parser(queryStatement.getParseSqlReq(), queryStatement);
} else {
queryStatement.getMetricReq().setNativeQuery(queryStructReq.getQueryType().isNativeAggQuery());
queryStatement = parser(queryStatement);
}
if (Strings.isNullOrEmpty(queryStatement.getSql())
|| Strings.isNullOrEmpty(queryStatement.getSourceId())) {
throw new RuntimeException("parse Exception: " + queryStatement.getErrMsg());
}
String querySql =
Objects.nonNull(queryStatement.getEnableLimitWrapper()) && queryStatement.getEnableLimitWrapper()
? String.format(SqlExecuteReq.LIMIT_WRAPPER,
queryStatement.getSql())
: queryStatement.getSql();
queryStatement.setSql(querySql);
}
public QueryStatement parser(ParseSqlReq parseSqlReq, QueryStatement queryStatement) {
log.info("parser MetricReq [{}] ", parseSqlReq);
try {
if (!CollectionUtils.isEmpty(parseSqlReq.getTables())) {
List<String[]> tables = new ArrayList<>();
Boolean isSingleTable = parseSqlReq.getTables().size() == 1;
for (MetricTable metricTable : parseSqlReq.getTables()) {
QueryStatement metricTableSql = parserSql(metricTable, isSingleTable, parseSqlReq, queryStatement);
if (isSingleTable && Objects.nonNull(metricTableSql.getViewSimplifySql())
&& !metricTableSql.getViewSimplifySql().isEmpty()) {
queryStatement.setSql(metricTableSql.getViewSimplifySql());
queryStatement.setParseSqlReq(parseSqlReq);
return queryStatement;
}
tables.add(new String[]{metricTable.getAlias(), metricTableSql.getSql()});
}
if (!tables.isEmpty()) {
String sql = "";
if (parseSqlReq.isSupportWith()) {
sql = "with " + String.join(",",
tables.stream().map(t -> String.format("%s as (%s)", t[0], t[1])).collect(
Collectors.toList())) + "\n" + parseSqlReq.getSql();
} else {
sql = parseSqlReq.getSql();
for (String[] tb : tables) {
sql = StringUtils.replace(sql, tb[0],
"(" + tb[1] + ") " + (parseSqlReq.isWithAlias() ? "" : tb[0]), -1);
}
}
queryStatement.setSql(sql);
queryStatement.setParseSqlReq(parseSqlReq);
return queryStatement;
}
}
} catch (Exception e) {
log.error("physicalSql error {}", e);
queryStatement.setErrMsg(e.getMessage());
}
return queryStatement;
}
public QueryStatement parser(QueryStatement queryStatement) {
return parser(queryStatement, AggOption.getAggregation(queryStatement.getMetricReq().isNativeQuery()));
}
public QueryStatement parser(QueryStatement queryStatement, AggOption isAgg) {
MetricQueryReq metricQueryReq = queryStatement.getMetricReq();
log.info("parser metricQueryReq [{}] isAgg [{}]", metricQueryReq, isAgg);
if (metricQueryReq.getRootPath().isEmpty()) {
queryStatement.setErrMsg("rootPath empty");
return queryStatement;
}
try {
return ComponentFactory.getSqlParser().explain(queryStatement, isAgg);
} catch (Exception e) {
queryStatement.setErrMsg(e.getMessage());
log.error("parser error metricQueryReq[{}] error [{}]", metricQueryReq, e);
}
return queryStatement;
}
private QueryStatement parserSql(MetricTable metricTable, Boolean isSingleMetricTable, ParseSqlReq parseSqlReq,
QueryStatement queryStatement) throws Exception {
MetricQueryReq metricReq = new MetricQueryReq();
metricReq.setMetrics(metricTable.getMetrics());
metricReq.setDimensions(metricTable.getDimensions());
metricReq.setWhere(StringUtil.formatSqlQuota(metricTable.getWhere()));
metricReq.setNativeQuery(!AggOption.isAgg(metricTable.getAggOption()));
metricReq.setRootPath(parseSqlReq.getRootPath());
QueryStatement tableSql = new QueryStatement();
tableSql.setIsS2SQL(false);
tableSql.setMetricReq(metricReq);
tableSql.setMinMaxTime(queryStatement.getMinMaxTime());
tableSql.setEnableOptimize(queryStatement.getEnableOptimize());
tableSql.setModelIds(queryStatement.getModelIds());
tableSql.setSemanticModel(queryStatement.getSemanticModel());
if (isSingleMetricTable) {
tableSql.setViewSql(parseSqlReq.getSql());
tableSql.setViewAlias(metricTable.getAlias());
}
tableSql = parser(tableSql, metricTable.getAggOption());
if (!tableSql.isOk()) {
throw new Exception(String.format("parser table [%s] error [%s]", metricTable.getAlias(),
tableSql.getErrMsg()));
}
queryStatement.setSourceId(tableSql.getSourceId());
return tableSql;
}
}

150
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/QueryParser.java
View File

@@ -1,156 +1,12 @@
package com.tencent.supersonic.headless.core.parser;
import com.google.common.base.Strings;
import com.tencent.supersonic.common.util.StringUtil;
import com.tencent.supersonic.headless.api.enums.AggOption;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.api.request.SqlExecuteReq;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.ComponentFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
/**
* logical parse from ParseSqlReq or MetricReq
* A query parser takes semantic query request and generates SQL to be executed.
*/
@Component
@Slf4j
@Primary
public class QueryParser {
public interface QueryParser {
public QueryStatement parse(QueryStatement queryStatement) throws Exception {
QueryStructReq queryStructReq = queryStatement.getQueryStructReq();
if (Objects.isNull(queryStatement.getParseSqlReq())) {
queryStatement.setParseSqlReq(new ParseSqlReq());
}
if (Objects.isNull(queryStatement.getMetricReq())) {
queryStatement.setMetricReq(new MetricQueryReq());
}
log.info("SemanticConverter before [{}]", queryStructReq);
for (HeadlessConverter headlessConverter : ComponentFactory.getSemanticConverters()) {
if (headlessConverter.accept(queryStatement)) {
log.info("SemanticConverter accept [{}]", headlessConverter.getClass().getName());
headlessConverter.convert(queryStatement);
}
}
log.info("SemanticConverter after {} {} {}", queryStructReq, queryStatement.getParseSqlReq(),
queryStatement.getMetricReq());
if (!queryStatement.getParseSqlReq().getSql().isEmpty()) {
queryStatement = parser(queryStatement.getParseSqlReq(), queryStatement);
} else {
queryStatement.getMetricReq().setNativeQuery(queryStructReq.getQueryType().isNativeAggQuery());
queryStatement = parser(queryStatement);
}
if (Strings.isNullOrEmpty(queryStatement.getSql())
|| Strings.isNullOrEmpty(queryStatement.getSourceId())) {
throw new RuntimeException("parse Exception: " + queryStatement.getErrMsg());
}
String querySql =
Objects.nonNull(queryStatement.getEnableLimitWrapper()) && queryStatement.getEnableLimitWrapper()
? String.format(SqlExecuteReq.LIMIT_WRAPPER,
queryStatement.getSql())
: queryStatement.getSql();
queryStatement.setSql(querySql);
return queryStatement;
}
public QueryStatement parser(ParseSqlReq parseSqlReq, QueryStatement queryStatement) {
log.info("parser MetricReq [{}] ", parseSqlReq);
try {
if (!CollectionUtils.isEmpty(parseSqlReq.getTables())) {
List<String[]> tables = new ArrayList<>();
Boolean isSingleTable = parseSqlReq.getTables().size() == 1;
for (MetricTable metricTable : parseSqlReq.getTables()) {
QueryStatement metricTableSql = parserSql(metricTable, isSingleTable, parseSqlReq, queryStatement);
if (isSingleTable && Objects.nonNull(metricTableSql.getViewSimplifySql())
&& !metricTableSql.getViewSimplifySql().isEmpty()) {
queryStatement.setSql(metricTableSql.getViewSimplifySql());
queryStatement.setParseSqlReq(parseSqlReq);
return queryStatement;
}
tables.add(new String[]{metricTable.getAlias(), metricTableSql.getSql()});
}
if (!tables.isEmpty()) {
String sql = "";
if (parseSqlReq.isSupportWith()) {
sql = "with " + String.join(",",
tables.stream().map(t -> String.format("%s as (%s)", t[0], t[1])).collect(
Collectors.toList())) + "\n" + parseSqlReq.getSql();
} else {
sql = parseSqlReq.getSql();
for (String[] tb : tables) {
sql = StringUtils.replace(sql, tb[0],
"(" + tb[1] + ") " + (parseSqlReq.isWithAlias() ? "" : tb[0]), -1);
}
}
queryStatement.setSql(sql);
queryStatement.setParseSqlReq(parseSqlReq);
return queryStatement;
}
}
} catch (Exception e) {
log.error("physicalSql error {}", e);
queryStatement.setErrMsg(e.getMessage());
}
return queryStatement;
}
public QueryStatement parser(QueryStatement queryStatement) {
return parser(queryStatement, AggOption.getAggregation(queryStatement.getMetricReq().isNativeQuery()));
}
public QueryStatement parser(QueryStatement queryStatement, AggOption isAgg) {
MetricQueryReq metricQueryReq = queryStatement.getMetricReq();
log.info("parser metricQueryReq [{}] isAgg [{}]", metricQueryReq, isAgg);
if (metricQueryReq.getRootPath().isEmpty()) {
queryStatement.setErrMsg("rootPath empty");
return queryStatement;
}
try {
return ComponentFactory.getSqlParser().explain(queryStatement, isAgg);
} catch (Exception e) {
queryStatement.setErrMsg(e.getMessage());
log.error("parser error metricQueryReq[{}] error [{}]", metricQueryReq, e);
}
return queryStatement;
}
private QueryStatement parserSql(MetricTable metricTable, Boolean isSingleMetricTable, ParseSqlReq parseSqlReq,
QueryStatement queryStatement) throws Exception {
MetricQueryReq metricReq = new MetricQueryReq();
metricReq.setMetrics(metricTable.getMetrics());
metricReq.setDimensions(metricTable.getDimensions());
metricReq.setWhere(StringUtil.formatSqlQuota(metricTable.getWhere()));
metricReq.setNativeQuery(!AggOption.isAgg(metricTable.getAggOption()));
metricReq.setRootPath(parseSqlReq.getRootPath());
QueryStatement tableSql = new QueryStatement();
tableSql.setIsS2SQL(false);
tableSql.setMetricReq(metricReq);
tableSql.setMinMaxTime(queryStatement.getMinMaxTime());
tableSql.setEnableOptimize(queryStatement.getEnableOptimize());
tableSql.setModelIds(queryStatement.getModelIds());
tableSql.setSemanticModel(queryStatement.getSemanticModel());
if (isSingleMetricTable) {
tableSql.setViewSql(parseSqlReq.getSql());
tableSql.setViewAlias(metricTable.getAlias());
}
tableSql = parser(tableSql, metricTable.getAggOption());
if (!tableSql.isOk()) {
throw new Exception(String.format("parser table [%s] error [%s]", metricTable.getAlias(),
tableSql.getErrMsg()));
}
queryStatement.setSourceId(tableSql.getSourceId());
return tableSql;
}
void parse(QueryStatement queryStatement) throws Exception;
}

1
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/CalculateAggConverter.java
View File

@@ -10,7 +10,6 @@ import com.tencent.supersonic.headless.api.enums.EngineType;
import com.tencent.supersonic.headless.api.pojo.MetricTable;
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.core.parser.HeadlessConverter;
import com.tencent.supersonic.headless.core.pojo.Database;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;

1
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/DefaultDimValueConverter.java
View File

@@ -3,7 +3,6 @@ package com.tencent.supersonic.headless.core.parser.converter;
import com.tencent.supersonic.common.pojo.Filter;
import com.tencent.supersonic.common.pojo.enums.FilterOperatorEnum;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.core.parser.HeadlessConverter;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import lombok.extern.slf4j.Slf4j;

2
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/HeadlessConverter.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/HeadlessConverter.java
View File

@@ -1,4 +1,4 @@
package com.tencent.supersonic.headless.core.parser;
package com.tencent.supersonic.headless.core.parser.converter;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;

1
headless/core/src/main/java/com/tencent/supersonic/headless/core/parser/converter/ParserDefaultConverter.java
View File

@@ -4,7 +4,6 @@ import com.tencent.supersonic.common.pojo.ColumnOrder;
import com.tencent.supersonic.headless.api.pojo.Param;
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.core.parser.HeadlessConverter;
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;

32
headless/core/src/main/java/com/tencent/supersonic/headless/core/planner/DefaultQueryPlanner.java Normal file
View File

@@ -0,0 +1,32 @@
package com.tencent.supersonic.headless.core.planner;
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.ComponentFactory;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class DefaultQueryPlanner implements QueryPlanner {
public QueryExecutor plan(QueryStatement queryStatement) {
optimize(queryStatement);
return route(queryStatement);
}
private void optimize(QueryStatement queryStatement) {
for (QueryOptimizer queryOptimizer : ComponentFactory.getQueryOptimizers()) {
queryOptimizer.rewrite(queryStatement);
}
}
public QueryExecutor route(QueryStatement queryStatement) {
for (QueryExecutor queryExecutor : ComponentFactory.getQueryExecutors()) {
if (queryExecutor.accept(queryStatement)) {
return queryExecutor;
}
}
return null;
}
}

7
headless/core/src/main/java/com/tencent/supersonic/headless/core/planner/DetailQuery.java → headless/core/src/main/java/com/tencent/supersonic/headless/core/planner/DetailQueryOptimizer.java
View File

@@ -14,11 +14,12 @@ import java.util.stream.Collectors;
* Remove the default metric added by the system when the query only has dimensions
*/
@Slf4j
@Component("DetailQuery")
public class DetailQuery implements QueryOptimizer {
@Component("DetailQueryOptimizer")
public class DetailQueryOptimizer implements QueryOptimizer {
@Override
public void rewrite(QueryStructReq queryStructCmd, QueryStatement queryStatement) {
public void rewrite(QueryStatement queryStatement) {
QueryStructReq queryStructCmd = queryStatement.getQueryStructReq();
String sqlRaw = queryStatement.getSql().trim();
if (Strings.isNullOrEmpty(sqlRaw)) {
throw new RuntimeException("sql is empty or null");

5
headless/core/src/main/java/com/tencent/supersonic/headless/core/planner/QueryOptimizer.java
View File

@@ -1,11 +1,10 @@
package com.tencent.supersonic.headless.core.planner;
import com.tencent.supersonic.headless.api.request.QueryStructReq;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
/**
* the interface that rewrites QueryStatement with some optimization rules
* A query optimizer rewrites QueryStatement with a set of optimization rules
*/
public interface QueryOptimizer {
void rewrite(QueryStructReq queryStructCmd, QueryStatement queryStatement);
void rewrite(QueryStatement queryStatement);
}

32
headless/core/src/main/java/com/tencent/supersonic/headless/core/planner/QueryPlanner.java
View File

@@ -2,31 +2,13 @@ package com.tencent.supersonic.headless.core.planner;
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.ComponentFactory;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@Slf4j
@Component
public class QueryPlanner {
/**
* A query planner takes parsed QueryStatement and generates an optimized execution plan.
* It interacts with the optimizer to determine the most efficient way to execute the query.
*/
public interface QueryPlanner {
QueryExecutor plan(QueryStatement queryStatement);
public QueryExecutor plan(QueryStatement queryStatement) {
optimizer(queryStatement);
return route(queryStatement);
}
public void optimizer(QueryStatement queryStatement) {
for (QueryOptimizer queryOptimizer : ComponentFactory.getQueryOptimizers()) {
queryOptimizer.rewrite(queryStatement.getQueryStructReq(), queryStatement);
}
}
public QueryExecutor route(QueryStatement queryStatement) {
for (QueryExecutor queryExecutor : ComponentFactory.getQueryExecutors()) {
if (queryExecutor.accept(queryStatement)) {
return queryExecutor;
}
}
return null;
}
QueryExecutor route(QueryStatement queryStatement);
}

37
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/DataModelYamlTpl.java
View File

@@ -1,37 +0,0 @@
package com.tencent.supersonic.headless.core.pojo.yaml;
import com.tencent.supersonic.headless.api.enums.ModelSourceType;
import com.tencent.supersonic.headless.api.pojo.Field;
import lombok.Data;
import java.util.List;
@Data
public class DataModelYamlTpl {
private Long id;
private String name;
private Long sourceId;
private String type;
private String sqlQuery;
private String tableQuery;
private List<IdentifyYamlTpl> identifiers;
private List<DimensionYamlTpl> dimensions;
private List<MeasureYamlTpl> measures;
private List<Field> fields;
private ModelSourceType modelSourceTypeEnum;
}

13
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/DimensionTimeTypeParamsTpl.java
View File

@@ -1,13 +0,0 @@
package com.tencent.supersonic.headless.core.pojo.yaml;
import lombok.Data;
@Data
public class DimensionTimeTypeParamsTpl {
private String isPrimary;
private String timeGranularity;
}

27
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/DimensionYamlTpl.java
View File

@@ -1,27 +0,0 @@
package com.tencent.supersonic.headless.core.pojo.yaml;
import com.tencent.supersonic.common.pojo.enums.DataTypeEnums;
import java.util.List;
import lombok.Data;
@Data
public class DimensionYamlTpl {
private String name;
private String bizName;
private String owners;
private String type;
private String expr;
private DimensionTimeTypeParamsTpl typeParams;
private DataTypeEnums dataType;
private List<String> defaultValues;
}

10
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/FieldParamYamlTpl.java
View File

@@ -1,10 +0,0 @@
package com.tencent.supersonic.headless.core.pojo.yaml;
import lombok.Data;
@Data
public class FieldParamYamlTpl {
private String fieldName;
}

20
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/IdentifyYamlTpl.java
View File

@@ -1,20 +0,0 @@
package com.tencent.supersonic.headless.core.pojo.yaml;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@AllArgsConstructor
@NoArgsConstructor
public class IdentifyYamlTpl {
private String name;
/**
* 主键 primary 外键 foreign
*/
private String type;
}

25
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/MeasureYamlTpl.java
View File

@@ -1,25 +0,0 @@
package com.tencent.supersonic.headless.core.pojo.yaml;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@AllArgsConstructor
@NoArgsConstructor
public class MeasureYamlTpl {
private String name;
private String agg;
private String expr;
private String constraint;
private String alias;
private String createMetric;
}

12
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/MetricParamYamlTpl.java
View File

@@ -1,12 +0,0 @@
package com.tencent.supersonic.headless.core.pojo.yaml;
import lombok.Data;
@Data
public class MetricParamYamlTpl {
private Long id;
private String bizName;
}

19
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/MetricTypeParamsYamlTpl.java
View File

@@ -1,19 +0,0 @@
package com.tencent.supersonic.headless.core.pojo.yaml;
import lombok.Data;
import org.apache.commons.compress.utils.Lists;
import java.util.List;
@Data
public class MetricTypeParamsYamlTpl {
private List<MeasureYamlTpl> measures = Lists.newArrayList();
private List<MetricParamYamlTpl> metrics = Lists.newArrayList();
private List<FieldParamYamlTpl> fields = Lists.newArrayList();
private String expr;
}

21
headless/core/src/main/java/com/tencent/supersonic/headless/core/pojo/yaml/MetricYamlTpl.java
View File

@@ -1,21 +0,0 @@
package com.tencent.supersonic.headless.core.pojo.yaml;
import lombok.Data;
import java.util.List;
@Data
public class MetricYamlTpl {
private String name;
private List<String> owners;
private String type;
private MetricTypeParamsYamlTpl typeParams;
}

6
headless/core/src/main/java/com/tencent/supersonic/headless/core/utils/ComponentFactory.java
View File

@@ -3,9 +3,9 @@ package com.tencent.supersonic.headless.core.utils;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.headless.core.executor.JdbcExecutor;
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
import com.tencent.supersonic.headless.core.planner.DetailQuery;
import com.tencent.supersonic.headless.core.planner.DetailQueryOptimizer;
import com.tencent.supersonic.headless.core.planner.QueryOptimizer;
import com.tencent.supersonic.headless.core.parser.HeadlessConverter;
import com.tencent.supersonic.headless.core.parser.converter.HeadlessConverter;
import com.tencent.supersonic.headless.core.parser.SqlParser;
import com.tencent.supersonic.headless.core.parser.calcite.CalciteSqlParser;
import com.tencent.supersonic.headless.core.parser.converter.CalculateAggConverter;
@@ -74,7 +74,7 @@ public class ComponentFactory {
}
private static void initQueryOptimizer() {
queryOptimizers.put("DetailQuery", getBean("DetailQuery", DetailQuery.class));
queryOptimizers.put("DetailQueryOptimizer", getBean("DetailQueryOptimizer", DetailQueryOptimizer.class));
}
private static void initQueryExecutors() {