mirror of
https://github.com/tencentmusic/supersonic.git
synced 2025-12-20 06:34:55 +00:00
(improvement)(headless) Refactor HeadlessModel-related code, remove modelSchemaResps. (#647)
This commit is contained in:
lexluo09
@@ -11,14 +11,14 @@ import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataType;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Dimension;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DimensionTimeTypeParams;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.HeadlessModel;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Identify;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.JoinRelation;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Materialization.TimePartType;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Measure;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Metric;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.MetricTypeParams;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.schema.HeadlessSchema;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.schema.SemanticSchema;
|
||||
import com.tencent.supersonic.headless.core.pojo.yaml.DataModelYamlTpl;
|
||||
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionTimeTypeParamsTpl;
|
||||
import com.tencent.supersonic.headless.core.pojo.yaml.DimensionYamlTpl;
|
||||
@@ -55,21 +55,21 @@ import java.util.stream.Collectors;
|
||||
public class HeadlessSchemaManager {
|
||||
|
||||
@Autowired
|
||||
private LoadingCache<String, HeadlessModel> loadingCache;
|
||||
private LoadingCache<String, SemanticModel> loadingCache;
|
||||
private final Catalog catalog;
|
||||
|
||||
public HeadlessSchemaManager(Catalog catalog) {
|
||||
this.catalog = catalog;
|
||||
}
|
||||
|
||||
public HeadlessModel reload(String rootPath) {
|
||||
HeadlessModel headlessModel = new HeadlessModel();
|
||||
headlessModel.setRootPath(rootPath);
|
||||
public SemanticModel reload(String rootPath) {
|
||||
SemanticModel semanticModel = new SemanticModel();
|
||||
semanticModel.setRootPath(rootPath);
|
||||
Set<Long> modelIds = Arrays.stream(rootPath.split(",")).map(s -> Long.parseLong(s.trim()))
|
||||
.collect(Collectors.toSet());
|
||||
if (modelIds.isEmpty()) {
|
||||
log.error("get modelIds empty {}", rootPath);
|
||||
return headlessModel;
|
||||
return semanticModel;
|
||||
}
|
||||
Map<String, List<DimensionYamlTpl>> dimensionYamlTpls = new HashMap<>();
|
||||
List<DataModelYamlTpl> dataModelYamlTpls = new ArrayList<>();
|
||||
@@ -77,33 +77,33 @@ public class HeadlessSchemaManager {
|
||||
Map<Long, String> modelIdName = new HashMap<>();
|
||||
catalog.getModelYamlTplByModelIds(modelIds, dimensionYamlTpls, dataModelYamlTpls, metricYamlTpls, modelIdName);
|
||||
DatabaseResp databaseResp = catalog.getDatabaseByModelId(modelIds.iterator().next());
|
||||
headlessModel.setDatabaseResp(databaseResp);
|
||||
semanticModel.setDatabaseResp(databaseResp);
|
||||
List<ModelRela> modelRelas = catalog.getModelRela(new ArrayList<>(modelIds));
|
||||
if (!CollectionUtils.isEmpty(modelRelas)) {
|
||||
headlessModel.setJoinRelations(getJoinRelation(modelRelas, modelIdName));
|
||||
semanticModel.setJoinRelations(getJoinRelation(modelRelas, modelIdName));
|
||||
}
|
||||
if (!dataModelYamlTpls.isEmpty()) {
|
||||
Map<String, DataSource> dataSourceMap = dataModelYamlTpls.stream().map(HeadlessSchemaManager::getDatasource)
|
||||
.collect(Collectors.toMap(DataSource::getName, item -> item, (k1, k2) -> k1));
|
||||
headlessModel.setDatasourceMap(dataSourceMap);
|
||||
semanticModel.setDatasourceMap(dataSourceMap);
|
||||
}
|
||||
if (!dimensionYamlTpls.isEmpty()) {
|
||||
Map<String, List<Dimension>> dimensionMap = new HashMap<>();
|
||||
for (Map.Entry<String, List<DimensionYamlTpl>> entry : dimensionYamlTpls.entrySet()) {
|
||||
dimensionMap.put(entry.getKey(), getDimensions(entry.getValue()));
|
||||
}
|
||||
headlessModel.setDimensionMap(dimensionMap);
|
||||
semanticModel.setDimensionMap(dimensionMap);
|
||||
}
|
||||
if (!metricYamlTpls.isEmpty()) {
|
||||
headlessModel.setMetrics(getMetrics(metricYamlTpls));
|
||||
semanticModel.setMetrics(getMetrics(metricYamlTpls));
|
||||
}
|
||||
return headlessModel;
|
||||
return semanticModel;
|
||||
}
|
||||
|
||||
//private Map<String, SemanticSchema> semanticSchemaMap = new HashMap<>();
|
||||
public HeadlessModel get(String rootPath) throws Exception {
|
||||
public SemanticModel get(String rootPath) throws Exception {
|
||||
rootPath = formatKey(rootPath);
|
||||
HeadlessModel schema = loadingCache.get(rootPath);
|
||||
SemanticModel schema = loadingCache.get(rootPath);
|
||||
if (schema == null) {
|
||||
return null;
|
||||
}
|
||||
@@ -182,6 +182,8 @@ public class HeadlessSchemaManager {
|
||||
dimension.setExpr(dimensionYamlTpl.getExpr());
|
||||
dimension.setName(dimensionYamlTpl.getName());
|
||||
dimension.setOwners(dimensionYamlTpl.getOwners());
|
||||
dimension.setBizName(dimensionYamlTpl.getBizName());
|
||||
dimension.setDefaultValues(dimensionYamlTpl.getDefaultValues());
|
||||
if (Objects.nonNull(dimensionYamlTpl.getDataType())) {
|
||||
dimension.setDataType(DataType.of(dimensionYamlTpl.getDataType().getType()));
|
||||
}
|
||||
@@ -235,13 +237,13 @@ public class HeadlessSchemaManager {
|
||||
return joinRelations;
|
||||
}
|
||||
|
||||
public static void update(HeadlessSchema schema, List<Metric> metric) throws Exception {
|
||||
public static void update(SemanticSchema schema, List<Metric> metric) throws Exception {
|
||||
if (schema != null) {
|
||||
updateMetric(metric, schema.getMetrics());
|
||||
}
|
||||
}
|
||||
|
||||
public static void update(HeadlessSchema schema, DataSource datasourceYamlTpl) throws Exception {
|
||||
public static void update(SemanticSchema schema, DataSource datasourceYamlTpl) throws Exception {
|
||||
if (schema != null) {
|
||||
String dataSourceName = datasourceYamlTpl.getName();
|
||||
Optional<Entry<String, DataSource>> datasourceYamlTplMap = schema.getDatasource().entrySet().stream()
|
||||
@@ -254,7 +256,7 @@ public class HeadlessSchemaManager {
|
||||
}
|
||||
}
|
||||
|
||||
public static void update(HeadlessSchema schema, String datasourceBizName, List<Dimension> dimensionYamlTpls)
|
||||
public static void update(SemanticSchema schema, String datasourceBizName, List<Dimension> dimensionYamlTpls)
|
||||
throws Exception {
|
||||
if (schema != null) {
|
||||
Optional<Map.Entry<String, List<Dimension>>> datasourceYamlTplMap = schema.getDimension().entrySet()
|
||||
@@ -320,15 +322,15 @@ public class HeadlessSchemaManager {
|
||||
private Integer maximumSize = 1000;
|
||||
|
||||
@Bean
|
||||
public LoadingCache<String, HeadlessModel> getCache() {
|
||||
LoadingCache<String, HeadlessModel> cache
|
||||
public LoadingCache<String, SemanticModel> getCache() {
|
||||
LoadingCache<String, SemanticModel> cache
|
||||
= CacheBuilder.newBuilder()
|
||||
.expireAfterWrite(saveMinutes, TimeUnit.MINUTES)
|
||||
.initialCapacity(10)
|
||||
.maximumSize(maximumSize).build(
|
||||
new CacheLoader<String, HeadlessModel>() {
|
||||
new CacheLoader<String, SemanticModel>() {
|
||||
@Override
|
||||
public HeadlessModel load(String key) {
|
||||
public SemanticModel load(String key) {
|
||||
log.info("load SemanticSchema [{}]", key);
|
||||
return HeadlessSchemaManager.this.reload(key);
|
||||
}
|
||||
|
||||
@@ -4,39 +4,34 @@ import com.tencent.supersonic.common.pojo.ItemDateResp;
|
||||
import com.tencent.supersonic.headless.api.request.MetricQueryReq;
|
||||
import com.tencent.supersonic.headless.api.request.ParseSqlReq;
|
||||
import com.tencent.supersonic.headless.api.request.QueryStructReq;
|
||||
import com.tencent.supersonic.headless.api.response.ModelSchemaResp;
|
||||
import com.tencent.supersonic.headless.api.response.QueryResultWithSchemaResp;
|
||||
import com.tencent.supersonic.headless.core.optimizer.QueryOptimizer;
|
||||
import com.tencent.supersonic.headless.core.parser.QueryParser;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.HeadlessModel;
|
||||
import com.tencent.supersonic.headless.core.parser.calcite.s2sql.SemanticModel;
|
||||
import com.tencent.supersonic.headless.core.pojo.QueryStatement;
|
||||
import com.tencent.supersonic.headless.core.utils.ComponentFactory;
|
||||
import com.tencent.supersonic.headless.core.executor.QueryExecutor;
|
||||
import com.tencent.supersonic.headless.server.manager.HeadlessSchemaManager;
|
||||
import com.tencent.supersonic.headless.server.service.Catalog;
|
||||
import com.tencent.supersonic.headless.server.service.HeadlessQueryEngine;
|
||||
import com.tencent.supersonic.headless.server.utils.QueryStructUtils;
|
||||
import com.tencent.supersonic.headless.server.utils.QueryUtils;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.CollectionUtils;
|
||||
import java.util.List;
|
||||
|
||||
@Slf4j
|
||||
@Component
|
||||
public class HeadlessQueryEngineImpl implements HeadlessQueryEngine {
|
||||
|
||||
private final QueryParser queryParser;
|
||||
private final Catalog catalog;
|
||||
private final QueryUtils queryUtils;
|
||||
private final QueryStructUtils queryStructUtils;
|
||||
private final HeadlessSchemaManager headlessSchemaManager;
|
||||
|
||||
public HeadlessQueryEngineImpl(QueryParser queryParser, Catalog catalog,
|
||||
public HeadlessQueryEngineImpl(QueryParser queryParser,
|
||||
QueryUtils queryUtils, HeadlessSchemaManager headlessSchemaManager,
|
||||
QueryStructUtils queryStructUtils) {
|
||||
this.queryParser = queryParser;
|
||||
this.catalog = catalog;
|
||||
this.queryUtils = queryUtils;
|
||||
this.headlessSchemaManager = headlessSchemaManager;
|
||||
this.queryStructUtils = queryStructUtils;
|
||||
@@ -57,7 +52,7 @@ public class HeadlessQueryEngineImpl implements HeadlessQueryEngine {
|
||||
|
||||
public QueryStatement plan(QueryStatement queryStatement) throws Exception {
|
||||
queryStatement.setEnableOptimize(queryUtils.enableOptimize());
|
||||
queryStatement.setHeadlessModel(getHeadLessModel(queryStatement));
|
||||
queryStatement.setSemanticModel(getSemanticModel(queryStatement));
|
||||
queryStatement = queryParser.logicSql(queryStatement);
|
||||
queryUtils.checkSqlParse(queryStatement);
|
||||
queryStatement.setModelIds(queryStatement.getQueryStructReq().getModelIds());
|
||||
@@ -82,31 +77,31 @@ public class HeadlessQueryEngineImpl implements HeadlessQueryEngine {
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryStatement physicalSql(QueryStructReq queryStructCmd, ParseSqlReq sqlCommend) {
|
||||
public QueryStatement physicalSql(QueryStructReq queryStructCmd, ParseSqlReq sqlCommend) throws Exception {
|
||||
QueryStatement queryStatement = new QueryStatement();
|
||||
queryStatement.setQueryStructReq(queryStructCmd);
|
||||
queryStatement.setParseSqlReq(sqlCommend);
|
||||
queryStatement.setSql(sqlCommend.getSql());
|
||||
queryStatement.setIsS2SQL(true);
|
||||
queryStatement.setSemanticModel(getSemanticModel(queryStatement));
|
||||
return optimize(queryStructCmd, queryParser.parser(sqlCommend, queryStatement));
|
||||
}
|
||||
|
||||
public QueryStatement physicalSql(QueryStructReq queryStructCmd, MetricQueryReq metricCommand) {
|
||||
public QueryStatement physicalSql(QueryStructReq queryStructCmd, MetricQueryReq metricCommand) throws Exception {
|
||||
QueryStatement queryStatement = new QueryStatement();
|
||||
queryStatement.setQueryStructReq(queryStructCmd);
|
||||
queryStatement.setMetricReq(metricCommand);
|
||||
queryStatement.setIsS2SQL(false);
|
||||
queryStatement.setSemanticModel(getSemanticModel(queryStatement));
|
||||
return queryParser.parser(queryStatement);
|
||||
}
|
||||
|
||||
private HeadlessModel getHeadLessModel(QueryStatement queryStatement) throws Exception {
|
||||
private SemanticModel getSemanticModel(QueryStatement queryStatement) throws Exception {
|
||||
QueryStructReq queryStructReq = queryStatement.getQueryStructReq();
|
||||
HeadlessModel headlessModel = headlessSchemaManager.get(queryStructReq.getModelIdStr());
|
||||
SemanticModel semanticModel = headlessSchemaManager.get(queryStructReq.getModelIdStr());
|
||||
ItemDateResp itemDateResp = queryStructUtils.getItemDateResp(queryStructReq);
|
||||
headlessModel.setDataDate(itemDateResp);
|
||||
List<ModelSchemaResp> modelSchemaResps = catalog.getModelSchema(queryStructReq.getModelIds());
|
||||
headlessModel.setModelSchemaResps(modelSchemaResps);
|
||||
return headlessModel;
|
||||
semanticModel.setDataDate(itemDateResp);
|
||||
return semanticModel;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user