[improvement][headless]Clean code logic of headless core.

This commit is contained in:
jerryjzhang
2024-11-19 20:13:49 +08:00
parent f4f0e58bfb
commit 62a4d60a0b
14 changed files with 96 additions and 101 deletions

View File

@@ -38,7 +38,7 @@ public class JdbcExecutor implements QueryExecutor {
SqlUtils sqlUtils = ContextUtils.getBean(SqlUtils.class); SqlUtils sqlUtils = ContextUtils.getBean(SqlUtils.class);
String sql = StringUtils.normalizeSpace(queryStatement.getSql()); String sql = StringUtils.normalizeSpace(queryStatement.getSql());
log.info("executing SQL: {}", sql); log.info("executing SQL: {}", sql);
Database database = queryStatement.getSemanticModel().getDatabase(); Database database = queryStatement.getOntology().getDatabase();
SemanticQueryResp queryResultWithColumns = new SemanticQueryResp(); SemanticQueryResp queryResultWithColumns = new SemanticQueryResp();
try { try {
SqlUtils sqlUtil = sqlUtils.init(database); SqlUtils sqlUtil = sqlUtils.init(database);

View File

@@ -2,7 +2,7 @@ package com.tencent.supersonic.headless.core.pojo;
import com.tencent.supersonic.headless.api.pojo.QueryParam; import com.tencent.supersonic.headless.api.pojo.QueryParam;
import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.SemanticModel; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Ontology;
import lombok.Data; import lombok.Data;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.ImmutablePair;
@@ -29,7 +29,7 @@ public class QueryStatement {
private String dataSetAlias; private String dataSetAlias;
private String dataSetSimplifySql; private String dataSetSimplifySql;
private Boolean enableLimitWrapper = false; private Boolean enableLimitWrapper = false;
private SemanticModel semanticModel; private Ontology ontology;
private SemanticSchemaResp semanticSchemaResp; private SemanticSchemaResp semanticSchemaResp;
private Integer limit = 1000; private Integer limit = 1000;
private Boolean isTranslated = false; private Boolean isTranslated = false;

View File

@@ -10,7 +10,7 @@ import com.tencent.supersonic.headless.api.pojo.enums.AggOption;
import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam; import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam;
import com.tencent.supersonic.headless.core.pojo.MetricQueryParam; import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.SemanticModel; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Ontology;
import com.tencent.supersonic.headless.core.translator.converter.QueryConverter; import com.tencent.supersonic.headless.core.translator.converter.QueryConverter;
import com.tencent.supersonic.headless.core.utils.ComponentFactory; import com.tencent.supersonic.headless.core.utils.ComponentFactory;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@@ -81,8 +81,8 @@ public class DefaultSemanticTranslator implements SemanticTranslator {
private QueryStatement doParse(DataSetQueryParam dataSetQueryParam, private QueryStatement doParse(DataSetQueryParam dataSetQueryParam,
QueryStatement queryStatement) { QueryStatement queryStatement) {
log.info("parse dataSetQuery [{}] ", dataSetQueryParam); log.info("parse dataSetQuery [{}] ", dataSetQueryParam);
SemanticModel semanticModel = queryStatement.getSemanticModel(); Ontology ontology = queryStatement.getOntology();
EngineType engineType = EngineType.fromString(semanticModel.getDatabase().getType()); EngineType engineType = EngineType.fromString(ontology.getDatabase().getType());
try { try {
if (!CollectionUtils.isEmpty(dataSetQueryParam.getTables())) { if (!CollectionUtils.isEmpty(dataSetQueryParam.getTables())) {
List<String[]> tables = new ArrayList<>(); List<String[]> tables = new ArrayList<>();
@@ -158,7 +158,7 @@ public class DefaultSemanticTranslator implements SemanticTranslator {
tableSql.setMinMaxTime(queryStatement.getMinMaxTime()); tableSql.setMinMaxTime(queryStatement.getMinMaxTime());
tableSql.setEnableOptimize(queryStatement.getEnableOptimize()); tableSql.setEnableOptimize(queryStatement.getEnableOptimize());
tableSql.setDataSetId(queryStatement.getDataSetId()); tableSql.setDataSetId(queryStatement.getDataSetId());
tableSql.setSemanticModel(queryStatement.getSemanticModel()); tableSql.setOntology(queryStatement.getOntology());
if (isSingleMetricTable) { if (isSingleMetricTable) {
tableSql.setDataSetSql(dataSetQueryParam.getSql()); tableSql.setDataSetSql(dataSetQueryParam.getSql());
tableSql.setDataSetAlias(metricTable.getAlias()); tableSql.setDataSetAlias(metricTable.getAlias());

View File

@@ -7,7 +7,7 @@ import com.tencent.supersonic.headless.core.pojo.MetricQueryParam;
import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.translator.QueryParser; import com.tencent.supersonic.headless.core.translator.QueryParser;
import com.tencent.supersonic.headless.core.translator.calcite.planner.AggPlanner; import com.tencent.supersonic.headless.core.translator.calcite.planner.AggPlanner;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.SemanticModel; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Ontology;
import com.tencent.supersonic.headless.core.translator.calcite.schema.RuntimeOptions; import com.tencent.supersonic.headless.core.translator.calcite.schema.RuntimeOptions;
import com.tencent.supersonic.headless.core.translator.calcite.schema.S2SemanticSchema; import com.tencent.supersonic.headless.core.translator.calcite.schema.S2SemanticSchema;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@@ -25,16 +25,16 @@ public class CalciteQueryParser implements QueryParser {
@Override @Override
public void parse(QueryStatement queryStatement, AggOption isAgg) throws Exception { public void parse(QueryStatement queryStatement, AggOption isAgg) throws Exception {
MetricQueryParam metricReq = queryStatement.getMetricQueryParam(); MetricQueryParam metricReq = queryStatement.getMetricQueryParam();
SemanticModel semanticModel = queryStatement.getSemanticModel(); Ontology ontology = queryStatement.getOntology();
if (semanticModel == null) { if (ontology == null) {
queryStatement.setErrMsg("semanticSchema not found"); queryStatement.setErrMsg("semanticSchema not found");
return; return;
} }
queryStatement.setMetricQueryParam(metricReq); queryStatement.setMetricQueryParam(metricReq);
S2SemanticSchema semanticSchema = getSemanticSchema(semanticModel, queryStatement); S2SemanticSchema semanticSchema = getSemanticSchema(ontology, queryStatement);
AggPlanner aggPlanner = new AggPlanner(semanticSchema); AggPlanner aggPlanner = new AggPlanner(semanticSchema);
aggPlanner.plan(queryStatement, isAgg); aggPlanner.plan(queryStatement, isAgg);
EngineType engineType = EngineType.fromString(semanticModel.getDatabase().getType()); EngineType engineType = EngineType.fromString(ontology.getDatabase().getType());
queryStatement.setSql(aggPlanner.getSql(engineType)); queryStatement.setSql(aggPlanner.getSql(engineType));
if (Objects.nonNull(queryStatement.getEnableOptimize()) if (Objects.nonNull(queryStatement.getEnableOptimize())
&& queryStatement.getEnableOptimize() && queryStatement.getEnableOptimize()
@@ -52,15 +52,14 @@ public class CalciteQueryParser implements QueryParser {
} }
} }
private S2SemanticSchema getSemanticSchema(SemanticModel semanticModel, private S2SemanticSchema getSemanticSchema(Ontology ontology, QueryStatement queryStatement) {
QueryStatement queryStatement) {
S2SemanticSchema semanticSchema = S2SemanticSchema semanticSchema =
S2SemanticSchema.newBuilder(semanticModel.getSchemaKey()).build(); S2SemanticSchema.newBuilder(ontology.getSchemaKey()).build();
semanticSchema.setSemanticModel(semanticModel); semanticSchema.setSemanticModel(ontology);
semanticSchema.setDatasource(semanticModel.getDatasourceMap()); semanticSchema.setDatasource(ontology.getDatasourceMap());
semanticSchema.setDimension(semanticModel.getDimensionMap()); semanticSchema.setDimension(ontology.getDimensionMap());
semanticSchema.setMetric(semanticModel.getMetrics()); semanticSchema.setMetric(ontology.getMetrics());
semanticSchema.setJoinRelations(semanticModel.getJoinRelations()); semanticSchema.setJoinRelations(ontology.getJoinRelations());
semanticSchema.setRuntimeOptions( semanticSchema.setRuntimeOptions(
RuntimeOptions.builder().minMaxTime(queryStatement.getMinMaxTime()) RuntimeOptions.builder().minMaxTime(queryStatement.getMinMaxTime())
.enableOptimize(queryStatement.getEnableOptimize()).build()); .enableOptimize(queryStatement.getEnableOptimize()).build());

View File

@@ -109,7 +109,7 @@ public class AggPlanner implements Planner {
// build a parse Node // build a parse Node
parse(); parse();
// optimizer // optimizer
Database database = queryStatement.getSemanticModel().getDatabase(); Database database = queryStatement.getOntology().getDatabase();
EngineType engineType = EngineType.fromString(database.getType()); EngineType engineType = EngineType.fromString(database.getType());
optimize(engineType); optimize(engineType);
} }

View File

@@ -11,7 +11,7 @@ import java.util.Map;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@Data @Data
public class SemanticModel { public class Ontology {
private String schemaKey; private String schemaKey;
private List<Metric> metrics = new ArrayList<>(); private List<Metric> metrics = new ArrayList<>();

View File

@@ -5,7 +5,7 @@ import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Dimension;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.JoinRelation; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.JoinRelation;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Materialization; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Materialization;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.SemanticModel; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Ontology;
import org.apache.calcite.schema.Schema; import org.apache.calcite.schema.Schema;
import org.apache.calcite.schema.SchemaVersion; import org.apache.calcite.schema.SchemaVersion;
import org.apache.calcite.schema.Table; import org.apache.calcite.schema.Table;
@@ -21,7 +21,7 @@ public class S2SemanticSchema extends AbstractSchema {
private final Map<String, Table> tableMap; private final Map<String, Table> tableMap;
private SemanticModel semanticModel = new SemanticModel(); private Ontology ontology = new Ontology();
private List<JoinRelation> joinRelations; private List<JoinRelation> joinRelations;
@@ -40,12 +40,12 @@ public class S2SemanticSchema extends AbstractSchema {
return schemaKey; return schemaKey;
} }
public void setSemanticModel(SemanticModel semanticModel) { public void setSemanticModel(Ontology ontology) {
this.semanticModel = semanticModel; this.ontology = ontology;
} }
public SemanticModel getSemanticModel() { public Ontology getSemanticModel() {
return semanticModel; return ontology;
} }
@Override @Override
@@ -59,35 +59,35 @@ public class S2SemanticSchema extends AbstractSchema {
} }
public Map<String, DataModel> getDatasource() { public Map<String, DataModel> getDatasource() {
return semanticModel.getDatasourceMap(); return ontology.getDatasourceMap();
} }
public void setDatasource(Map<String, DataModel> datasource) { public void setDatasource(Map<String, DataModel> datasource) {
semanticModel.setDatasourceMap(datasource); ontology.setDatasourceMap(datasource);
} }
public Map<String, List<Dimension>> getDimension() { public Map<String, List<Dimension>> getDimension() {
return semanticModel.getDimensionMap(); return ontology.getDimensionMap();
} }
public void setDimension(Map<String, List<Dimension>> dimensions) { public void setDimension(Map<String, List<Dimension>> dimensions) {
semanticModel.setDimensionMap(dimensions); ontology.setDimensionMap(dimensions);
} }
public List<Metric> getMetrics() { public List<Metric> getMetrics() {
return semanticModel.getMetrics(); return ontology.getMetrics();
} }
public void setMetric(List<Metric> metric) { public void setMetric(List<Metric> metric) {
semanticModel.setMetrics(metric); ontology.setMetrics(metric);
} }
public void setMaterializationList(List<Materialization> materializationList) { public void setMaterializationList(List<Materialization> materializationList) {
semanticModel.setMaterializationList(materializationList); ontology.setMaterializationList(materializationList);
} }
public List<Materialization> getMaterializationList() { public List<Materialization> getMaterializationList() {
return semanticModel.getMaterializationList(); return ontology.getMaterializationList();
} }
public void setJoinRelations(List<JoinRelation> joinRelations) { public void setJoinRelations(List<JoinRelation> joinRelations) {

View File

@@ -97,7 +97,7 @@ public class CalculateAggConverter implements QueryConverter {
@Override @Override
public void convert(QueryStatement queryStatement) throws Exception { public void convert(QueryStatement queryStatement) throws Exception {
Database database = queryStatement.getSemanticModel().getDatabase(); Database database = queryStatement.getOntology().getDatabase();
DataSetQueryParam dataSetQueryParam = generateSqlCommend(queryStatement, DataSetQueryParam dataSetQueryParam = generateSqlCommend(queryStatement,
EngineType.fromString(database.getType().toUpperCase()), database.getVersion()); EngineType.fromString(database.getType().toUpperCase()), database.getVersion());
queryStatement.setDataSetQueryParam(dataSetQueryParam); queryStatement.setDataSetQueryParam(dataSetQueryParam);

View File

@@ -34,7 +34,7 @@ public class DefaultDimValueConverter implements QueryConverter {
@Override @Override
public void convert(QueryStatement queryStatement) { public void convert(QueryStatement queryStatement) {
List<Dimension> dimensions = queryStatement.getSemanticModel().getDimensions().stream() List<Dimension> dimensions = queryStatement.getOntology().getDimensions().stream()
.filter(dimension -> !CollectionUtils.isEmpty(dimension.getDefaultValues())) .filter(dimension -> !CollectionUtils.isEmpty(dimension.getDefaultValues()))
.collect(Collectors.toList()); .collect(Collectors.toList());
if (CollectionUtils.isEmpty(dimensions)) { if (CollectionUtils.isEmpty(dimensions)) {

View File

@@ -60,7 +60,7 @@ public class ParserDefaultConverter implements QueryConverter {
// support detail query // support detail query
if (queryParam.getQueryType().isNativeAggQuery() if (queryParam.getQueryType().isNativeAggQuery()
&& CollectionUtils.isEmpty(metricQueryParam.getMetrics())) { && CollectionUtils.isEmpty(metricQueryParam.getMetrics())) {
Map<Long, DataModel> modelMap = queryStatement.getSemanticModel().getModelMap(); Map<Long, DataModel> modelMap = queryStatement.getOntology().getModelMap();
for (Long modelId : modelMap.keySet()) { for (Long modelId : modelMap.keySet()) {
String modelBizName = modelMap.get(modelId).getName(); String modelBizName = modelMap.get(modelId).getName();
String internalMetricName = String internalMetricName =

View File

@@ -39,8 +39,8 @@ public class SqlVariableParseConverter implements QueryConverter {
SqlVariableParseUtils.parse(modelResp.getModelDetail().getSqlQuery(), SqlVariableParseUtils.parse(modelResp.getModelDetail().getSqlQuery(),
modelResp.getModelDetail().getSqlVariables(), modelResp.getModelDetail().getSqlVariables(),
queryStatement.getQueryParam().getParams()); queryStatement.getQueryParam().getParams());
DataModel dataModel = queryStatement.getSemanticModel().getDatasourceMap() DataModel dataModel =
.get(modelResp.getBizName()); queryStatement.getOntology().getDatasourceMap().get(modelResp.getBizName());
dataModel.setSqlQuery(sqlParsed); dataModel.setSqlQuery(sqlParsed);
} }
} }

View File

@@ -34,7 +34,7 @@ import com.tencent.supersonic.headless.core.cache.QueryCache;
import com.tencent.supersonic.headless.core.executor.QueryExecutor; import com.tencent.supersonic.headless.core.executor.QueryExecutor;
import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.translator.SemanticTranslator; import com.tencent.supersonic.headless.core.translator.SemanticTranslator;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.SemanticModel; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Ontology;
import com.tencent.supersonic.headless.core.utils.ComponentFactory; import com.tencent.supersonic.headless.core.utils.ComponentFactory;
import com.tencent.supersonic.headless.server.annotation.S2DataPermission; import com.tencent.supersonic.headless.server.annotation.S2DataPermission;
import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService; import com.tencent.supersonic.headless.server.facade.service.SemanticLayerService;
@@ -51,6 +51,7 @@ import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.util.ArrayList; import java.util.ArrayList;
@@ -271,23 +272,6 @@ public class S2SemanticLayerService implements SemanticLayerService {
return metricService.getMetrics(metaFilter); return metricService.getMetrics(metaFilter);
} }
private QueryStatement buildSqlQueryStatement(QuerySqlReq querySqlReq, User user)
throws Exception {
// If dataSetId or DataSetName is empty, parse dataSetId from the SQL
if (querySqlReq.needGetDataSetId()) {
Long dataSetId = dataSetService.getDataSetIdFromSql(querySqlReq.getSql(), user);
querySqlReq.setDataSetId(dataSetId);
}
SchemaFilterReq filter = buildSchemaFilterReq(querySqlReq);
SemanticSchemaResp semanticSchemaResp = schemaService.fetchSemanticSchema(filter);
QueryStatement queryStatement = queryReqConverter.convert(querySqlReq, semanticSchemaResp);
queryStatement.setModelIds(querySqlReq.getModelIds());
queryStatement.setEnableOptimize(queryUtils.enableOptimize());
queryStatement.setSemanticSchemaResp(semanticSchemaResp);
queryStatement.setSemanticModel(semanticSchemaManager.getSemanticModel(semanticSchemaResp));
return queryStatement;
}
private QueryStatement buildQueryStatement(SemanticQueryReq semanticQueryReq, User user) private QueryStatement buildQueryStatement(SemanticQueryReq semanticQueryReq, User user)
throws Exception { throws Exception {
QueryStatement queryStatement = null; QueryStatement queryStatement = null;
@@ -310,18 +294,29 @@ public class S2SemanticLayerService implements SemanticLayerService {
return queryStatement; return queryStatement;
} }
private QueryStatement buildSqlQueryStatement(QuerySqlReq querySqlReq, User user) {
// If dataSetId or DataSetName is empty, parse dataSetId from the SQL
if (querySqlReq.needGetDataSetId()) {
Long dataSetId = dataSetService.getDataSetIdFromSql(querySqlReq.getSql(), user);
querySqlReq.setDataSetId(dataSetId);
}
SchemaFilterReq filter = buildSchemaFilterReq(querySqlReq);
SemanticSchemaResp semanticSchemaResp = schemaService.fetchSemanticSchema(filter);
return queryReqConverter.buildQueryStatement(querySqlReq, semanticSchemaResp);
}
private QueryStatement buildStructQueryStatement(QueryStructReq queryStructReq) { private QueryStatement buildStructQueryStatement(QueryStructReq queryStructReq) {
SchemaFilterReq filter = buildSchemaFilterReq(queryStructReq); SchemaFilterReq filter = buildSchemaFilterReq(queryStructReq);
SemanticSchemaResp semanticSchemaResp = schemaService.fetchSemanticSchema(filter); SemanticSchemaResp semanticSchemaResp = schemaService.fetchSemanticSchema(filter);
QueryStatement queryStatement = new QueryStatement(); QueryStatement queryStatement = new QueryStatement();
QueryParam queryParam = new QueryParam(); QueryParam queryParam = new QueryParam();
queryReqConverter.convert(queryStructReq, queryParam); BeanUtils.copyProperties(queryStructReq, queryParam);
queryStatement.setQueryParam(queryParam); queryStatement.setQueryParam(queryParam);
queryStatement.setIsS2SQL(false); queryStatement.setIsS2SQL(false);
queryStatement.setEnableOptimize(queryUtils.enableOptimize()); queryStatement.setEnableOptimize(queryUtils.enableOptimize());
queryStatement.setDataSetId(queryStructReq.getDataSetId()); queryStatement.setDataSetId(queryStructReq.getDataSetId());
queryStatement.setSemanticSchemaResp(semanticSchemaResp); queryStatement.setSemanticSchemaResp(semanticSchemaResp);
queryStatement.setSemanticModel(semanticSchemaManager.getSemanticModel(semanticSchemaResp)); queryStatement.setOntology(semanticSchemaManager.buildOntology(semanticSchemaResp));
return queryStatement; return queryStatement;
} }
@@ -330,9 +325,9 @@ public class S2SemanticLayerService implements SemanticLayerService {
List<QueryStatement> sqlParsers = new ArrayList<>(); List<QueryStatement> sqlParsers = new ArrayList<>();
for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) { for (QueryStructReq queryStructReq : queryMultiStructReq.getQueryStructReqs()) {
QueryStatement queryStatement = buildQueryStatement(queryStructReq, user); QueryStatement queryStatement = buildQueryStatement(queryStructReq, user);
SemanticModel semanticModel = queryStatement.getSemanticModel(); Ontology ontology = queryStatement.getOntology();
queryStatement.setModelIds(queryStructReq.getModelIds()); queryStatement.setModelIds(queryStructReq.getModelIds());
queryStatement.setSemanticModel(semanticModel); queryStatement.setOntology(ontology);
queryStatement.setEnableOptimize(queryUtils.enableOptimize()); queryStatement.setEnableOptimize(queryUtils.enableOptimize());
semanticTranslator.translate(queryStatement); semanticTranslator.translate(queryStatement);
sqlParsers.add(queryStatement); sqlParsers.add(queryStatement);

View File

@@ -18,7 +18,7 @@ import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Materializa
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Measure; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Measure;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Metric;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.MetricTypeParams; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.MetricTypeParams;
import com.tencent.supersonic.headless.core.translator.calcite.s2sql.SemanticModel; import com.tencent.supersonic.headless.core.translator.calcite.s2sql.Ontology;
import com.tencent.supersonic.headless.core.translator.calcite.schema.S2SemanticSchema; import com.tencent.supersonic.headless.core.translator.calcite.schema.S2SemanticSchema;
import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl; import com.tencent.supersonic.headless.server.pojo.yaml.DataModelYamlTpl;
import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl; import com.tencent.supersonic.headless.server.pojo.yaml.DimensionTimeTypeParamsTpl;
@@ -57,9 +57,9 @@ public class SemanticSchemaManager {
this.schemaService = schemaService; this.schemaService = schemaService;
} }
public SemanticModel getSemanticModel(SemanticSchemaResp semanticSchemaResp) { public Ontology buildOntology(SemanticSchemaResp semanticSchemaResp) {
SemanticModel semanticModel = new SemanticModel(); Ontology ontology = new Ontology();
semanticModel.setSchemaKey(semanticSchemaResp.getSchemaKey()); ontology.setSchemaKey(semanticSchemaResp.getSchemaKey());
Map<String, List<DimensionYamlTpl>> dimensionYamlTpls = new HashMap<>(); Map<String, List<DimensionYamlTpl>> dimensionYamlTpls = new HashMap<>();
List<DataModelYamlTpl> dataModelYamlTpls = new ArrayList<>(); List<DataModelYamlTpl> dataModelYamlTpls = new ArrayList<>();
List<MetricYamlTpl> metricYamlTpls = new ArrayList<>(); List<MetricYamlTpl> metricYamlTpls = new ArrayList<>();
@@ -67,36 +67,35 @@ public class SemanticSchemaManager {
schemaService.getSchemaYamlTpl(semanticSchemaResp, dimensionYamlTpls, dataModelYamlTpls, schemaService.getSchemaYamlTpl(semanticSchemaResp, dimensionYamlTpls, dataModelYamlTpls,
metricYamlTpls, modelIdName); metricYamlTpls, modelIdName);
DatabaseResp databaseResp = semanticSchemaResp.getDatabaseResp(); DatabaseResp databaseResp = semanticSchemaResp.getDatabaseResp();
semanticModel.setDatabase(DatabaseConverter.convert(databaseResp)); ontology.setDatabase(DatabaseConverter.convert(databaseResp));
if (!CollectionUtils.isEmpty(semanticSchemaResp.getModelRelas())) { if (!CollectionUtils.isEmpty(semanticSchemaResp.getModelRelas())) {
semanticModel.setJoinRelations( ontology.setJoinRelations(
getJoinRelation(semanticSchemaResp.getModelRelas(), modelIdName)); getJoinRelation(semanticSchemaResp.getModelRelas(), modelIdName));
} }
if (!dataModelYamlTpls.isEmpty()) { if (!dataModelYamlTpls.isEmpty()) {
Map<String, DataModel> dataSourceMap = Map<String, DataModel> dataSourceMap =
dataModelYamlTpls.stream().map(SemanticSchemaManager::getDatasource).collect( dataModelYamlTpls.stream().map(SemanticSchemaManager::getDatasource).collect(
Collectors.toMap(DataModel::getName, item -> item, (k1, k2) -> k1)); Collectors.toMap(DataModel::getName, item -> item, (k1, k2) -> k1));
semanticModel.setDatasourceMap(dataSourceMap); ontology.setDatasourceMap(dataSourceMap);
} }
if (!dimensionYamlTpls.isEmpty()) { if (!dimensionYamlTpls.isEmpty()) {
Map<String, List<Dimension>> dimensionMap = new HashMap<>(); Map<String, List<Dimension>> dimensionMap = new HashMap<>();
for (Map.Entry<String, List<DimensionYamlTpl>> entry : dimensionYamlTpls.entrySet()) { for (Map.Entry<String, List<DimensionYamlTpl>> entry : dimensionYamlTpls.entrySet()) {
dimensionMap.put(entry.getKey(), getDimensions(entry.getValue())); dimensionMap.put(entry.getKey(), getDimensions(entry.getValue()));
} }
semanticModel.setDimensionMap(dimensionMap); ontology.setDimensionMap(dimensionMap);
} }
if (!metricYamlTpls.isEmpty()) { if (!metricYamlTpls.isEmpty()) {
semanticModel.setMetrics(getMetrics(metricYamlTpls)); ontology.setMetrics(getMetrics(metricYamlTpls));
} }
return semanticModel; return ontology;
} }
public SemanticModel getTagSemanticModel(SemanticSchemaResp semanticSchemaResp) public Ontology getTagSemanticModel(SemanticSchemaResp semanticSchemaResp) throws Exception {
throws Exception {
if (CollectionUtils.isEmpty(semanticSchemaResp.getTags())) { if (CollectionUtils.isEmpty(semanticSchemaResp.getTags())) {
throw new Exception("semanticSchemaResp tag is empty"); throw new Exception("semanticSchemaResp tag is empty");
} }
SemanticModel semanticModel = getSemanticModel(semanticSchemaResp); Ontology ontology = buildOntology(semanticSchemaResp);
// Map<String, List<Dimension>> dimensions = new HashMap<>(); // Map<String, List<Dimension>> dimensions = new HashMap<>();
Map<Long, List<TagResp>> tagMap = new HashMap<>(); Map<Long, List<TagResp>> tagMap = new HashMap<>();
for (TagResp tagResp : semanticSchemaResp.getTags()) { for (TagResp tagResp : semanticSchemaResp.getTags()) {
@@ -105,24 +104,24 @@ public class SemanticSchemaManager {
} }
tagMap.get(tagResp.getModelId()).add(tagResp); tagMap.get(tagResp.getModelId()).add(tagResp);
} }
if (Objects.nonNull(semanticModel.getDatasourceMap()) if (Objects.nonNull(ontology.getDatasourceMap())
&& !semanticModel.getDatasourceMap().isEmpty()) { && !ontology.getDatasourceMap().isEmpty()) {
for (Map.Entry<String, DataModel> entry : semanticModel.getDatasourceMap().entrySet()) { for (Map.Entry<String, DataModel> entry : ontology.getDatasourceMap().entrySet()) {
List<Dimension> modelDimensions = new ArrayList<>(); List<Dimension> modelDimensions = new ArrayList<>();
if (!semanticModel.getDimensionMap().containsKey(entry.getKey())) { if (!ontology.getDimensionMap().containsKey(entry.getKey())) {
semanticModel.getDimensionMap().put(entry.getKey(), modelDimensions); ontology.getDimensionMap().put(entry.getKey(), modelDimensions);
} else { } else {
modelDimensions = semanticModel.getDimensionMap().get(entry.getKey()); modelDimensions = ontology.getDimensionMap().get(entry.getKey());
} }
if (tagMap.containsKey(entry.getValue().getId())) { if (tagMap.containsKey(entry.getValue().getId())) {
for (TagResp tagResp : tagMap.get(entry.getValue().getId())) { for (TagResp tagResp : tagMap.get(entry.getValue().getId())) {
addTagModel(tagResp, modelDimensions, semanticModel.getMetrics()); addTagModel(tagResp, modelDimensions, ontology.getMetrics());
} }
} }
} }
} }
return semanticModel; return ontology;
} }
private void addTagModel(TagResp tagResp, List<Dimension> modelDimensions, private void addTagModel(TagResp tagResp, List<Dimension> modelDimensions,

View File

@@ -29,6 +29,7 @@ import com.tencent.supersonic.headless.core.adaptor.db.DbAdaptorFactory;
import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam; import com.tencent.supersonic.headless.core.pojo.DataSetQueryParam;
import com.tencent.supersonic.headless.core.pojo.QueryStatement; import com.tencent.supersonic.headless.core.pojo.QueryStatement;
import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils;
import com.tencent.supersonic.headless.server.manager.SemanticSchemaManager;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.lang3.tuple.Pair;
@@ -58,8 +59,14 @@ public class QueryReqConverter {
@Autowired @Autowired
private SqlGenerateUtils sqlGenerateUtils; private SqlGenerateUtils sqlGenerateUtils;
public QueryStatement convert(QuerySqlReq querySQLReq, SemanticSchemaResp semanticSchemaResp) @Autowired
throws Exception { private QueryUtils queryUtils;
@Autowired
private SemanticSchemaManager semanticSchemaManager;
public QueryStatement buildQueryStatement(QuerySqlReq querySQLReq,
SemanticSchemaResp semanticSchemaResp) {
if (semanticSchemaResp == null) { if (semanticSchemaResp == null) {
return new QueryStatement(); return new QueryStatement();
@@ -87,17 +94,14 @@ public class QueryReqConverter {
List<String> metrics = List<String> metrics =
metricSchemas.stream().map(m -> m.getBizName()).collect(Collectors.toList()); metricSchemas.stream().map(m -> m.getBizName()).collect(Collectors.toList());
QueryStructReq queryStructReq = new QueryStructReq(); QueryStructReq queryStructReq = new QueryStructReq();
MetricTable metricTable = new MetricTable(); MetricTable metricTable = new MetricTable();
metricTable.setMetrics(metrics); metricTable.getMetrics().addAll(metrics);
Set<String> dimensions = getDimensions(semanticSchemaResp, allFields); Set<String> dimensions = getDimensions(semanticSchemaResp, allFields);
metricTable.getDimensions().addAll(dimensions);
metricTable.setDimensions(new ArrayList<>(dimensions));
metricTable.setAlias(tableName.toLowerCase()); metricTable.setAlias(tableName.toLowerCase());
// if metric empty , fill model default // if metric empty , fill model default
if (CollectionUtils.isEmpty(metricTable.getMetrics())) { if (CollectionUtils.isEmpty(metricTable.getMetrics())) {
metricTable.setMetrics(new ArrayList<>());
metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName( metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName(
getDefaultModel(semanticSchemaResp, metricTable.getDimensions()))); getDefaultModel(semanticSchemaResp, metricTable.getDimensions())));
} else { } else {
@@ -122,14 +126,15 @@ public class QueryReqConverter {
} }
// 7. do deriveMetric // 7. do deriveMetric
generateDerivedMetric(semanticSchemaResp, aggOption, result); generateDerivedMetric(semanticSchemaResp, aggOption, result);
// 8.physicalSql by ParseSqlReq
// 8.physicalSql by ParseSqlReq
queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySQLReq.getSql())); queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySQLReq.getSql()));
queryStructReq.setDataSetId(querySQLReq.getDataSetId()); queryStructReq.setDataSetId(querySQLReq.getDataSetId());
queryStructReq.setQueryType(getQueryType(aggOption)); queryStructReq.setQueryType(getQueryType(aggOption));
log.debug("QueryReqConverter queryStructReq[{}]", queryStructReq); log.debug("QueryReqConverter queryStructReq[{}]", queryStructReq);
QueryParam queryParam = new QueryParam(); QueryParam queryParam = new QueryParam();
convert(queryStructReq, queryParam); BeanUtils.copyProperties(queryStructReq, queryParam);
QueryStatement queryStatement = new QueryStatement(); QueryStatement queryStatement = new QueryStatement();
queryStatement.setQueryParam(queryParam); queryStatement.setQueryParam(queryParam);
queryStatement.setDataSetQueryParam(result); queryStatement.setDataSetQueryParam(result);
@@ -137,17 +142,14 @@ public class QueryReqConverter {
queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq)); queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq));
queryStatement.setDataSetId(querySQLReq.getDataSetId()); queryStatement.setDataSetId(querySQLReq.getDataSetId());
queryStatement.setLimit(querySQLReq.getLimit()); queryStatement.setLimit(querySQLReq.getLimit());
queryStatement.setModelIds(querySQLReq.getModelIds());
queryStatement.setEnableOptimize(queryUtils.enableOptimize());
queryStatement.setSemanticSchemaResp(semanticSchemaResp);
queryStatement.setOntology(semanticSchemaManager.buildOntology(semanticSchemaResp));
return queryStatement; return queryStatement;
} }
public void convert(QueryStructReq queryStructReq, QueryParam queryParam) {
BeanUtils.copyProperties(queryStructReq, queryParam);
queryParam.setOrders(queryStructReq.getOrders());
queryParam.setMetrics(queryStructReq.getMetrics());
queryParam.setGroups(queryStructReq.getGroups());
}
private AggOption getAggOption(QuerySqlReq databaseReq, List<MetricSchemaResp> metricSchemas) { private AggOption getAggOption(QuerySqlReq databaseReq, List<MetricSchemaResp> metricSchemas) {
String sql = databaseReq.getSql(); String sql = databaseReq.getSql();
if (!SqlSelectFunctionHelper.hasAggregateFunction(sql) && !SqlSelectHelper.hasGroupBy(sql) if (!SqlSelectFunctionHelper.hasAggregateFunction(sql) && !SqlSelectHelper.hasGroupBy(sql)